FE

conf/configuration.yml

@@ -5,3 +5,10 @@ SERVICE_CONFIG:
   LOGSTASH_HOST: 192.168.1.47
   LOGSTASH_PORT: 5000
 
+#----------------------If read conf from mongo------------#
+FOR_EACH_MONGO_CONFIG:
+  READ_FROM_MONGO: true
+  MONGO_URI: mongodb://192.168.0.210:27017
+  MONGO_DB: iLensAiPipeline
+  MONGO_RUN_COLL: runMetadata
+  MONGO_SITE_COLL: siteMetadata

logs/HttpRequestComponent.log

@@ -111,3 +111,13 @@
 2021-02-22 16:48:36,796  INFO HttpRequestComponent {'shared_volume': 'test', 'method': 'GET', 'url': 'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/aluva?unitGroup=metric&key=QMCRVJEPKK8YD7BP83BZTMZ7A', 'headers': {}, 'payload': {}, 'max_retries': 1, 'authentication': {}}
 2021-02-22 16:48:37,991  INFO HttpRequestComponent Receiving data...
 2021-02-22 16:48:38,001  INFO HttpRequestComponent Writing data to response.json file
+2021-02-25 11:51:26,721  INFO HttpRequestComponent Parsing user requests
+2021-02-25 11:51:26,798  INFO HttpRequestComponent Proceeding without authentication because username and password are missing
+2021-02-25 11:51:26,798  INFO HttpRequestComponent {'json_filename': 'response.json', 'shared_volume': '/mnt/ilens', 'method': 'GET', 'url': 'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/aluva?unitGroup=metric&key=QMCRVJEPKK8YD7BP83BZTMZ7A', 'headers': {'content-type': 'application/json'}, 'payload': {}, 'max_retries': 1, 'authentication': {}}
+2021-02-25 11:51:28,892  INFO HttpRequestComponent Receiving data...
+2021-02-25 11:51:28,897  INFO HttpRequestComponent Writing data to response.json file
+2021-02-25 11:52:13,453  INFO HttpRequestComponent Parsing user requests
+2021-02-25 11:52:13,453  INFO HttpRequestComponent Proceeding without authentication because username and password are missing
+2021-02-25 11:52:13,453  INFO HttpRequestComponent {'json_filename': 'response.json', 'shared_volume': 'test', 'method': 'GET', 'url': 'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/aluva?unitGroup=metric&key=QMCRVJEPKK8YD7BP83BZTMZ7A', 'headers': {'content-type': 'application/json'}, 'payload': {}, 'max_retries': 1, 'authentication': {}}
+2021-02-25 11:52:14,863  INFO HttpRequestComponent Receiving data...
+2021-02-25 11:52:14,867  INFO HttpRequestComponent Writing data to response.json file

main.py

@@ -91,7 +91,7 @@ class RequestComponent:
 if __name__ == '__main__':
     obj = RequestComponent()
     data = obj.execute_request(config)
-    logger.info("Writing data to response.json file")
+    logger.info("Writing data to " + os.path.join(config['shared_volume'], config['json_filename']))
     json_object = json.dumps(data, indent=4)
-    with open(os.path.join(config['shared_volume'], 'response.json'), "w") as outfile:
+    with open(os.path.join(config['shared_volume'], config['json_filename']), "w") as outfile:
         outfile.write(json_object)

scripts/common/config_parser.py

@@ -3,6 +3,8 @@ import os
 import sys
 import yaml
 import json
+from pymongo import MongoClient, DESCENDING
+from scripts.common.constants import RequestConstants
 
 config_path = os.path.join(os.getcwd(), "conf", "configuration.yml")
 if os.path.exists(config_path):
@@ -16,6 +18,63 @@ else:
     sys.stderr.write("Exiting....")
     sys.exit(1)
 
+# ----------------- Mongo -----------------------------------------------------------------------
+
+READ_FROM_MONGO = _config.get("FOR_EACH_MONGO_CONFIG", {}).get('READ_FROM_MONGO', False)
+COMPONENT_NAME = os.environ.get("type", RequestConstants.COMPONENT_NAME)
+pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
+
+if READ_FROM_MONGO:
+    MONGO_URI = os.environ.get("MONGO_URI", _config.get("FOR_EACH_MONGO_CONFIG", {}).get('MONGO_URI'))
+    MONGO_DB = os.environ.get("MONGO_DB", _config.get("FOR_EACH_MONGO_CONFIG", {}).get('MONGO_DB'))
+    MONGO_RUN_COLL = _config.get("FOR_EACH_MONGO_CONFIG", {}).get('MONGO_RUN_COLL')
+    MONGO_SITE_COLL = _config.get("FOR_EACH_MONGO_CONFIG", {}).get('MONGO_SITE_COLL')
+
+    db = MongoClient(MONGO_URI)[MONGO_DB]
+    get_run_info = db[MONGO_RUN_COLL].find_one({}, sort=[("run_start_time", DESCENDING)])
+
+    if get_run_info is None:
+        raise Exception('No run info found')
+
+    if not get_run_info['job_metadata']['in_progress']:
+        raise Exception('No job in progress')
+
+    _tag_hierarchy = get_run_info['job_metadata']['in_progress'][0]
+    sys.stdout.write(f"_tag_hierarchy --> {_tag_hierarchy}\n")
+    _tag_hierarchy = _tag_hierarchy.split('$')
+    site_id = _tag_hierarchy[0]
+    dept_id = _tag_hierarchy[1]
+    line_id = _tag_hierarchy[2]
+    equipment_id = _tag_hierarchy[3]
+    tag_id = _tag_hierarchy[4]
+
+    get_conf = db[MONGO_SITE_COLL].find_one({'site_id': site_id})
+
+    dept_dict = list(filter(lambda x: x['dept_id'] == dept_id, get_conf['dept']))
+    line_dict = list(filter(lambda x: x['line_id'] == line_id, dept_dict[0]['line']))
+    equipment_dict = list(filter(lambda x: x['equipment_id'] == equipment_id, line_dict[0]['equipment']))
+    tag_dict = list(filter(lambda x: x['tag_id'] == tag_id, equipment_dict[0]['tag']))
+
+    if len(tag_dict) != 1:
+        raise Exception(f"Tag details not found for hierarchy {_tag_hierarchy}")
+    pipeline_conf = list(filter(lambda x: x['pipeline_id'] == pipeline_id, tag_dict[0]['pipeline_config']))
+    if len(pipeline_conf) != 1:
+        raise Exception(
+            f"Tag details not found for hierarchy {'$'.join(_tag_hierarchy)} not found for pipeline {pipeline_id}")
+    component_conf = list(
+        filter(lambda x: x['component_type'] == COMPONENT_NAME, pipeline_conf[0]['component_config']))
+
+    if len(component_conf) == 0:
+        raise Exception(f"Configuration for component {COMPONENT_NAME} not found")
+    component_conf = component_conf[0]['env_variables']
+
+    for each_key, each_value in component_conf.items():
+        if not isinstance(each_value, str):
+            component_conf[each_key] = json.dumps(each_value)
+
+    os.environ.update(component_conf)
+
+# ---------------- END MONGO -----------------------------------------------------------------
 BASE_LOG_PATH = os.path.join(os.getcwd(), "logs")
 if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
     os.mkdir(os.path.join(os.getcwd(), 'logs'))
@@ -34,6 +93,7 @@ LOGSTASH_PORT = str(_config.get('SERVICE_CONFIG', {}).get('LOGSTASH_PORT'))
 # os.environ['max_retries'] = '1'
 # os.environ['authentication'] = '{"username":"vamshi","password":"sai"}'
 config = {
+    "json_filename": os.environ.get("json_filename", default='response.json'),
     "shared_volume": os.environ.get("shared_volume"),
     "method": os.environ.get("method"),
     "url": os.environ.get("url"),

scripts/common/constants.py

 #!/usr/bin/env python
 
 class RequestConstants:
+    COMPONENT_NAME = "RequestModuleFE"
     Request_auth_username = "username"
     Request_auth_password = "password"
     Request_authentication = "authentication"