Added Relative Start and End Date. Handle empty end date

main.py

@@ -10,19 +10,31 @@ from scripts.common.constants import KariosConstants, ComponentExceptions
 
 def get_data(query):
     logger.info("Parsing user requests")
+
+    #  ------------------------ Validating Kairos URL Key ------------------------ #
     if KariosConstants.KAIROS_URL_KEY in config.keys():
         kairosdb_server = config[KariosConstants.KAIROS_URL_KEY]
     else:
         raise Exception(ComponentExceptions.INVALID_KAIROS_URL_EXCEPTION)
+
+    #  ---------------------- Validating Tag Hierarchy Key ----------------------- #
     if config[KariosConstants.TAG_HIERARCHY_KEY] is not None and len(config[KariosConstants.TAG_HIERARCHY_KEY]) > 0:
         pass
     else:
         raise Exception(ComponentExceptions.INVALID_TAG_HIERARCHY_EXCEPTION)
+
+    #  ------------------------ Validating Metric Name Key ----------------------- #
     if KariosConstants.METRIC_NAME_KEY in config.keys():
         query[KariosConstants.METRICS_KEY][0][KariosConstants.NAME_KEY] = config[KariosConstants.METRIC_NAME_KEY]
     else:
         raise Exception(ComponentExceptions.INVALID_METRIC_NAME_EXCEPTION)
-    if KariosConstants.START_ABSOLUTE_KEY in config.keys():
+
+    #  ------------------------ Parsing Start Time ------------------------------- #
+    """
+    If start absolute Key is present, it will take precedence over start relative key.
+    For start relative key is there, proper unit has to be provided.
+    """
+    if config[KariosConstants.START_ABSOLUTE_KEY] != '':
         start_time = config[KariosConstants.START_ABSOLUTE_KEY]
         try:
             start_timestamp = datetime.strptime(start_time, KariosConstants.DATETIME_FORMAT)
@@ -31,18 +43,26 @@ def get_data(query):
             query.pop(KariosConstants.START_RELATIVE_KEY)
         except ValueError as e:
             raise Exception(f"{ComponentExceptions.INVALID_START_TIME_FORMAT_EXCEPTION} - {str(e)}")
-    elif KariosConstants.START_RELATIVE_KEY in config.keys():
-        if config[KariosConstants.START_RELATIVE_KEY][KariosConstants.VALUE_KEY] > 0:
+
+    elif config[KariosConstants.START_RELATIVE_KEY] != '':
+        if int(config[KariosConstants.START_RELATIVE_KEY]) > 0:
             query[KariosConstants.START_RELATIVE_KEY][KariosConstants.VALUE_KEY] = config[KariosConstants.
-                START_RELATIVE_KEY][KariosConstants.VALUE_KEY]
+                START_RELATIVE_KEY]
             query[KariosConstants.START_RELATIVE_KEY][KariosConstants.UNIT_KEY] = config[KariosConstants.
-                START_RELATIVE_KEY][KariosConstants.UNIT_KEY]
+                START_RELATIVE_UNIT_KEY].lower()
             query.pop(KariosConstants.START_ABSOLUTE_KEY)
         else:
             raise Exception(ComponentExceptions.INVALID_START_TIME_VALUE_EXCEPTION)
     else:
         raise Exception(ComponentExceptions.MISSING_START_TIME_VALUE_EXCEPTION)
-    if KariosConstants.END_ABSOLUTE_KEY in config.keys():
+
+    #  ------------------------ Parsing End Time ------------------------------- #
+    """
+    If end absolute Key is present, it will take precedence over end relative key.
+    For end relative key is there, proper unit has to be provided.
+    If neither of them are there, then data will be queried for eternity.
+    """
+    if config[KariosConstants.END_ABSOLUTE_KEY] != '':
         end_time = config[KariosConstants.END_ABSOLUTE_KEY]
         try:
             end_timestamp = datetime.strptime(end_time, KariosConstants.DATETIME_FORMAT)
@@ -52,13 +72,12 @@ def get_data(query):
         except ValueError as e:
             raise Exception(f"{ComponentExceptions.INVALID_END_TIME_FORMAT_EXCEPTION} - {str(e)}")
 
-    elif KariosConstants.END_RELATIVE_KEY in config.keys():
-
-        if config[KariosConstants.END_RELATIVE_KEY][KariosConstants.VALUE_KEY] > 0:
+    elif config[KariosConstants.END_RELATIVE_KEY] != '':
+        if int(config[KariosConstants.END_RELATIVE_KEY]) > 0:
             query[KariosConstants.END_RELATIVE_KEY][KariosConstants.VALUE_KEY] = config[KariosConstants.
                 END_RELATIVE_KEY][KariosConstants.VALUE_KEY]
             query[KariosConstants.END_RELATIVE_KEY][KariosConstants.UNIT_KEY] = config[KariosConstants.
-                END_RELATIVE_KEY][KariosConstants.UNIT_KEY]
+                END_RELATIVE_UNIT_KEY].lower()
             query.pop(KariosConstants.END_ABSOLUTE_KEY)
         else:
             raise Exception(ComponentExceptions.INVALID_START_END_VALUE_EXCEPTION)
@@ -66,10 +85,13 @@ def get_data(query):
         query.pop(KariosConstants.END_ABSOLUTE_KEY)
         query.pop(KariosConstants.END_RELATIVE_KEY)
 
+    #  ------------------------ Parsing Aggregation Key ------------------------------- #
     if config.get(KariosConstants.AGGREGATION_OPS) != "None" or config.get(KariosConstants.AGGREGATION_OPS) is not None:
         query[KariosConstants.METRICS_KEY][0][KariosConstants.AGGREGATORS_KEY].append(get_aggregation_query(config))
     else:
         query[KariosConstants.METRICS_KEY][0].pop(KariosConstants.AGGREGATORS_KEY)
+
+    #  ----------------- Parsing Group BY Key - [TO BE IMPLEMENTED] ------------------- #
     if KariosConstants.GROUPBY_KEY not in config.keys():
         query[KariosConstants.METRICS_KEY][0].pop(KariosConstants.GROUPBY_KEY)
     if KariosConstants.TAG_HIERARCHY_KEY in config.keys():
@@ -78,8 +100,9 @@ def get_data(query):
     else:
         raise Exception(ComponentExceptions.MISSING_TAG_HIERARCHY_EXCEPTION)
 
+    #  --------------------------- Querying KairosDB --------------------------------- #
     logger.info("Querying Karios DB...")
-    logger.debug("Query --> {}".format(query))
+    logger.debug(KariosConstants.LOG_VAR_MESSAGE.format("Query", json.dumps(query, indent=1)))
     response = requests.post(kairosdb_server + KariosConstants.KARIOS_API, data=json.dumps(query))
     if response.status_code == KariosConstants.REQUEST_SUCCESS_CODE:
         logger.info("Receiving data...")

scripts/common/config_parser.py

@@ -12,6 +12,23 @@ from scripts.utils import str2bool
 #                        '"end_absolute": "30/11/2020 00:00:00",' \
 #                        '"shared_volume": "/mnt"}'
 
+
+# os.environ["shared_volume"] = "/mnt"
+# os.environ["start_absolute"] = ""
+# os.environ["end_absolute"] = ""
+# os.environ["start_relative"] = "3"
+# os.environ["start_relative_unit"] = "Years"
+# os.environ["end_relative"] = ""
+# os.environ["end_relative_unit"] = ""
+# os.environ["tag_hierarchy"] = '{"energy_values":"site_153$dept_1036$line_312$equipment_3876$tag_22745"}'
+# os.environ["metric_name"] = "ilens.live_data.raw"
+# os.environ["kairosdb_url"] = "http://192.168.0.207:8080"
+# os.environ["aggregation_ops"] = "LAST"
+# os.environ["aggregation_sampling_value"] = "1"
+# os.environ["aggregation_sampling_unit"] = "Days"
+# os.environ["aggregation_alignment"] = "None"
+# os.environ["LOG_LEVEL"] = "debug"
+
 config_path = os.path.join(os.getcwd(), "conf", "configuration.yml")
 if os.path.exists(config_path):
     sys.stderr.write("Reading config from --> {}".format(config_path))
@@ -31,14 +48,21 @@ if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
 LOG_LEVEL = os.environ.get("LOG_LEVEL" ,_config.get('SERVICE_CONFIG', {}).get("LOG_LEVEL", "INFO")).upper()
 LOG_HANDLER_NAME = _config.get('SERVICE_CONFIG', {}).get("LOG_HANDLER_NAME", "ReadFromKairos")
 
-# config = json.loads(os.environ.get('config'))
 config = {
     "shared_volume": os.environ.get("shared_volume"),
-    "end_absolute": os.environ.get("end_absolute"),
+
     "start_absolute": os.environ.get("start_absolute"),
+    "end_absolute": os.environ.get("end_absolute", ''),
+
+    "start_relative": os.environ.get("start_relative"),
+    "start_relative_unit": os.environ.get("start_relative_unit"),
+    "end_relative": os.environ.get("end_relative", ''),
+    "end_relative_unit": os.environ.get("end_relative_unit"),
+
     "tag_hierarchy": json.loads(os.environ.get("tag_hierarchy")),
     "metric_name": os.environ.get("metric_name"),
     "kairosdb_url": os.environ.get("kairosdb_url"),
+
     "aggregation_ops": os.environ.get("aggregation_ops"),
     "aggregation_sampling_value": os.environ.get("aggregation_sampling_value"),
     "aggregation_sampling_unit": os.environ.get("aggregation_sampling_unit"),

scripts/common/constants.py

@@ -7,8 +7,10 @@ class KariosConstants:
     SHARED_VOLUME_KEY = "shared_volume"
     START_ABSOLUTE_KEY = "start_absolute"
     START_RELATIVE_KEY = "start_relative"
+    START_RELATIVE_UNIT_KEY = "start_relative_unit"
     END_ABSOLUTE_KEY = "end_absolute"
     END_RELATIVE_KEY = "end_relative"
+    END_RELATIVE_UNIT_KEY = "end_relative_unit"
     APPLY_AGGREGATION_KEY = "apply_aggregation"
     AGGREGATION_OPS = "aggregation_ops"
     AGGREGATION_SAMPLING_VALUE = "aggregation_sampling_value"
@@ -74,6 +76,6 @@ class ComponentExceptions:
     INVALID_START_TIME_FORMAT_EXCEPTION = "start time should be dd/mm/yyyy hh/MM/SS"
     INVALID_END_TIME_FORMAT_EXCEPTION = "end time should be dd/mm/yyyy hh/MM/SS"
     INVALID_START_TIME_VALUE_EXCEPTION = "start time should be greater than zero"
-    MISSING_START_TIME_VALUE_EXCEPTION = "need start date key"
+    MISSING_START_TIME_VALUE_EXCEPTION = "need start date for querying"
     MISSING_TAG_HIERARCHY_EXCEPTION = "required tag hierarchy"
     INVALID_START_END_VALUE_EXCEPTION = "end time should be greater than zero and less than start time"