pkl

main.py

@@ -41,10 +41,11 @@ class PreProcessComponent:
 
         sc_X = self.read_pickle_file(standard_scalar_pkl_path)
         X = sc_X.transform(forcast_data)
-        return X
+        return X, self.date
 
 
 if __name__ == '__main__':
+    logger.info(config)
     # Checking shared Volume
     if PreProcessConstants.SHARED_VOLUME in config.keys():
         shared_volume = config[PreProcessConstants.SHARED_VOLUME]
@@ -60,6 +61,10 @@ if __name__ == '__main__':
     # Checking pickle path for standard scalar
     if PreProcessConstants.STANDARD_SCALAR_PATH in config.keys():
         standard_scalar_path = config[PreProcessConstants.STANDARD_SCALAR_PATH]
+        if standard_scalar_path is None:
+            standard_scalar_path = "StandardScaler.pkl"
+        else:
+            pass
     else:
         raise Exception(ComponentExceptions.INVALID_JSON_PATH)
 
@@ -68,14 +73,16 @@ if __name__ == '__main__':
         fillna_method = config[PreProcessConstants.FILLNA_METHOD]
     else:
         raise Exception(ComponentExceptions.INVALID_Fillna_Method)
-
     obj = PreProcessComponent()
-    data = obj.preprocess(json_path, standard_scalar_path, fillna_method)
-    logger.info("Got the data writing it to pickle file (preprocessed_X.pkl)")
+    data, date_pkl = obj.preprocess(json_path, standard_scalar_path, fillna_method)
+    logger.info("Got the data writing it to pickle files (preprocessed_X.pkl and date_X.pkl)")
     try:
         output = open(os.path.join(shared_volume, 'preprocessed_X.pkl'), 'wb')
         pickle.dump(data, output)
         output.close()
+        output1 = open(os.path.join(shared_volume, 'date_X.pkl'), 'wb')
+        pickle.dump(date_pkl, output1)
+        output1.close()
         logger.info("Component executed Successfully")
     except Exception as e:
         raise Exception(e)

scripts/common/config_parser.py

@@ -30,6 +30,7 @@ LOGSTASH_PORT = str(_config.get('SERVICE_CONFIG', {}).get('LOGSTASH_PORT'))
 # os.environ["json_path"] = r"E:\welspun-defects\preprocess_data-ilendev\response.json"
 # os.environ["fillna_method"] = "ffill"
 # os.environ["standard_scalar_path"] = r"E:\welspun-defects\preprocess_data-ilendev\StandardScaler.pkl"
+print(os.getcwd())
 PKL_path = os.path.join(os.getcwd(), 'StandardScaler.pkl')
 config = {
     "shared_volume": os.environ.get("shared_volume"),