pkl

logs/PreProccessComponent.log

@@ -67,3 +67,8 @@
 2021-02-23 20:04:25,913  INFO PreProccessComponent Parsing the pickle file
 2021-02-23 20:04:27,159  INFO PreProccessComponent Got the data writing it to pickle file (preprocessed_X.pkl)
 2021-02-23 20:04:27,162  INFO PreProccessComponent Component executed Successfully
+2021-02-23 20:56:37,844  INFO PreProccessComponent {'shared_volume': 'test', 'json_path': 'E:\\welspun-defects\\preprocess_data-ilendev\\response.json', 'fillna_method': 'ffill', 'standard_scalar_path': ''}
+2021-02-23 20:56:37,844  INFO PreProccessComponent Reading Json file
+2021-02-23 20:56:37,855  INFO PreProccessComponent Parsing the pickle file at location E:\iLens-AI\Git\preprocess-data\StandardScaler.pkl ......
+2021-02-23 20:56:39,016  INFO PreProccessComponent Got the data writing it to pickle files (preprocessed_X.pkl and date_X.pkl)
+2021-02-23 20:56:39,018  INFO PreProccessComponent Component executed Successfully

main.py

@@ -10,8 +10,7 @@ from scripts.common.logsetup import logger
 
 class PreProcessComponent:
     def __init__(self):
-        self.ds, self.tempmax, self.tempmin, self.temp, self.humidity, self.precip, self.winddir, self.cloudcover, self.visibility, self.date, self.windspeed = (
-        [] for i in range(11))
+        self.ds, self.tempmax, self.tempmin, self.temp, self.humidity, self.precip, self.winddir, self.cloudcover, self.visibility, self.date, self.windspeed = ([] for i in range(11))
 
     def read_pickle_file(self, file):
         logger.info("Parsing the pickle file at location " + file + " ......")
@@ -63,7 +62,7 @@ if __name__ == '__main__':
     if PreProcessConstants.STANDARD_SCALAR_PATH in config.keys():
         standard_scalar_path = config[PreProcessConstants.STANDARD_SCALAR_PATH]
         if len(standard_scalar_path) < 5:
-            standard_scalar_path = "StandardScaler.pkl"
+            standard_scalar_path = os.path.join(os.getcwd(), 'StandardScaler.pkl')
         else:
             pass
     else:

scripts/common/config_parser.py

@@ -36,7 +36,7 @@ config = {
     "shared_volume": os.environ.get("shared_volume"),
     "json_path": os.environ.get("json_path"),
     "fillna_method": os.environ.get("fillna_method", default="ffill"),
-    "standard_scalar_path": os.environ.get("standard_scalar_path", default=PKL_path),
+    "standard_scalar_path": os.environ.get("standard_scalar_path", default=str(PKL_path)),
 }
 if not os.path.exists(config['shared_volume']):
     sys.stderr.write("Shared path does not exist!")