pkl

logs/PreProccessComponent.log

@@ -49,3 +49,21 @@
 2021-02-23 19:03:14,519  INFO PreProccessComponent Parsing the pickle file
 2021-02-23 19:03:15,704  INFO PreProccessComponent Got the data writing it to pickle file (preprocessed_X.pkl)
 2021-02-23 19:03:15,705  INFO PreProccessComponent Component executed Successfully
+2021-02-23 20:01:03,806  INFO PreProccessComponent Reading Json file
+2021-02-23 20:01:03,816  INFO PreProccessComponent Parsing the pickle file
+2021-02-23 20:01:10,052  INFO PreProccessComponent Reading Json file
+2021-02-23 20:01:10,063  INFO PreProccessComponent Parsing the pickle file
+2021-02-23 20:01:11,229  INFO PreProccessComponent Got the data writing it to pickle file (preprocessed_X.pkl)
+2021-02-23 20:01:11,230  INFO PreProccessComponent Component executed Successfully
+2021-02-23 20:01:18,361  INFO PreProccessComponent Reading Json file
+2021-02-23 20:01:18,371  INFO PreProccessComponent Parsing the pickle file
+2021-02-23 20:01:19,794  INFO PreProccessComponent Got the data writing it to pickle file (preprocessed_X.pkl)
+2021-02-23 20:01:19,795  INFO PreProccessComponent Component executed Successfully
+2021-02-23 20:01:31,699  INFO PreProccessComponent Reading Json file
+2021-02-23 20:01:31,708  INFO PreProccessComponent Parsing the pickle file
+2021-02-23 20:01:32,896  INFO PreProccessComponent Got the data writing it to pickle file (preprocessed_X.pkl)
+2021-02-23 20:01:32,896  INFO PreProccessComponent Component executed Successfully
+2021-02-23 20:04:25,902  INFO PreProccessComponent Reading Json file
+2021-02-23 20:04:25,913  INFO PreProccessComponent Parsing the pickle file
+2021-02-23 20:04:27,159  INFO PreProccessComponent Got the data writing it to pickle file (preprocessed_X.pkl)
+2021-02-23 20:04:27,162  INFO PreProccessComponent Component executed Successfully

main.py

@@ -13,7 +13,7 @@ class PreProcessComponent:
         self.ds, self.tempmax, self.tempmin, self.temp, self.humidity, self.precip, self.winddir, self.cloudcover, self.visibility, self.date, self.windspeed = ([] for i in range(11))
 
     def read_pickle_file(self, file):
-        logger.info("Parsing the pickle file")
+        logger.info("Parsing the pickle file at location "+file)
         return pickle.load(open(file, "rb"))
 
     def preprocess(self, forcast_json, standard_scalar_pkl_path, null_value_method):

scripts/common/config_parser.py

@@ -30,12 +30,12 @@ LOGSTASH_PORT = str(_config.get('SERVICE_CONFIG', {}).get('LOGSTASH_PORT'))
 # os.environ["json_path"] = r"E:\welspun-defects\preprocess_data-ilendev\response.json"
 # os.environ["fillna_method"] = "ffill"
 # os.environ["standard_scalar_path"] = r"E:\welspun-defects\preprocess_data-ilendev\StandardScaler.pkl"
-
+PKL_path = os.path.join(os.getcwd(), 'StandardScaler.pkl')
 config = {
     "shared_volume": os.environ.get("shared_volume"),
     "json_path": os.environ.get("json_path"),
     "fillna_method": os.environ.get("fillna_method", default="ffill"),
-    "standard_scalar_path": os.environ.get("standard_scalar_path", default=r"./scripts/common/StandardScaler.pkl"),
+    "standard_scalar_path": os.environ.get("standard_scalar_path", default=PKL_path),
 }
 if not os.path.exists(config['shared_volume']):
     sys.stderr.write("Shared path does not exist!")