init

conf/configuration.yml

@@ -7,8 +7,8 @@ SERVICE_CONFIG:
 
 #--------------System Configurations--------------------#
 SYSTEM_CONFIG:
-  shared_mount_base_ai_job: /mnt/ilens/ai-job/
-
+  shared_mount_base_ai_job: /mnt/ilens/ai-jobs/
+  timestamp_label: timestamp
 
 #----------------------If read conf from mongo------------#
 FOR_EACH_MONGO_CONFIG:

main.py

@@ -56,6 +56,7 @@ class PreProcessComponent:
         except:
             logger.info(traceback.format_exc())
 
+
 if __name__ == '__main__':
     try:
         # Checking json path
@@ -75,7 +76,7 @@ if __name__ == '__main__':
     try:
         logger.info("Writing forcast data to csv file at " + os.path.join(config['component_output_dir'], 'data.csv'))
         output2 = os.path.join(config['component_output_dir'], 'data.csv')
-        forcast_df.to_csv(output2, index=True)
+        forcast_df.to_csv(output2, index=True, index_label=config['timestamp_label'])
         logger.info("Component executed Successfully")
     except Exception as e:
         raise Exception(e)

scripts/common/config_parser.py

@@ -23,8 +23,10 @@ else:
 pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
 shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job",
                                           _config.get("SYSTEM_CONFIG", {}).get('shared_mount_base_ai_job'))
+timestamp_label = os.environ.get("timestamp_label", _config.get("SYSTEM_CONFIG", {}).get('timestamp_label'))
 # read from $shared_mount_base_ai_job/$pipeline_id/run.config
-run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run_config.json"
+# run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run_config.json"
+run_id_path = os.path.join(shared_mount_base_ai_job, pipeline_id, "run_config.json")
 try:
     sys.stderr.write("Checking for run id parameters at path " + run_id_path + "\n")
     with open(run_id_path) as f:
@@ -35,10 +37,14 @@ run_id = run_id_param['run_id']
 fillna_method = os.environ.get("fillna_method", default="ffill")
 
 # shared_mount_base_ai_job/$pipeline_id/$run_id/PreprocessWeatherData/input/$.json
-component_input = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + PreprocessWeatherDataConstants.COMPONENT_NAME + "/input"
-# shared_mount_base_ai_job/$pipeline_id/$run_id/RunModel/input/data.csv
-component_output_dir = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + PreprocessWeatherDataConstants.NEXT_COMPONENT + "/input"
+# component_input = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + PreprocessWeatherDataConstants.COMPONENT_NAME + "/input"
+component_input = os.path.join(shared_mount_base_ai_job, pipeline_id, run_id,
+                               PreprocessWeatherDataConstants.COMPONENT_NAME, "input")
 
+# shared_mount_base_ai_job/$pipeline_id/$run_id/RunModel/input/data.csv
+# component_output_dir = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + PreprocessWeatherDataConstants.NEXT_COMPONENT + "/input"
+component_output_dir = os.path.join(shared_mount_base_ai_job, pipeline_id, run_id,
+                                    PreprocessWeatherDataConstants.NEXT_COMPONENT, "input")
 
 BASE_LOG_PATH = os.path.join(os.getcwd(), "logs")
 if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
@@ -61,7 +67,8 @@ config = {
     'shared_mount_base_ai_job': shared_mount_base_ai_job,
     'fillna_method': fillna_method,
     'component_input': component_input,
-    'component_output_dir': component_output_dir
+    'component_output_dir': component_output_dir,
+    'timestamp_label': timestamp_label
 }
 if not os.path.exists(config['shared_mount_base_ai_job']):
     sys.stderr.write("Shared path does not exist!" + "\n")

temp/mnt/ilens/ai-jobs/pipe1/run1/GetDataFromStore/param.json

+{
+	"artifact_name": "response.json"
+}
\ No newline at end of file

temp/mnt/ilens/ai-jobs/pipe1/run1/RunModel/input/data.csv

+timestamp,Maximum Temperature,Minimum Temperature,Temperature,Precipitation,Wind Speed,Wind Direction,Visibility,Cloud Cover,Relative Humidity,month
+2021-03-02,36.0,24.0,28.4,0.0,18.4,126.4,9.8,7.6,71.7,3
+2021-03-03,30.4,23.0,26.9,0.0,19.1,68.0,24.1,0.3,73.3,3
+2021-03-04,29.8,23.1,26.5,0.1,21.2,54.0,24.1,17.3,72.0,3
+2021-03-05,30.6,22.3,26.3,0.0,19.4,66.9,24.1,0.3,71.9,3
+2021-03-06,30.5,22.9,26.7,0.0,20.9,51.9,24.1,0.1,73.9,3
+2021-03-07,30.4,23.9,27.2,0.09,20.2,59.2,24.1,19.0,75.9,3
+2021-03-08,31.3,24.8,27.7,0.35,20.9,52.7,24.1,20.9,75.0,3
+2021-03-09,31.5,24.4,27.6,4.46,22.3,58.5,22.6,28.5,75.9,3
+2021-03-10,31.9,24.2,27.8,0.14,21.2,66.2,24.1,5.0,76.6,3
+2021-03-11,30.8,24.1,27.5,0.0,21.2,56.9,24.1,7.9,75.5,3
+2021-03-12,31.2,24.1,27.7,0.51,21.6,65.7,24.1,28.6,74.0,3
+2021-03-13,31.0,24.8,27.6,0.71,20.9,55.5,24.1,7.2,76.2,3
+2021-03-14,31.9,23.7,27.4,0.0,22.0,64.0,24.1,2.9,74.6,3
+2021-03-15,31.1,23.6,27.3,0.0,20.5,60.0,24.1,9.3,75.5,3
+2021-03-16,30.6,23.5,27.2,0.0,21.2,59.6,24.1,10.7,72.7,3

temp/mnt/ilens/ai-jobs/pipe1/run_config.json

+{
+	"run_id" : "run1"
+}
\ No newline at end of file