init

conf/configuration.yml

@@ -7,7 +7,7 @@ SERVICE_CONFIG:
 
 #--------------System Configurations--------------------#
 SYSTEM_CONFIG:
-  shared_mount_base_ai_job: /mnt/ilens/ai-job/
+  shared_mount_base_ai_job: /mnt/ilens/ai-jobs/
   connection_string: DefaultEndpointsProtocol=https;AccountName=azrabsilensqa01;AccountKey=DN6q6kX98JM8yUwtuJh2bAaXUGFo1zRS5HJSsa/ZA+MlmctjC000eHP7bdXiQqkI/MVtADhS8c9E88LI5T4UHw==;EndpointSuffix=core.windows.net
 
 

main.py

@@ -43,7 +43,7 @@ if __name__ == '__main__':
             azure_container_name = config[GetDataFromStoreConstants.CONTAINER_NAME]
         else:
             raise Exception(ComponentExceptions.INVALID_CONTAINER_NAME)
-        logger.info(config)
+        # logger.info(config)
         mydict = {
             "azure_file_path": os.path.join(azure_file_path, azure_file_name),
             "local_file_path": os.path.join(config['component_output_dir'], azure_file_name),

scripts/common/config_parser.py

@@ -26,7 +26,8 @@ pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
 shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job",
                                           _config.get("SYSTEM_CONFIG", {}).get('shared_mount_base_ai_job'))
 # read from $shared_mount_base_ai_job/$pipeline_id/run.config
-run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run_config.json"
+# run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run_config.json"
+run_id_path = os.path.join(shared_mount_base_ai_job, pipeline_id, "run_config.json")
 try:
     sys.stderr.write("Checking for run id parameters at path " + run_id_path + "\n")
     with open(run_id_path) as f:
@@ -39,7 +40,10 @@ container_name = os.environ.get('container_name', default=GetDataFromStoreConsta
 connection_string = os.environ.get('connection_string', _config.get("SYSTEM_CONFIG", {}).get('connection_string'))
 artifact_base_path = os.environ.get('artifact_base_path')
 # Component Parameter:Read from $shared_mount_base_ai_job/$pipeline_id/$run_id/GetDataFromStore/param.json
-component_parameter_path = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + GetDataFromStoreConstants.COMPONENT_NAME + "/param.json"
+# component_parameter_path = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + GetDataFromStoreConstants.COMPONENT_NAME + "/param.json"
+component_parameter_path = os.path.join(shared_mount_base_ai_job, pipeline_id, run_id,
+                                        GetDataFromStoreConstants.COMPONENT_NAME, "param.json")
+
 try:
     sys.stderr.write("Checking for component parameters at path " + component_parameter_path + "\n")
     with open(component_parameter_path) as f:
@@ -48,8 +52,9 @@ except Exception as e:
     raise Exception(e)
 artifact_name = component_parameter['artifact_name']
 # $shared_mount_base_ai_job/$pipeline_id/$run_id/PreprocessWeatherData/input/$artifact_Name
-component_output_dir = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + GetDataFromStoreConstants.NEXT_COMPONENT + "/input"
-
+# component_output_dir = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + GetDataFromStoreConstants.NEXT_COMPONENT + "/input"
+component_output_dir = os.path.join(shared_mount_base_ai_job, pipeline_id, run_id,
+                                    GetDataFromStoreConstants.NEXT_COMPONENT, "input")
 BASE_LOG_PATH = os.path.join(os.getcwd(), "logs")
 if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
     os.mkdir(os.path.join(os.getcwd(), 'logs'))

temp/mnt/ilens/ai-jobs/pipe1/run1/GetDataFromStore/param.json

+{
+	"artifact_name": "response.json"
+}
\ No newline at end of file

temp/mnt/ilens/ai-jobs/pipe1/run_config.json

+{
+	"run_id" : "run1"
+}
\ No newline at end of file