Skip to content

A
AddtoModelStore
Commit 30e58a74 authored by dasharatha.vamshi's avatar dasharatha.vamshi
Browse files
Options

changes

parent 6abb9e5d
Show whitespace changes
Inline Side-by-side
Showing with 104 additions and 166 deletions
main.py
View file @ 30e58a74
......@@ -24,31 +24,21 @@ class AddtoModelStore:
self.component_input_dir = self.query['component_input_dir']
self.artifact_current_path = self.query['artifact_current_path']
self.artifact_archived_path = self.query['artifact_archived_path']
self.new_model_path = os.path.join(self.component_input_dir, os.listdir(self.component_input_dir)[0])
self.new_model_name = os.listdir(self.component_input_dir)[0]
self.new_model_name = self.get_pkl()
self.new_model_path = os.path.join(self.component_input_dir, self.new_model_name)
self.file_rename = self.new_model_name.split('.')[0] + "_" + str(uuid.uuid4()).split('-')[0] + "." + \
self.new_model_name.split('.')[1]
os.rename(self.new_model_path, os.path.join(self.component_input_dir, self.file_rename))
self.new_model_path = os.path.join(self.component_input_dir, os.listdir(self.component_input_dir)[0])
self.new_model_name = os.listdir(self.component_input_dir)[0]
self.new_model_path = os.path.join(self.component_input_dir, self.get_pkl())
self.new_model_name = self.get_pkl()
self.meta_data_file = self.query['metadata_file']
self.update_current_model = {
"model_name": "randomforest",
"model_params": None,
"training_date": str(date.today()),
"framework": None,
"serializedObjectType": "pkl",
"model_fl_name": self.new_model_name
}
self.first_run = [
{
'id': self.ilens_tag_hierarchy,
'current_model': {
'model_name': 'randomforest',
'model_params': None,
'training_date': str(date.today()),
'framework': None,
'serializedObjectType': 'pkl',
'model_fl_name': self.new_model_name
},
'archived': [
......@@ -58,6 +48,28 @@ class AddtoModelStore:
]
self.new_tag = self.first_run[0]
self.status = False
self.input_meta = os.path.join(self.component_input_dir, "metadata.json")
self.input_meta_data = self.read_input_meta()
self.update_current_model.update(self.input_meta_data)
self.first_run[0]['current_model'].update(self.input_meta_data)
def get_pkl(self):
input_list = os.listdir(self.component_input_dir)
for i in input_list:
if i.endswith(".pkl"):
return i
def read_input_meta(self):
try:
logger.info("Reading input meatadata Json file")
with open(self.input_meta) as f:
input_meta_data = json.load(f)
return input_meta_data
except Exception as e:
logger.info(e)
logger.info("Failed reading input metadata Json File")
logger.info(traceback.format_exc())
def move_files(self, file_name):
try:
logger.info("started moving files from current to archived folder......")
......@@ -79,16 +91,14 @@ class AddtoModelStore:
def check_meta_data(self):
return path.exists(self.query['metadata_file'])
def upload_to_blob(self):
def upload_to_blob(self, blob_path, local_path):
try:
logger.info("started uploading data..............")
blob_client = self.blob_service_client.get_blob_client(container=self.container,
blob=os.path.join(self.artifact_current_path,
self.new_model_name))
with open(self.new_model_path, "rb") as data:
blob=blob_path)
with open(local_path, "rb") as data:
blob_client.upload_blob(data, overwrite=True)
logger.info("File at " + self.new_model_path + " is uploaded to " + os.path.join(self.artifact_current_path,
self.new_model_name))
logger.info("File at " + local_path + " is uploaded to " + blob_path)
return True
except Exception as e:
logger.info(traceback.format_exc())
......@@ -138,14 +148,16 @@ class AddtoModelStore:
old_model_name = self.get_file_name_from_meta_data(data)
if old_model_name:
self.move_files(old_model_name)
self.upload_to_blob()
self.upload_to_blob(os.path.join(self.artifact_current_path, self.new_model_name),
self.new_model_path)
data = self.update_meta_data(data)
json_object = json.dumps(data, indent=4)
with open(self.query['metadata_file'], "w") as outfile:
outfile.write(json_object)
else:
logger.info("There is no model for this site")
self.upload_to_blob()
self.upload_to_blob(os.path.join(self.artifact_current_path, self.new_model_name),
self.new_model_path)
data = self.update_meta_data(data)
json_object = json.dumps(data, indent=4)
with open(self.query['metadata_file'], "w") as outfile:
......@@ -154,7 +166,8 @@ class AddtoModelStore:
else:
logger.info(
"metadata json file is not present so created metadata json file at " + self.query['metadata_file'])
if self.upload_to_blob():
if self.upload_to_blob(os.path.join(self.artifact_current_path, self.new_model_name),
self.new_model_path):
json_object = json.dumps(self.first_run, indent=4)
with open(self.query['metadata_file'], "w") as outfile:
outfile.write(json_object)
......@@ -169,8 +182,12 @@ class AddtoModelStore:
if __name__ == '__main__':
try:
obj = AddtoModelStore(config)
obj.run()
if obj.run():
logger.info("Uploading metedata json file to azure..................")
if obj.upload_to_blob(os.path.join(config['artifact_base_path'], "metadata.json"), config['metadata_file']):
logger.info("Component ran successfully")
else:
logger.info("Component Failed")
except Exception as e:
logger.info("Model Object Component Failed")
logger.info(traceback.format_exc())
scripts/common/config_parser.py
View file @ 30e58a74
......@@ -20,7 +20,6 @@ else:
# uncomment for Testing
# os.environ['pipeline_id'] = "pipe1"
# os.environ['artifact_base_path'] = "/data/ai-models"
# os.environ['ilens_tag_hierarchy'] = 'site1'
# os.environ['container_name'] = 'test'
# ------------------------ Configurations -----------------------------------------------------------------------------
pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
......@@ -45,7 +44,7 @@ data_store = os.environ.get('data_store', default="Azure")
container_name = os.environ.get('container_name', default=AddtoModelStoreConstants.CONTAINER)
# change prod to test for testing
connection_string = os.environ.get('connection_string', _config.get("SYSTEM_CONFIG", {}).get('test_connection_string'))
connection_string = os.environ.get('connection_string', _config.get("SYSTEM_CONFIG", {}).get('prod_connection_string'))
artifact_base_path = os.environ.get('artifact_base_path')
artifact_current_path = os.path.join(artifact_base_path, "current")
......
temp/mnt/ilens/ai-jobs/models/metadata.json
View file @ 30e58a74
[
{
"id": "site-new",
"id": "site-v1",
"current_model": {
"model_fl_name": "model_51a6f7f7.pkl",
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_da4ccf80_ab258458_6025cc0e_bef1840c_e6a3ee66_1a517dec_05dbb188_53901b54.pkl"
},
"archived": [
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-11",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_48956f4d.pkl"
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-11",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_78a80044.pkl"
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_e750f617.pkl"
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_bd650614.pkl"
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_bd650614_ba1250bd.pkl"
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_da4ccf80.pkl"
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_da4ccf80_ab258458.pkl"
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_da4ccf80_ab258458_6025cc0e.pkl"
"model_params": {
"bootstrap": true,
"max_features": "sqrt",
"min_samples_leaf": 4,
"min_samples_split": 10,
"n_estimators": 900,
"criterion": "mse",
"max_depth": 100
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_da4ccf80_ab258458_6025cc0e_bef1840c.pkl"
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_da4ccf80_ab258458_6025cc0e_bef1840c_e6a3ee66.pkl"
"framework": "sklearn"
},
"archived": [
{
"model_fl_name": "model_a4b950ad.pkl",
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_da4ccf80_ab258458_6025cc0e_bef1840c_e6a3ee66_1a517dec.pkl"
"model_params": {
"bootstrap": true,
"max_features": "sqrt",
"min_samples_leaf": 4,
"min_samples_split": 10,
"n_estimators": 900,
"criterion": "mse",
"max_depth": 100
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_da4ccf80_ab258458_6025cc0e_bef1840c_e6a3ee66_1a517dec_05dbb188.pkl"
"framework": "sklearn"
}
]
},
{
"id": "site-new1",
"id": "site-v2",
"current_model": {
"model_fl_name": "model_ac7f23b6.pkl",
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-11",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_faed1e93.pkl"
},
"archived": []
"model_params": {
"bootstrap": true,
"max_features": "sqrt",
"min_samples_leaf": 4,
"min_samples_split": 10,
"n_estimators": 910,
"criterion": "mse",
"max_depth": 100
},
{
"id": "site-new2",
"current_model": {
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-11",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_b5d4bc8b.pkl"
},
"archived": []
},
{
"id": "site-v1",
"current_model": {
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_2785bcb5_97bea2c9_02661874.pkl"
"framework": "sklearn"
},
"archived": [
{
"model_fl_name": "model_f38d24ff.pkl",
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_2785bcb5.pkl"
"model_params": {
"bootstrap": true,
"max_features": "sqrt",
"min_samples_leaf": 4,
"min_samples_split": 10,
"n_estimators": 900,
"criterion": "mse",
"max_depth": 100
},
{
"model_name": "randomforest",
"model_params": null,
"training_date": "2021-03-12",
"framework": null,
"serializedObjectType": "pkl",
"model_fl_name": "model_2785bcb5_97bea2c9.pkl"
"framework": "sklearn"
}
]
}
......
temp/mnt/ilens/ai-jobs/pipe1/default_run_id/AddtoModelStore/input/metadata.json 0 → 100644
View file @ 30e58a74
{
"model_name": "randomforest",
"model_params": {
"bootstrap": true,
"max_features": "sqrt",
"min_samples_leaf": 4,
"min_samples_split": 10,
"n_estimators": 910,
"criterion": "mse",
"max_depth": 100
},
"serializedObjectType": "pkl",
"framework": "sklearn"
}
\ No newline at end of file
temp/mnt/ilens/ai-jobs/pipe1/default_run_id/AddtoModelStore/param.json
View file @ 30e58a74
{
"ilens_tag_hierarchy": "site-v1"
"ilens_tag_hierarchy": "site-v2"
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment