Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
A
AddtoModelStore
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
CI / CD Analytics
Repository Analytics
Value Stream Analytics
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
dasharatha.vamshi
AddtoModelStore
Commits
461e5210
Commit
461e5210
authored
Mar 11, 2021
by
dasharatha.vamshi
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
bug fix
parent
a265efe0
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
46 additions
and
41 deletions
+46
-41
main.py
main.py
+27
-13
temp/mnt/ilens/ai-jobs/models/metadata.json
temp/mnt/ilens/ai-jobs/models/metadata.json
+16
-28
temp/mnt/ilens/ai-jobs/pipe1/default_run_id/AddtoModelStore/input/model_78a80044.pkl
...1/default_run_id/AddtoModelStore/input/model_78a80044.pkl
+0
-0
temp/mnt/ilens/ai-jobs/pipe1/default_run_id/AddtoModelStore/param.json
...s/ai-jobs/pipe1/default_run_id/AddtoModelStore/param.json
+3
-0
No files found.
main.py
View file @
461e5210
...
@@ -56,7 +56,8 @@ class AddtoModelStore:
...
@@ -56,7 +56,8 @@ class AddtoModelStore:
]
]
}
}
]
]
self
.
new_tag
=
self
.
first_run
[
0
]
self
.
status
=
False
def
move_files
(
self
,
file_name
):
def
move_files
(
self
,
file_name
):
try
:
try
:
logger
.
info
(
"started moving files from current to archived folder......"
)
logger
.
info
(
"started moving files from current to archived folder......"
)
...
@@ -98,13 +99,17 @@ class AddtoModelStore:
...
@@ -98,13 +99,17 @@ class AddtoModelStore:
logger
.
info
(
"Updating metadata json file..............."
)
logger
.
info
(
"Updating metadata json file..............."
)
for
i
in
data
:
for
i
in
data
:
if
i
[
'id'
]
==
self
.
ilens_tag_hierarchy
:
if
i
[
'id'
]
==
self
.
ilens_tag_hierarchy
:
self
.
status
=
True
i
[
'archived'
]
.
append
(
i
[
'current_model'
])
i
[
'archived'
]
.
append
(
i
[
'current_model'
])
i
[
'current_model'
]
=
self
.
update_current_model
i
[
'current_model'
]
=
self
.
update_current_model
# print(data)
if
self
.
status
:
return
data
else
:
logger
.
info
(
"There is no site so add added the tag to the metadata json"
)
data
.
append
(
self
.
new_tag
)
return
data
return
data
except
Exception
as
e
:
except
Exception
as
e
:
logger
.
info
(
traceback
.
format_exc
())
logger
.
info
(
e
)
raise
KeyError
(
e
)
def
get_file_name_from_meta_data
(
self
,
data
):
def
get_file_name_from_meta_data
(
self
,
data
):
try
:
try
:
...
@@ -113,8 +118,8 @@ class AddtoModelStore:
...
@@ -113,8 +118,8 @@ class AddtoModelStore:
if
i
[
'id'
]
==
self
.
ilens_tag_hierarchy
:
if
i
[
'id'
]
==
self
.
ilens_tag_hierarchy
:
return
i
[
'current_model'
][
'model_fl_name'
]
return
i
[
'current_model'
][
'model_fl_name'
]
except
Exception
as
e
:
except
Exception
as
e
:
logger
.
info
(
traceback
.
format_exc
()
)
logger
.
info
(
e
)
r
aise
KeyError
(
e
)
r
eturn
False
def
run
(
self
):
def
run
(
self
):
try
:
try
:
...
@@ -131,12 +136,21 @@ class AddtoModelStore:
...
@@ -131,12 +136,21 @@ class AddtoModelStore:
logger
.
info
(
"Failed reading Json File"
)
logger
.
info
(
"Failed reading Json File"
)
logger
.
info
(
traceback
.
format_exc
())
logger
.
info
(
traceback
.
format_exc
())
old_model_name
=
self
.
get_file_name_from_meta_data
(
data
)
old_model_name
=
self
.
get_file_name_from_meta_data
(
data
)
if
old_model_name
:
self
.
move_files
(
old_model_name
)
self
.
move_files
(
old_model_name
)
self
.
upload_to_blob
()
self
.
upload_to_blob
()
data
=
self
.
update_meta_data
(
data
)
data
=
self
.
update_meta_data
(
data
)
json_object
=
json
.
dumps
(
data
,
indent
=
4
)
json_object
=
json
.
dumps
(
data
,
indent
=
4
)
with
open
(
self
.
query
[
'metadata_file'
],
"w"
)
as
outfile
:
with
open
(
self
.
query
[
'metadata_file'
],
"w"
)
as
outfile
:
outfile
.
write
(
json_object
)
outfile
.
write
(
json_object
)
else
:
logger
.
info
(
"There is no model for this site"
)
self
.
upload_to_blob
()
data
=
self
.
update_meta_data
(
data
)
json_object
=
json
.
dumps
(
data
,
indent
=
4
)
with
open
(
self
.
query
[
'metadata_file'
],
"w"
)
as
outfile
:
outfile
.
write
(
json_object
)
else
:
else
:
logger
.
info
(
logger
.
info
(
"metadata json file is not present so created metadata json file at "
+
self
.
query
[
'metadata_file'
])
"metadata json file is not present so created metadata json file at "
+
self
.
query
[
'metadata_file'
])
...
...
temp/mnt/ilens/ai-jobs/models/metadata.json
View file @
461e5210
[
[
{
{
"id"
:
"site
1
"
,
"id"
:
"site
-new
"
,
"current_model"
:
{
"current_model"
:
{
"model_name"
:
"randomforest"
,
"model_name"
:
"randomforest"
,
"model_params"
:
null
,
"model_params"
:
null
,
"training_date"
:
"2021-03-11"
,
"training_date"
:
"2021-03-11"
,
"framework"
:
null
,
"framework"
:
null
,
"serializedObjectType"
:
"pkl"
,
"serializedObjectType"
:
"pkl"
,
"model_fl_name"
:
"model_
9547643f_59d89573
.pkl"
"model_fl_name"
:
"model_
78a80044
.pkl"
},
},
"archived"
:
[
"archived"
:
[
{
{
"model_name"
:
"randomforest"
,
"model_name"
:
"randomforest"
,
"model_params"
:
null
,
"model_params"
:
null
,
"training_date"
:
"2021-03-10"
,
"training_date"
:
"2021-03-11"
,
"framework"
:
null
,
"serializedObjectType"
:
"pkl"
,
"model_fl_name"
:
"model_111175b9.pkl"
},
{
"model_name"
:
"randomforest"
,
"model_params"
:
null
,
"training_date"
:
"2021-03-10"
,
"framework"
:
null
,
"framework"
:
null
,
"serializedObjectType"
:
"pkl"
,
"serializedObjectType"
:
"pkl"
,
"model_fl_name"
:
"model_3f0d66a2.pkl"
"model_fl_name"
:
"model_48956f4d.pkl"
}
]
},
},
{
{
"id"
:
"site-new1"
,
"current_model"
:
{
"model_name"
:
"randomforest"
,
"model_name"
:
"randomforest"
,
"model_params"
:
null
,
"model_params"
:
null
,
"training_date"
:
"2021-03-10
"
,
"training_date"
:
"2021-03-11
"
,
"framework"
:
null
,
"framework"
:
null
,
"serializedObjectType"
:
"pkl"
,
"serializedObjectType"
:
"pkl"
,
"model_fl_name"
:
"model_7f17bf54
.pkl"
"model_fl_name"
:
"model_faed1e93
.pkl"
},
},
{
"archived"
:
[]
"model_name"
:
"randomforest"
,
"model_params"
:
null
,
"training_date"
:
"2021-03-10"
,
"framework"
:
null
,
"serializedObjectType"
:
"pkl"
,
"model_fl_name"
:
"model_07080eaa.pkl"
}
]
}
}
]
]
\ No newline at end of file
temp/mnt/ilens/ai-jobs/pipe1/default_run_id/AddtoModelStore/input/model_
9547643f_59d89573
.pkl
→
temp/mnt/ilens/ai-jobs/pipe1/default_run_id/AddtoModelStore/input/model_
78a80044
.pkl
View file @
461e5210
File moved
temp/mnt/ilens/ai-jobs/pipe1/default_run_id/AddtoModelStore/param.json
0 → 100644
View file @
461e5210
{
"ilens_tag_hierarchy"
:
"site-new"
}
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment