Update delete_local_data

parent 42aa8e78
import os import os
import shutil import shutil
from pathlib import Path from pathlib import Path
from sqlalchemy import create_engine
import logging import logging
job_id = os.environ.get('JOB_ID', 'job_1') postgres_database = os.environ.get('postgres_database', default='ilens_jubilant')
directory = os.environ.get("LOCAL_PATH", 'ilens/prod/jubilant') postgre_user = os.environ.get('postgre_user', default='ilens')
postgre_password = os.environ.get('postgre_password', default='iLens#4321')
postgre_host = os.environ.get('postgre_host', default='postgres-db-service.ilens-infra.svc.cluster.local')
postgre_port = os.environ.get('postgre_port', default='5432')
job_id = os.environ.get('JOB_ID', 'job_6')
directory = os.environ.get("LOCAL_PATH", '/mnt/ilens/prod/jubilant')
directory = os.path.join(directory, job_id) directory = os.path.join(directory, job_id)
class DeleteLocalData: class DeleteLocalData:
def __init__(self):
try:
print("Connecting to postgres .......")
self.engine_data = 'postgresql+psycopg2://{}:{}@{}:{}/{}'.format(postgre_user, postgre_password,
postgre_host,
postgre_port, postgres_database)
self.engine = create_engine(self.engine_data)
except Exception as e:
print(e)
def delete_local_data(self): def delete_local_data(self):
try: try:
...@@ -16,7 +32,7 @@ class DeleteLocalData: ...@@ -16,7 +32,7 @@ class DeleteLocalData:
dis = os.path.join(directory, filename) dis = os.path.join(directory, filename)
if os.path.isfile(dis): if os.path.isfile(dis):
os.remove(dis) os.remove(dis)
logging.info("Removed CSV files from base directory") print("Removed CSV files from base directory")
dis_list = [] dis_list = []
for filename in os.listdir(directory): for filename in os.listdir(directory):
...@@ -30,10 +46,26 @@ class DeleteLocalData: ...@@ -30,10 +46,26 @@ class DeleteLocalData:
path_ = Path(dis) path_ = Path(dis)
if os.path.isdir(dis): if os.path.isdir(dis):
shutil.rmtree(dis) shutil.rmtree(dis)
logging.info("Removed all the subdirectories in Base directory") print("Removed all the subdirectories in Base directory")
conn = self.engine.raw_connection()
cur = conn.cursor()
cur.execute("SELECT job_id, status from audit_data_table")
row = cur.fetchall()
data_name_list = []
data_status_list = []
curr_act = 'deleted'
for i in row:
data_name_list.append(i[0])
data_status_list.append(i[1])
data_updated_dict = dict(zip(data_name_list, data_status_list))
for key, value in data_updated_dict.items():
if value == 'Success':
cur.execute('''UPDATE audit_data_table SET current_activity = '{}' WHERE status = '{}';'''.format(curr_act, value))
conn.commit()
except Exception as e: except Exception as e:
print(e)
logging.error(e) logging.error(e)
delete_local_files = DeleteLocalData() delete_local_files = DeleteLocalData()
delete_local_files.delete_local_data() delete_local_files.delete_local_data()
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment