Add new file

delete_local_data/delete_local_data

+import os
+import shutil
+from pathlib import Path
+import logging
+job_id = os.environ.get('JOB_ID', 'job_1')
+directory = os.environ.get("LOCAL_PATH", 'ilens/prod/jubilant')
+directory = os.path.join(directory, job_id)
+
+
+class DeleteLocalData:
+
+    def delete_local_data(self):
+        try:
+            for filename in os.listdir(directory):
+                if filename.endswith('.csv'):
+                    dis = os.path.join(directory, filename)
+                    if os.path.isfile(dis):
+                        os.remove(dis)
+            logging.info("Removed CSV files from base directory")
+
+            dis_list = []
+            for filename in os.listdir(directory):
+                filename = filename.split('.')
+                if len(filename) == 1:
+                    if filename[0] not in dis_list:
+                        dis_list.append(filename[0])
+            for filename_ in dis_list:
+                dis = ''
+                dis = os.path.join(directory, filename_)
+                path_ = Path(dis)
+                if os.path.isdir(dis):
+                    shutil.rmtree(dis)
+            logging.info("Removed all the subdirectories in Base directory")
+        except Exception as e:
+            logging.error(e)
+
+
+delete_local_files = DeleteLocalData()
+delete_local_files.delete_local_data()