clean up FTP

clean_up.py

@@ -16,7 +16,7 @@ __console_handler = StreamHandler(sys.stdout)
 __console_handler.setFormatter(__formatter)
 logger.addHandler(__console_handler)
 
-check_for_n_hours = int(os.environ.get('check_for_n_hours', default='3'))
+check_for_n_hours = int(os.environ.get('check_for_n_hours', default='1'))
 THREAD_POOL_LIMIT = os.environ.get("THREAD_POOL_LIMIT", default=10)
 FTP_HOST = os.environ.get("FTP_HOST", default="hfe-ftp.ilens.io")
 FTP_USER = os.environ.get("FTP_USER", default="ilenshero1")
@@ -26,6 +26,7 @@ FTP_IO_DIRECTORY = os.environ.get("FTP_IO_DIRECTORY", default="/upload/DATA/IO/"
 FTP_WPM_DIRECTORY = os.environ.get("FTP_WPM_DIRECTORY", default="/upload/DATA/")
 FTP_TARGET_DIRECTORY = os.environ.get("FTP_DIRECTORY")
 NUMBER_OF_SPLITS = int(os.environ.get("NUMBER_OF_SPLITS", 20))
+enable_traceback = False
 
 
 class SmartFTP(FTP):
@@ -55,6 +56,7 @@ if __name__ == "__main__":
     pg = Postgres()
     time_stamp = datetime.datetime.utcnow() - datetime.timedelta(hours=check_for_n_hours)
     file_list = pg.get_file_names('hfe_master_file_status', time_stamp)
+    pg.close_connection()
     final_file_list = []
     WPM_list = []
     for i in range(0, len(file_list)):
@@ -62,15 +64,13 @@ if __name__ == "__main__":
             WPM_list.append(file_list[i])
         else:
             final_file_list.append(file_list[i])
-    print(len(file_list), file_list)
+    logger.info(f"Total files after {time_stamp} are " + str(len(file_list)))
     IO_list = [i for i in final_file_list if 'IO' in i]
     MODBUS_list = [i for i in final_file_list if 'MODBUS' in i]
-    print(len(final_file_list), final_file_list)
-    print(len(WPM_list), WPM_list)
-    print(len(IO_list), IO_list)
-    print(len(MODBUS_list), MODBUS_list)
-    logger.info("Closing Postgre Connection")
-    pg.close_connection()
+    logger.debug(f"Total WPM files after {time_stamp} are " + str(len(WPM_list)))
+    logger.debug(f"Total IO files after {time_stamp} are " + str(len(IO_list)))
+    logger.debug(f"Total MODBUS files after {time_stamp} are " + str(len(MODBUS_list)))
+    logger.debug("Closing Postgre Connection")
     THREAD_POOL_LIMIT = 20
     pool = ThreadPool(int(THREAD_POOL_LIMIT))
     if len(WPM_list) > 0:
@@ -79,7 +79,7 @@ if __name__ == "__main__":
             prod_x = partial(remove, source_directory=FTP_WPM_DIRECTORY)
             map_output = pool.map(prod_x, WPM_list)
         except Exception as e:
-            logger.info(e)
+            logger.error(f"Error when removing WPM files from FTP {e}", exc_info=enable_traceback)
     else:
         logger.info(f"No WPM files after {time_stamp}")
     if len(IO_list) > 0:
@@ -88,7 +88,7 @@ if __name__ == "__main__":
             prod_x = partial(remove, source_directory=FTP_IO_DIRECTORY)
             map_output = pool.map(prod_x, IO_list)
         except Exception as e:
-            logger.info(e)
+            logger.error(f"Error when removing IO files from FTP {e}", exc_info=enable_traceback)
     else:
         logger.info(f"No IO files after {time_stamp}")
     if len(MODBUS_list) > 0:
@@ -97,6 +97,6 @@ if __name__ == "__main__":
             prod_x = partial(remove, source_directory=FTP_MODBUS_DIRECTORY)
             map_output = pool.map(prod_x, MODBUS_list)
         except Exception as e:
-            logger.info(e)
+            logger.error(f"Error when removing MODBUS files from FTP {e}", exc_info=enable_traceback)
     else:
         logger.info(f"No MODBUS files after {time_stamp}")