initial commit

.env

+# Database Configuration
+DB_HOST=192.168.0.220
+DB_PORT=3311
+DB_USER_NAME=pnp_ilens
+DB_PASSWORD=password
+DB_DATABASE=pnp_dev
+
+# Logger configuration
+LOG_FILE_NAME=ilens
+LOG_PATH=logs/
+LOG_LEVEL=DEBUG
+LOG_HANDLER=rotating_file_handler
+LOG_MAX_BYTES=10000000
+LOG_BACK_UP_COUNT=10
+
+USERNAME=FZ1-11D021$
+DB_TABLE_NAME=ms_access_config
\ No newline at end of file

Dockerfile

+FROM python:3.6.9
+RUN apt update && apt install -y gcc \
+    g++ build-essential\
+    libffi-dev openssl \
+    libxml2
+RUN apt install -y mdbtools
+RUN pip install --upgrade pip
+COPY requirements.txt /app/requirements.txt
+RUN pip install -r /app/requirements.txt
+RUN pip install meza
+COPY . /app
+WORKDIR /app
+CMD [ "python","app.py" ]
\ No newline at end of file

README.md

+# iLens Rockwell MS Access Acquisition
+
+
+### python: 3.6.10
+
+## Installation:
+```bash
+pip install -r requirements.txt
+```
+
+## Usage
+```bash
+python app.py
+```
+
+
+## For creating an installer
+```bash
+
+Add the following imports in the app.py
+
+import ijson.backends.yajl2_c
+import ijson.backends.yajl2_cffi
+import ijson.backends.yajl2
+import ijson.backends.python
+
+
+pip install PyInstaller==3.6
+
+apt-get install libyajl2 (ubuntu)
+cd {conda_env_path}/lib/python3.7/site-packages/PyInstaller/hooks/
+vi hook-pkg_resources.py
+Add "hiddenimports.append('pkg_resources.py2_warn')" above the last line to the file and save.
+
+pyinstaller app.py
+
+cp -r conf dist/app/
+
+```
+
+

app.py

+import time
+from dotenv import load_dotenv
+load_dotenv()
+from scripts.core.read_access_data import MsAccessMount
+
+if __name__=='__main__':
+    time.sleep(300)
+    MsAccessMount().process_data()
+

conf/application.conf

+[db]
+host= $DB_HOST
+port= $DB_PORT
+user_name= $DB_USER_NAME
+password= $DB_PASSWORD
+db_name= $DB_DATABASE
+table_name= $DB_TABLE_NAME
+
+
+[log]
+file_name= $LOG_FILE_NAME
+path= $LOG_PATH
+level= $LOG_LEVEL
+handler= $LOG_HANDLER
+max_bytes= $LOG_MAX_BYTES
+back_up_count= $LOG_BACK_UP_COUNT
+
+[service]
+user_name = $USERNAME
\ No newline at end of file

requirements.txt

+certifi==2020.4.5.1
+chardet==3.0.4
+dbfread==2.0.4
+idna==2.9
+ijson==2.6.1
+mdbtools==0.3.14
+meza==0.41.1
+mysql-connector-python==8.0.20
+numpy==1.18.4
+pandas==1.0.3
+protobuf==3.12.1
+pygogo==0.12.0
+pymongo==3.10.1
+python-dateutil==2.8.1
+python-slugify==1.2.6
+pytz==2020.1
+PyYAML==3.13
+requests==2.23.0
+six==1.15.0
+soupsieve==2.0.1
+Unidecode==1.1.1
+urllib3==1.25.9
+xlrd==1.2.0
+python-dotenv
\ No newline at end of file

scripts/constants/app_configuration.py

+import configparser
+import os
+
+class EnvInterpolation(configparser.BasicInterpolation):
+    """Interpolation which expands environment variables in values."""
+
+    def before_get(self, parser, section, option, value, defaults):
+        value = super().before_get(parser, section, option, value, defaults)
+
+        if not os.path.expandvars(value).startswith("$"):
+            return os.path.expandvars(value)
+        else:
+            return
+
+config = configparser.ConfigParser(interpolation=EnvInterpolation())
+CONFIGURATION_FILE = "conf/application.conf"
+config.read(CONFIGURATION_FILE)
+
+
+KEY_LOG = 'log'
+KEY_DB = 'db'
+KEY_SERVICE = 'service'
+# Log section
+LOG_FILE_NAME = config[KEY_LOG]["file_name"]
+LOG_PATH = config[KEY_LOG]["path"]
+LOG_LEVEL = config[KEY_LOG]["level"]
+LOG_HANDLER = config[KEY_LOG]["handler"]
+LOG_MAX_BYTES = int(config[KEY_LOG]["max_bytes"])
+LOG_BACKUP_COUNT = int(config[KEY_LOG]["back_up_count"])
+
+# DB
+
+HOST = config[KEY_DB]["host"]
+PORT = config[KEY_DB]["port"]
+USER_NAME = config[KEY_DB]["user_name"]
+PASSWORD = config[KEY_DB]["password"]
+DB = config[KEY_DB]["db_name"]
+TABLE = config[KEY_DB]["table_name"]
+
+# Service
+USERNAME = config.get(KEY_SERVICE, "user_name", fallback="None")
\ No newline at end of file

scripts/core/read_access_data.py

+
+import os
+import traceback
+from datetime import datetime
+import mysql.connector
+import pandas as pd
+from scripts.constants import app_configuration
+from scripts.utils.logger import logger as log
+
+DATA_DB_FILE = "manage.mdb"
+META_DB_FILE = "NOZZLE.MDB"
+META_TABLE = "Noztab1"
+
+MS_ACCESS_CONFIG = "ms_access_config"
+
+db_config = {
+    "host": app_configuration.HOST,
+    "port": app_configuration.PORT,
+    "user": app_configuration.USER_NAME,
+    "password": app_configuration.PASSWORD,
+    "database": app_configuration.DB,
+    "requestTimeout": 180000,
+    "options": {
+        "encrypt": True
+    }
+}
+
+
+class MsAccessMount(object):
+
+    def __init__(self):
+        self.conn_flag = 0
+        self.my_cursor = None
+        self.create_connection()
+        self.plant_dir_meta = None
+        self.last_modified = {}
+        self.plant_meta = []
+        self.table = app_configuration.TABLE
+        self.username = app_configuration.USERNAME
+
+    def read_plant_meta(self):
+        try:
+            log.info("Reading plant meta")
+            query = f"select * from {self.table} where user_name='{self.username}' ;"
+            return pd.read_sql(query, con=self.mydb)
+        except Exception as e:
+            log.error(f"Error occurred while reading plant meta data : {str(e)}")
+            raise Exception("Failed to read plant meta")
+
+    def create_connection(self):
+        try:
+            log.info("*****Creating mysql connection*****")
+            if not self.conn_flag:
+                self.mydb = mysql.connector.connect(
+                    host=db_config["host"],
+                    port=db_config ["port"],
+                    user=db_config["user"],
+                    passwd=db_config["password"],
+                    database=db_config["database"]
+                )
+                self.my_cursor = self.mydb.cursor()
+                log.info("Created connection --->")
+                self.conn_flag = 1
+        except Exception as e:
+            log.error("Error while connecting to mysql : {} !!!!!!!!!!!".format(str(e)))
+            raise Exception("Failed to connect to mysql!!!!!!!!!!!!!!")
+
+    def close_connection(self):
+        try:
+            log.info("*****Closing mysql connection*****")
+            if self.conn_flag:
+                self.my_cursor.close()
+                self.mydb.close()
+                self.conn_flag = 0
+        except Exception as e:
+            log.error("Error occurred while closing mysql connection : {} !!!!!!".format(str(e)))
+            self.my_cursor.close()
+            self.mydb.close()
+            self.conn_flag = 0
+
+
+    def __del__(self):
+        self.my_cursor.close()
+        self.mydb.close()
+
+    def validate_path(self, path):
+        try:
+            # Check if the path is a directory and if the dir exists
+            if not os.path.exists(path) and not os.path.isdir(path):
+                log.error("File Path not found")
+                self.update_failed_status(message="File Path not found")
+                return
+
+            # Check if the directory is readable
+            if not os.access(path, os.R_OK):
+                log.error("Directory is not readable")
+                self.update_failed_status(message="Directory is not readable")
+                return
+            
+            # Check if the required files with the given extension exist
+            required_files = ["NOZZLE.MDB", "manage.1db", "manage.mdb"]
+            missing_files = []
+            for filename in required_files:
+                lowercase_filename = filename.lower()
+                file_path = os.path.join(path, filename)
+                if not os.path.exists(file_path) and not os.path.exists(os.path.join(path, lowercase_filename)):
+                    missing_files.append(filename)
+
+            if missing_files:
+                log.error("Some files are missing")
+                self.update_failed_status(message="Some files are missing")
+                return
+
+            self.update_success_status(message="MS Access mounted successfully")
+            return
+        except Exception:
+            log.error("Exception while validating file path", exc_info=True)
+
+    def update_failed_status(self, message):
+        try:
+            update_ms_file_access_check = f"update {self.table} set status='failed', message={message} where user_name = '{self.username}'"
+            resp = self.my_cursor.execute(update_ms_file_access_check)
+            self.mydb.commit()
+        except Exception as e:
+            log.exception(f"Exception when updating status as Failed :{str(e)}")
+            
+    def update_success_status(self, message):
+        try:
+            log.info("Mounts validated successfully")
+            update_ms_file_access_check = f"update {self.table} set status='success', message={message} where user_name = '{self.username}'"
+            resp = self.my_cursor.execute(update_ms_file_access_check)
+            self.mydb.commit()
+        except Exception as e:
+            log.exception(f"Exception when updating status as Success {str(e)}")
+            
+    def process_data(self):
+        log.debug("Started Mountd path testing for Access DB")
+        try:
+            start_time = datetime.now()
+            self.conn_flag = 0
+            self.create_connection()
+            log.info("Reading and refreshing the plant meta")
+            self.plant_meta = self.read_plant_meta()
+            self.plant_dir_meta = self.plant_meta.to_dict(orient='records')
+            log.info(f"---------------> META ----------------> {self.plant_dir_meta}")
+            if len(self.plant_dir_meta):
+                self.validate_path(self.plant_dir_meta[0]["mounted_path"])
+            end_time = datetime.now()
+            diff = (end_time - start_time).total_seconds()
+            log.debug("Time taken for execution : {} seconds".format(diff))
+            self.close_connection()
+            # break
+        except Exception as ex:
+            log.error("Error occurred while reading the data - main method --> %s", str(ex))
+            traceback.print_exc()
+            pass
\ No newline at end of file

scripts/utils/logger.py

+import logging
+import os
+from logging.handlers import RotatingFileHandler
+from scripts.constants import app_configuration
+
+_log_file_name__ = str(app_configuration.LOG_FILE_NAME)
+__log_path__ = str(app_configuration.LOG_PATH)
+__log_level__ = str(app_configuration.LOG_LEVEL)
+__handler_type__ = str(app_configuration.LOG_HANDLER)
+__max_bytes__ = int(app_configuration.LOG_MAX_BYTES)
+__backup_count__ = int(app_configuration.LOG_BACKUP_COUNT)
+complete_log_path = os.path.join(__log_path__, _log_file_name__)
+if not os.path.isdir(__log_path__):
+    os.makedirs(__log_path__)
+
+
+def get_logger(log_file_name=complete_log_path, log_level=__log_level__, time_format="%Y-%m-%d %H:%M:%S",
+               handler_type=__handler_type__, max_bytes=__max_bytes__, backup_count=__backup_count__):
+    """
+     Creates a rotating log
+     """
+    log_file = os.path.join(log_file_name + '.log')
+    __logger__ = logging.getLogger(log_file_name)
+    __logger__.setLevel(log_level.strip().upper())
+    debug_formatter = '%(asctime)s - %(levelname)-6s - %(name)s - [%(threadName)5s:%(filename)5s:' \
+                      '%(funcName)5s():''%(lineno)s] - %(message)s'
+    formatter_string = '%(asctime)s - %(levelname)-6s - %(name)s - %(levelname)3s - %(message)s'
+
+    if log_level.strip().upper() == log_level:
+        formatter_string = debug_formatter
+
+    formatter = logging.Formatter(formatter_string, time_format)
+
+    # if str(handler_type).lower() == "console_handler":
+    # Console Handler
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+    # if __logger__.hasHandlers():
+    #     __logger__.handlers.clear()
+    __logger__.addHandler(console_handler)
+
+    if str(handler_type).lower() == "rotating_file_handler":
+        # Rotating File Handler
+        handler = RotatingFileHandler(log_file, maxBytes=max_bytes, backupCount=backup_count)
+        handler.setFormatter(formatter)
+        # if __logger__.hasHandlers():
+        #     __logger__.handlers.clear()
+        __logger__.addHandler(handler)
+
+    else:
+        # File Handler
+        hdlr_service = logging.FileHandler(log_file)
+        hdlr_service.setFormatter(formatter)
+        # if __logger__.hasHandlers():
+        #     __logger__.handlers.clear()
+        __logger__.addHandler(hdlr_service)
+
+    return __logger__
+
+
+logger = get_logger()