Commit 672a4256 authored by obbinti.rao's avatar obbinti.rao

initial commit

parent d7144568
# Database Configuration
DB_HOST=192.168.0.220
DB_PORT=3311
DB_USER_NAME=pnp_ilens
DB_PASSWORD=password
DB_DATABASE=pnp_dev
# Logger configuration
LOG_FILE_NAME=ilens
LOG_PATH=logs/
LOG_LEVEL=DEBUG
LOG_HANDLER=rotating_file_handler
LOG_MAX_BYTES=10000000
LOG_BACK_UP_COUNT=10
USERNAME=FZ1-11D021$
DB_TABLE_NAME=ms_access_config
\ No newline at end of file
# Created by .ignore support plugin (hsz.mobi)
### JetBrains template
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
reports/*.pdf
reports/*.csv
reports/*.xlsx
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
.idea
logs
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
### Python template
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
### VisualStudio template
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
##
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
# User-specific files
*.rsuser
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Mono auto generated files
mono_crash.*
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
[Ww][Ii][Nn]32/
[Aa][Rr][Mm]/
[Aa][Rr][Mm]64/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
[Ll]ogs/
# Visual Studio 2015/2017 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# Visual Studio 2017 auto generated files
Generated\ Files/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUnit
*.VisualState.xml
TestResult.xml
nunit-*.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# Benchmark Results
BenchmarkDotNet.Artifacts/
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
# ASP.NET Scaffolding
ScaffoldingReadMe.txt
# StyleCop
StyleCopReport.xml
# Files built by Visual Studio
*_i.c
*_p.c
*_h.h
*.ilk
*.meta
*.obj
*.iobj
*.pch
*.pdb
*.ipdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*_wpftmp.csproj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# Visual Studio Trace Files
*.e2e
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# AxoCover is a Code Coverage Tool
.axoCover/*
!.axoCover/settings.json
# Coverlet is a free, cross platform Code Coverage Tool
coverage*.json
coverage*.xml
coverage*.info
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# Note: Comment the next line if you want to checkin your web deploy settings,
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# NuGet Symbol Packages
*.snupkg
# The packages folder can be ignored because of Package Restore
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
*.appx
*.appxbundle
*.appxupload
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!?*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
ServiceFabricBackup/
*.rptproj.bak
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
*.rptproj.rsuser
*- [Bb]ackup.rdl
*- [Bb]ackup ([0-9]).rdl
*- [Bb]ackup ([0-9][0-9]).rdl
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# CodeRush personal settings
.cr/personal
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Tabs Studio
*.tss
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
*.binlog
# NVidia Nsight GPU debugger configuration file
*.nvuser
# MFractors (Xamarin productivity tool) working folder
.mfractor/
# Local History for Visual Studio
.localhistory/
# BeatPulse healthcheck temp database
healthchecksdb
# Backup folder for Package Reference Convert tool in Visual Studio 2017
MigrationBackup/
# Ionide (cross platform F# VS Code tools) working folder
.ionide/
# Fody - auto-generated XML schema
FodyWeavers.xsd
### JupyterNotebooks template
# gitignore template for Jupyter Notebooks
# website: http://jupyter.org/
*/.ipynb_checkpoints/*
# Remove previous ipynb_checkpoints
# git rm -r .ipynb_checkpoints/
.temp/
.vscode/settings.json
.vscode/launch.json
.zip
.csv
FROM python:3.6.9
RUN apt update && apt install -y gcc \
g++ build-essential\
libffi-dev openssl \
libxml2
RUN apt install -y mdbtools
RUN pip install --upgrade pip
COPY requirements.txt /app/requirements.txt
RUN pip install -r /app/requirements.txt
RUN pip install meza
COPY . /app
WORKDIR /app
CMD [ "python","app.py" ]
\ No newline at end of file
# iLens Rockwell MS Access Acquisition
### python: 3.6.10
## Installation:
```bash
pip install -r requirements.txt
```
## Usage
```bash
python app.py
```
## For creating an installer
```bash
Add the following imports in the app.py
import ijson.backends.yajl2_c
import ijson.backends.yajl2_cffi
import ijson.backends.yajl2
import ijson.backends.python
pip install PyInstaller==3.6
apt-get install libyajl2 (ubuntu)
cd {conda_env_path}/lib/python3.7/site-packages/PyInstaller/hooks/
vi hook-pkg_resources.py
Add "hiddenimports.append('pkg_resources.py2_warn')" above the last line to the file and save.
pyinstaller app.py
cp -r conf dist/app/
```
import time
from dotenv import load_dotenv
load_dotenv()
from scripts.core.read_access_data import MsAccessMount
if __name__=='__main__':
time.sleep(300)
MsAccessMount().process_data()
[db]
host= $DB_HOST
port= $DB_PORT
user_name= $DB_USER_NAME
password= $DB_PASSWORD
db_name= $DB_DATABASE
table_name= $DB_TABLE_NAME
[log]
file_name= $LOG_FILE_NAME
path= $LOG_PATH
level= $LOG_LEVEL
handler= $LOG_HANDLER
max_bytes= $LOG_MAX_BYTES
back_up_count= $LOG_BACK_UP_COUNT
[service]
user_name = $USERNAME
\ No newline at end of file
certifi==2020.4.5.1
chardet==3.0.4
dbfread==2.0.4
idna==2.9
ijson==2.6.1
mdbtools==0.3.14
meza==0.41.1
mysql-connector-python==8.0.20
numpy==1.18.4
pandas==1.0.3
protobuf==3.12.1
pygogo==0.12.0
pymongo==3.10.1
python-dateutil==2.8.1
python-slugify==1.2.6
pytz==2020.1
PyYAML==3.13
requests==2.23.0
six==1.15.0
soupsieve==2.0.1
Unidecode==1.1.1
urllib3==1.25.9
xlrd==1.2.0
python-dotenv
\ No newline at end of file
import configparser
import os
class EnvInterpolation(configparser.BasicInterpolation):
"""Interpolation which expands environment variables in values."""
def before_get(self, parser, section, option, value, defaults):
value = super().before_get(parser, section, option, value, defaults)
if not os.path.expandvars(value).startswith("$"):
return os.path.expandvars(value)
else:
return
config = configparser.ConfigParser(interpolation=EnvInterpolation())
CONFIGURATION_FILE = "conf/application.conf"
config.read(CONFIGURATION_FILE)
KEY_LOG = 'log'
KEY_DB = 'db'
KEY_SERVICE = 'service'
# Log section
LOG_FILE_NAME = config[KEY_LOG]["file_name"]
LOG_PATH = config[KEY_LOG]["path"]
LOG_LEVEL = config[KEY_LOG]["level"]
LOG_HANDLER = config[KEY_LOG]["handler"]
LOG_MAX_BYTES = int(config[KEY_LOG]["max_bytes"])
LOG_BACKUP_COUNT = int(config[KEY_LOG]["back_up_count"])
# DB
HOST = config[KEY_DB]["host"]
PORT = config[KEY_DB]["port"]
USER_NAME = config[KEY_DB]["user_name"]
PASSWORD = config[KEY_DB]["password"]
DB = config[KEY_DB]["db_name"]
TABLE = config[KEY_DB]["table_name"]
# Service
USERNAME = config.get(KEY_SERVICE, "user_name", fallback="None")
\ No newline at end of file
import os
import traceback
from datetime import datetime
import mysql.connector
import pandas as pd
from scripts.constants import app_configuration
from scripts.utils.logger import logger as log
DATA_DB_FILE = "manage.mdb"
META_DB_FILE = "NOZZLE.MDB"
META_TABLE = "Noztab1"
MS_ACCESS_CONFIG = "ms_access_config"
db_config = {
"host": app_configuration.HOST,
"port": app_configuration.PORT,
"user": app_configuration.USER_NAME,
"password": app_configuration.PASSWORD,
"database": app_configuration.DB,
"requestTimeout": 180000,
"options": {
"encrypt": True
}
}
class MsAccessMount(object):
def __init__(self):
self.conn_flag = 0
self.my_cursor = None
self.create_connection()
self.plant_dir_meta = None
self.last_modified = {}
self.plant_meta = []
self.table = app_configuration.TABLE
self.username = app_configuration.USERNAME
def read_plant_meta(self):
try:
log.info("Reading plant meta")
query = f"select * from {self.table} where user_name='{self.username}' ;"
return pd.read_sql(query, con=self.mydb)
except Exception as e:
log.error(f"Error occurred while reading plant meta data : {str(e)}")
raise Exception("Failed to read plant meta")
def create_connection(self):
try:
log.info("*****Creating mysql connection*****")
if not self.conn_flag:
self.mydb = mysql.connector.connect(
host=db_config["host"],
port=db_config ["port"],
user=db_config["user"],
passwd=db_config["password"],
database=db_config["database"]
)
self.my_cursor = self.mydb.cursor()
log.info("Created connection --->")
self.conn_flag = 1
except Exception as e:
log.error("Error while connecting to mysql : {} !!!!!!!!!!!".format(str(e)))
raise Exception("Failed to connect to mysql!!!!!!!!!!!!!!")
def close_connection(self):
try:
log.info("*****Closing mysql connection*****")
if self.conn_flag:
self.my_cursor.close()
self.mydb.close()
self.conn_flag = 0
except Exception as e:
log.error("Error occurred while closing mysql connection : {} !!!!!!".format(str(e)))
self.my_cursor.close()
self.mydb.close()
self.conn_flag = 0
def __del__(self):
self.my_cursor.close()
self.mydb.close()
def validate_path(self, path):
try:
# Check if the path is a directory and if the dir exists
if not os.path.exists(path) and not os.path.isdir(path):
log.error("File Path not found")
self.update_failed_status(message="File Path not found")
return
# Check if the directory is readable
if not os.access(path, os.R_OK):
log.error("Directory is not readable")
self.update_failed_status(message="Directory is not readable")
return
# Check if the required files with the given extension exist
required_files = ["NOZZLE.MDB", "manage.1db", "manage.mdb"]
missing_files = []
for filename in required_files:
lowercase_filename = filename.lower()
file_path = os.path.join(path, filename)
if not os.path.exists(file_path) and not os.path.exists(os.path.join(path, lowercase_filename)):
missing_files.append(filename)
if missing_files:
log.error("Some files are missing")
self.update_failed_status(message="Some files are missing")
return
self.update_success_status(message="MS Access mounted successfully")
return
except Exception:
log.error("Exception while validating file path", exc_info=True)
def update_failed_status(self, message):
try:
update_ms_file_access_check = f"update {self.table} set status='failed', message={message} where user_name = '{self.username}'"
resp = self.my_cursor.execute(update_ms_file_access_check)
self.mydb.commit()
except Exception as e:
log.exception(f"Exception when updating status as Failed :{str(e)}")
def update_success_status(self, message):
try:
log.info("Mounts validated successfully")
update_ms_file_access_check = f"update {self.table} set status='success', message={message} where user_name = '{self.username}'"
resp = self.my_cursor.execute(update_ms_file_access_check)
self.mydb.commit()
except Exception as e:
log.exception(f"Exception when updating status as Success {str(e)}")
def process_data(self):
log.debug("Started Mountd path testing for Access DB")
try:
start_time = datetime.now()
self.conn_flag = 0
self.create_connection()
log.info("Reading and refreshing the plant meta")
self.plant_meta = self.read_plant_meta()
self.plant_dir_meta = self.plant_meta.to_dict(orient='records')
log.info(f"---------------> META ----------------> {self.plant_dir_meta}")
if len(self.plant_dir_meta):
self.validate_path(self.plant_dir_meta[0]["mounted_path"])
end_time = datetime.now()
diff = (end_time - start_time).total_seconds()
log.debug("Time taken for execution : {} seconds".format(diff))
self.close_connection()
# break
except Exception as ex:
log.error("Error occurred while reading the data - main method --> %s", str(ex))
traceback.print_exc()
pass
\ No newline at end of file
import logging
import os
from logging.handlers import RotatingFileHandler
from scripts.constants import app_configuration
_log_file_name__ = str(app_configuration.LOG_FILE_NAME)
__log_path__ = str(app_configuration.LOG_PATH)
__log_level__ = str(app_configuration.LOG_LEVEL)
__handler_type__ = str(app_configuration.LOG_HANDLER)
__max_bytes__ = int(app_configuration.LOG_MAX_BYTES)
__backup_count__ = int(app_configuration.LOG_BACKUP_COUNT)
complete_log_path = os.path.join(__log_path__, _log_file_name__)
if not os.path.isdir(__log_path__):
os.makedirs(__log_path__)
def get_logger(log_file_name=complete_log_path, log_level=__log_level__, time_format="%Y-%m-%d %H:%M:%S",
handler_type=__handler_type__, max_bytes=__max_bytes__, backup_count=__backup_count__):
"""
Creates a rotating log
"""
log_file = os.path.join(log_file_name + '.log')
__logger__ = logging.getLogger(log_file_name)
__logger__.setLevel(log_level.strip().upper())
debug_formatter = '%(asctime)s - %(levelname)-6s - %(name)s - [%(threadName)5s:%(filename)5s:' \
'%(funcName)5s():''%(lineno)s] - %(message)s'
formatter_string = '%(asctime)s - %(levelname)-6s - %(name)s - %(levelname)3s - %(message)s'
if log_level.strip().upper() == log_level:
formatter_string = debug_formatter
formatter = logging.Formatter(formatter_string, time_format)
# if str(handler_type).lower() == "console_handler":
# Console Handler
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
# if __logger__.hasHandlers():
# __logger__.handlers.clear()
__logger__.addHandler(console_handler)
if str(handler_type).lower() == "rotating_file_handler":
# Rotating File Handler
handler = RotatingFileHandler(log_file, maxBytes=max_bytes, backupCount=backup_count)
handler.setFormatter(formatter)
# if __logger__.hasHandlers():
# __logger__.handlers.clear()
__logger__.addHandler(handler)
else:
# File Handler
hdlr_service = logging.FileHandler(log_file)
hdlr_service.setFormatter(formatter)
# if __logger__.hasHandlers():
# __logger__.handlers.clear()
__logger__.addHandler(hdlr_service)
return __logger__
logger = get_logger()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment