Skip to content

D
DB Upgrade Python Script
Commit cf8fef09 authored by archana's avatar archana
Browse files
Options

constant collection site template update

parents
Hide whitespace changes
Inline Side-by-side
Showing with 668 additions and 0 deletions
.idea/.gitignore 0 → 100644
View file @ cf8fef09
# Default ignored files
/shelf/
/workspace.xml
.idea/inspectionProfiles/Project_Default.xml 0 → 100644
View file @ cf8fef09
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="57">
<item index="0" class="java.lang.String" itemvalue="scipy" />
<item index="1" class="java.lang.String" itemvalue="pydantic" />
<item index="2" class="java.lang.String" itemvalue="Flask-Cors" />
<item index="3" class="java.lang.String" itemvalue="waitress" />
<item index="4" class="java.lang.String" itemvalue="xhtml2pdf" />
<item index="5" class="java.lang.String" itemvalue="reportlab" />
<item index="6" class="java.lang.String" itemvalue="numpy" />
<item index="7" class="java.lang.String" itemvalue="requests" />
<item index="8" class="java.lang.String" itemvalue="gunicorn" />
<item index="9" class="java.lang.String" itemvalue="Flask-Compress" />
<item index="10" class="java.lang.String" itemvalue="pandas" />
<item index="11" class="java.lang.String" itemvalue="word2number" />
<item index="12" class="java.lang.String" itemvalue="paho-mqtt" />
<item index="13" class="java.lang.String" itemvalue="pycryptodome" />
<item index="14" class="java.lang.String" itemvalue="PyPDF2" />
<item index="15" class="java.lang.String" itemvalue="pytz" />
<item index="16" class="java.lang.String" itemvalue="python-jose" />
<item index="17" class="java.lang.String" itemvalue="pyjwt" />
<item index="18" class="java.lang.String" itemvalue="bson" />
<item index="19" class="java.lang.String" itemvalue="ujson" />
<item index="20" class="java.lang.String" itemvalue="getmac" />
<item index="21" class="java.lang.String" itemvalue="boto3" />
<item index="22" class="java.lang.String" itemvalue="botocore" />
<item index="23" class="java.lang.String" itemvalue="captcha" />
<item index="24" class="java.lang.String" itemvalue="python-dotenv" />
<item index="25" class="java.lang.String" itemvalue="cryptography" />
<item index="26" class="java.lang.String" itemvalue="boltons" />
<item index="27" class="java.lang.String" itemvalue="iteration-utilities" />
<item index="28" class="java.lang.String" itemvalue="pymongo" />
<item index="29" class="java.lang.String" itemvalue="pyyaml" />
<item index="30" class="java.lang.String" itemvalue="werkzeug" />
<item index="31" class="java.lang.String" itemvalue="six" />
<item index="32" class="java.lang.String" itemvalue="flask-cors" />
<item index="33" class="java.lang.String" itemvalue="gevent" />
<item index="34" class="java.lang.String" itemvalue="flask" />
<item index="35" class="java.lang.String" itemvalue="redis" />
<item index="36" class="java.lang.String" itemvalue="bcrypt" />
<item index="37" class="java.lang.String" itemvalue="Werkzeug" />
<item index="38" class="java.lang.String" itemvalue="pyaml" />
<item index="39" class="java.lang.String" itemvalue="aniso8601" />
<item index="40" class="java.lang.String" itemvalue="pycryptodomex" />
<item index="41" class="java.lang.String" itemvalue="Jinja2" />
<item index="42" class="java.lang.String" itemvalue="iteration_utilities" />
<item index="43" class="java.lang.String" itemvalue="rsa" />
<item index="44" class="java.lang.String" itemvalue="greenlet" />
<item index="45" class="java.lang.String" itemvalue="simple-crypt" />
<item index="46" class="java.lang.String" itemvalue="PyYAML" />
<item index="47" class="java.lang.String" itemvalue="pillow" />
<item index="48" class="java.lang.String" itemvalue="xlsxwriter" />
<item index="49" class="java.lang.String" itemvalue="pyopenssl" />
<item index="50" class="java.lang.String" itemvalue="APScheduler" />
<item index="51" class="java.lang.String" itemvalue="Flask" />
<item index="52" class="java.lang.String" itemvalue="openpyxl" />
<item index="53" class="java.lang.String" itemvalue="crypto" />
<item index="54" class="java.lang.String" itemvalue="PyJWT" />
<item index="55" class="java.lang.String" itemvalue="configparser" />
<item index="56" class="java.lang.String" itemvalue="msal" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N806" />
<option value="N801" />
<option value="N812" />
<option value="N803" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyShadowingBuiltinsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredNames">
<list>
<option value="type" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="scripts.constants.app_configuration.configparser" />
<option value="str.__setitem__" />
<option value="dict.__getitem__" />
<option value="dict.update" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
View file @ cf8fef09
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
.idea/misc.xml 0 → 100644
View file @ cf8fef09
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (meta-services)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
.idea/modules.xml 0 → 100644
View file @ cf8fef09
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/tag-hierarchy-update.iml" filepath="$PROJECT_DIR$/.idea/tag-hierarchy-update.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/tag-hierarchy-update.iml 0 → 100644
View file @ cf8fef09
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.8 (meta-services)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
.idea/vcs.xml 0 → 100644
View file @ cf8fef09
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CommitMessageInspectionProfile">
<profile version="1.0">
<inspection_tool class="CommitFormat" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="CommitNamingConvention" enabled="true" level="WARNING" enabled_by_default="true" />
</profile>
</component>
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
Log/ilens.log 0 → 100644
View file @ cf8fef09
2021-06-21 11:49:43 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-06-21 11:50:15 - INFO - Log/ilens - [MainThread:update_handler.py:tag_hierarchy_update():22] - Records fetched from following collection Successfully
2021-06-21 11:50:15 - INFO - Log/ilens - [MainThread:update_handler.py:tag_hierarchy_update():23] - 'database_name': 'ilens_configuration' 'collection_name': 'site_conf'
2021-06-21 11:50:17 - INFO - Log/ilens - [MainThread:update_handler.py:tag_hierarchy_update():27] - Deleted all records from following collection
2021-06-21 11:50:17 - INFO - Log/ilens - [MainThread:update_handler.py:tag_hierarchy_update():28] - 'database_name': 'ilens_configuration' 'collection_name': 'tag_hierarchy'
2021-06-21 11:51:03 - INFO - Log/ilens - [MainThread:update_handler.py:tag_hierarchy_update():33] - Updated the following collection Successfully
2021-06-21 11:51:03 - INFO - Log/ilens - [MainThread:update_handler.py:tag_hierarchy_update():34] - 'database_name': 'ilens_configuration' 'collection_name': 'tag_hierarchy'
2021-07-28 16:36:53 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-07-28 16:36:53 - ERROR - Log/ilens - [MainThread:main.py:<module>():13] - Exception in the main block__init__() got an unexpected keyword argument 'type'
Traceback (most recent call last):
File "E:/user-project-update/main.py", line 7, in <module>
status = UpdateHandler().constants_update()
File "E:\user-project-update\scripts\handler\update_handler.py", line 43, in constants_update
constants_records = self.constants_mongo.find_constant(_type="site_template")
File "E:\user-project-update\scripts\db\mongo\collections\constants.py", line 56, in find_constant
return ConstantsSchema(**record)
TypeError: __init__() got an unexpected keyword argument 'type'
2021-07-28 16:37:07 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-07-28 16:37:07 - ERROR - Log/ilens - [MainThread:main.py:<module>():13] - Exception in the main block__init__() got an unexpected keyword argument 'type'
Traceback (most recent call last):
File "E:/user-project-update/main.py", line 7, in <module>
status = UpdateHandler().constants_update()
File "E:\user-project-update\scripts\handler\update_handler.py", line 43, in constants_update
constants_records = self.constants_mongo.find_constant(_type="site_template")
File "E:\user-project-update\scripts\db\mongo\collections\constants.py", line 56, in find_constant
return ConstantsSchema(**record)
TypeError: __init__() got an unexpected keyword argument 'type'
2021-07-28 16:37:21 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-07-28 16:37:55 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-07-28 16:40:18 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-07-28 16:41:02 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-07-28 16:41:48 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-07-28 16:42:09 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
2021-07-28 16:44:49 - DEBUG - Log/ilens - [MainThread:main.py:<module>():6] - main block started
README.md 0 → 100644
View file @ cf8fef09
Constant collection update
\ No newline at end of file
conf/service.yml 0 → 100644
View file @ cf8fef09
log:
file_name: ilens
path: Log/
level: DEBUG
handler: rotating_file_handler
max_bytes: 10000000
back_up_count: 10
mongo_db:
URI: mongodb://192.168.0.220:2717/
\ No newline at end of file
main.py 0 → 100644
View file @ cf8fef09
from scripts.logging.logger import logger
from scripts.handler.update_handler import UpdateHandler
if __name__ == '__main__':
try:
logger.debug("main block started")
status = UpdateHandler().constants_update()
if status:
print("Updated successfully")
else:
print("Some records are not updated successfully, Please check the log file")
except Exception as e:
logger.exception("Exception in the main block" + str(e))
scripts/config/__init__.py 0 → 100644
View file @ cf8fef09
from pydantic import BaseSettings
import sys
import yaml
# this method is to read the configuration from backup.conf
def read_configuration(file_name):
"""
:param file_name:
:return: all the configuration constants
"""
with open(file_name, 'r') as stream:
try:
return yaml.full_load(stream)
except yaml.YAMLError as exc:
print(exc)
config = read_configuration("conf/service.yml")
class _Configuration(BaseSettings):
MONGO_URI = config["mongo_db"]["URI"]
_conf = _Configuration()
class DB(object):
class MongoDb:
uri = _conf.MONGO_URI
print(uri)
if not uri:
print("Error, environment variable MONGO_URI not set")
sys.exit(1)
class LOG:
LOG_FILE_NAME = config["log"]["file_name"]
LOG_PATH = config["log"]["path"]
LOG_LEVEL = config["log"]["level"]
LOG_HANDLER = config["log"]["handler"]
LOG_MAX_BYTES = int(config["log"]["max_bytes"])
LOG_BACKUP_COUNT = int(config["log"]["back_up_count"])
scripts/constants/__init__.py 0 → 100644
View file @ cf8fef09
class DatabaseNames:
ilens_configuration = "ilens_configuration"
local_host = "database"
class CollectionNames:
constants = "constants"
def ConstantKeys():
return None
\ No newline at end of file
scripts/db/mongo/__init__.py 0 → 100644
View file @ cf8fef09
from scripts.utils.mongo_util import MongoConnect
from scripts.config import DB
mongo_client = MongoConnect(uri=DB.MongoDb.uri)()
scripts/db/mongo/collections/constants.py 0 → 100644
View file @ cf8fef09
from typing import Any, Optional
from scripts.constants import DatabaseNames, CollectionNames
from scripts.utils.mongo_util import MongoCollectionBaseClass
class ConstantsSchema(MongoCollectionBaseClass):
"""
This is the Schema for the Mongo DB Collection.
All datastore and general responses will be following the schema.
"""
type: Optional[str]
data: Optional[Any]
map_json: Optional[Any]
content_type: Optional[Any]
content: Optional[Any]
class Constants(MongoCollectionBaseClass):
def __init__(self, mongo_client):
super().__init__(mongo_client, database=DatabaseNames.ilens_configuration,
collection=CollectionNames.constants)
def find_constant_by_dict(self, _type):
"""
The following function will give one record for a given set of
search parameters as keyword arguments
:param _type:
:return:
"""
record = self.find_one(query={"type": _type})
if not record:
return dict(record)
return dict(record)
def find_constant(self, _type, filter_dict=None):
"""
The following function will give one record for a given set of
search parameters as keyword arguments
:param _type:
:param filter_dict:
:return:
"""
query = {"type": _type}
record = self.find_one(query=query, filter_dict=filter_dict)
if not record:
return dict()
return record
def find_constant_by_query(self, query,filter_dict=None):
record = self.find_one(query=query, filter_dict=filter_dict)
if not record:
return dict()
return ConstantsSchema(**record)
def insert_one_constant(self, data):
"""
The following function will insert one tag in the
tags collections
:param self:
:param data:
:return:
"""
return self.insert_one(data)
def find_constant_by_content(self, content_type):
"""
The following function will give one record for a given set of
search parameters as keyword arguments
"""
query = {"content_type": content_type}
record = self.find_one(query=query)
if not record:
return dict()
return record
scripts/handler/update_handler.py 0 → 100644
View file @ cf8fef09
from scripts.db.mongo.collections.constants import Constants
from scripts.logging.logger import logger
from scripts.db.mongo import mongo_client
class UpdateHandler:
def __init__(self):
self.constants_mongo = Constants(mongo_client=mongo_client)
def constants_update(self):
try:
status = False
constants_records = self.constants_mongo.find_constant(_type="site_template")
data = constants_records.get("data")
for each in data:
if each.get("key_list"):
each.update({"access_list":each.get("key_list")[:-1]})
logger.debug(f'constant data: {data}')
# self.constants_mongo.update_one(query={"type":"site_template"},data={"data":data})
status = True
return status
except Exception as e:
logger.exception(str(e))
raise
scripts/logging/logger.py 0 → 100644
View file @ cf8fef09
import os
import logging
from scripts.config import LOG
from logging.handlers import RotatingFileHandler
__log_path__ = str(LOG.LOG_PATH)
__log_level__ = str(LOG.LOG_LEVEL)
__max_bytes__ = int(LOG.LOG_MAX_BYTES)
__handler_type__ = str(LOG.LOG_HANDLER)
_log_file_name__ = str(LOG.LOG_FILE_NAME)
__backup_count__ = int(LOG.LOG_BACKUP_COUNT)
complete_log_path = os.path.join(__log_path__, _log_file_name__)
if not os.path.isdir(complete_log_path):
os.makedirs(complete_log_path)
def get_logger(log_file_name=complete_log_path, log_level=__log_level__, time_format="%Y-%m-%d %H:%M:%S",
handler_type=__handler_type__, max_bytes=__max_bytes__, backup_count=__backup_count__):
"""
Creates a rotating log
"""
log_file = os.path.join(log_file_name + '.log')
__logger__ = logging.getLogger(log_file_name)
__logger__.setLevel(log_level.strip().upper())
debug_formatter = '%(asctime)s - %(levelname)-6s - %(name)s - ' \
'[%(threadName)5s:%(filename)5s:%(funcName)5s():''%(lineno)s] - %(message)s'
formatter_string = '%(asctime)s - %(levelname)-6s - %(name)s - %(levelname)3s - %(message)s'
if log_level.strip().upper() == log_level:
formatter_string = debug_formatter
formatter = logging.Formatter(formatter_string, time_format)
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
if __logger__.hasHandlers():
__logger__.handlers.clear()
if str(handler_type).lower() == "rotating_file_handler":
# Rotating File Handler
handler = RotatingFileHandler(log_file, maxBytes=max_bytes, backupCount=backup_count)
handler.setFormatter(formatter)
if __logger__.hasHandlers():
__logger__.handlers.clear()
__logger__.addHandler(handler)
else:
# File Handler
hdlr_service = logging.FileHandler(log_file)
hdlr_service.setFormatter(formatter)
if __logger__.hasHandlers():
__logger__.handlers.clear()
__logger__.addHandler(hdlr_service)
return __logger__
logger = get_logger()
scripts/utils/mongo_util.py 0 → 100644
View file @ cf8fef09
"""
Mongo Utility
Author: Irfanuddin Shafi Ahmed
Reference: Pymongo Documentation
"""
from typing import Dict, List, Optional
from pymongo import MongoClient
from pymongo.cursor import Cursor
class MongoConnect:
def __init__(self, uri):
try:
self.uri = uri
self.client = MongoClient(self.uri, connect=False)
except Exception:
raise
def __call__(self, *args, **kwargs):
return self.client
def __repr__(self):
return f"Mongo Client(uri:{self.uri}, server_info={self.client.server_info()})"
class MongoCollectionBaseClass:
def __init__(self, mongo_client, database, collection):
self.client = mongo_client
self.database = database
self.collection = collection
def __repr__(self):
return f"{self.__class__.__name__}(database={self.database}, collection={self.collection})"
def insert_one(self, data: Dict):
"""
The function is used to inserting a document to a collection in a Mongo Database.
:param data: Data to be inserted
:return: Insert ID
"""
try:
database_name = self.database
collection_name = self.collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.insert_one(data)
return response.inserted_id
except Exception:
raise
def insert_many(self, data: List):
"""
The function is used to inserting documents to a collection in a Mongo Database.
:param data: List of Data to be inserted
:return: Insert IDs
"""
try:
database_name = self.database
collection_name = self.collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.insert_many(data)
return response.inserted_ids
except Exception:
raise
def find(self,
query: Dict,
filter_dict: Optional[Dict] = None,
sort=None,
skip: Optional[int] = 0,
limit: Optional[int] = None) -> Cursor:
"""
The function is used to query documents from a given collection in a Mongo Database
:param query: Query Dictionary
:param filter_dict: Filter Dictionary
:param sort: List of tuple with key and direction. [(key, -1), ...]
:param skip: Skip Number
:param limit: Limit Number
:return: List of Documents
"""
if sort is None:
sort = list()
if filter_dict is None:
filter_dict = {"_id": 0}
database_name = self.database
collection_name = self.collection
try:
db = self.client[database_name]
collection = db[collection_name]
if len(sort) > 0:
cursor = collection.find(query, filter_dict, ).sort(sort).skip(skip)
else:
cursor = collection.find(query, filter_dict, ).skip(skip)
if limit:
cursor = cursor.limit(limit)
return cursor
except Exception:
raise
def find_one(self,
query: Dict,
filter_dict: Optional[Dict] = None):
try:
database_name = self.database
collection_name = self.collection
if filter_dict is None:
filter_dict = {"_id": 0}
db = self.client[database_name]
collection = db[collection_name]
response = collection.find_one(query, filter_dict)
return response
except Exception:
raise
def update_one(self,
query: Dict,
data: Dict,
upsert: bool = False):
"""
:param upsert:
:param query:
:param data:
:return:
"""
try:
database_name = self.database
collection_name = self.collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.update_one(query, {"$set": data}, upsert=upsert)
return response.modified_count
except Exception:
raise
def update_many(self,
query: Dict,
data: Dict,
upsert: bool = False):
"""
:param upsert:
:param query:
:param data:
:return:
"""
try:
database_name = self.database
collection_name = self.collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.update(query, {"$set": data}, upsert=upsert)
return response.get("nModified")
except Exception:
raise
def delete_many(self, query: Dict):
"""
:param query:
:return:
"""
try:
database_name = self.database
collection_name = self.collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.delete_many(query)
return response.deleted_count
except Exception:
raise
def delete_one(self, query: Dict):
"""
:param query:
:return:
"""
try:
database_name = self.database
collection_name = self.collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.delete_one(query)
return response.deleted_count
except Exception:
raise
def distinct(self, query_key: str, filter_json: Optional[Dict] = None):
"""
:param query_key:
:param filter_json:
:return:
"""
try:
database_name = self.database
collection_name = self.collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.distinct(query_key, filter_json)
return response
except Exception:
raise
def aggregate(self, pipelines: List, ):
try:
database_name = self.database
collection_name = self.collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.aggregate(pipelines)
return response
except Exception:
raise
def find_count(self, json_data, database_name, collection_name):
"""
:param json_data:
:param database_name: The database to which the collection/ documents belongs to.
:param collection_name: The collection to which the documents belongs to.
:return:
"""
try:
db = self.client[database_name]
mongo_response = db[collection_name].find(json_data).count()
return mongo_response
except Exception as e:
raise
class MongoAggregateBaseClass:
def __init__(self, mongo_client, database, ):
self.client = mongo_client
self.database = database
def aggregate(self, collection, pipelines: List, ):
try:
database_name = self.database
collection_name = collection
db = self.client[database_name]
collection = db[collection_name]
response = collection.aggregate(pipelines)
return response
except Exception:
raise
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment