migrate to gitlab-pm

.idea/.gitignore

+# Default ignored files
+/shelf/
+/workspace.xml

.idea/.name

+task8aggregations
\ No newline at end of file

.idea/inspectionProfiles/profiles_settings.xml

+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
\ No newline at end of file

.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (task8redis)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file

.idea/modules.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/task8aggregations.iml" filepath="$PROJECT_DIR$/.idea/task8aggregations.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

.idea/task8aggregations.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file

.idea/vcs.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

app.py

+# using mongo aggregation operations
+import uvicorn
+from scripts.config.applications_config import uvicorn_port
+from scripts.logging.loggers import logger
+
+# starting the application
+if __name__ == "__main__":
+    try:
+        print("Mongo Aggregations task")
+        uvicorn.run("main:app", port=int(uvicorn_port))
+    except Exception as e:
+        logger.error("Interruption occurred: ", e)
+
+# pip install python-multipart

conf/applications.conf

+[path]
+base_path = scripts/
+sub_path = temp/
+log_path = log/
+
+[file]
+file_name_xlsx = data.xlsx
+
+[connection]
+mongodb = mongodb://localhost:27017
+
+[mongodb]
+projection = $project
+db_name = violations_logs
+collection_name = logs
+
+
+[uvicorn]
+uvicorn_port = 8000
+uvicorn_app = scripts.services.app_services_run:app
+
+[api_routes]
+route_index = /upload/
+route_submit = /submit_file/
+
+[log]
+formatter_time = asctime
+formatter_level = levelname

main.py

+# using mongo aggregation operations
+import uvicorn
+
+from fastapi import FastAPI
+from scripts.config.applications_config import uvicorn_port
+from scripts.logging.loggers import logger
+from scripts.services.app_services_run import business_violate
+
+app = FastAPI()
+app.include_router(business_violate)
+
+
+# starting the application
+if __name__ == "__main__":
+    try:
+        print("Mongo Aggregations task")
+        uvicorn.run(app, port=int(uvicorn_port))
+    except Exception as e:
+        logger.error("Interruption occurred: ", e)
+
+# pip install python-multipart

scripts/config/applications_config.py

+# reading conf file
+import configparser
+
+
+config = configparser.RawConfigParser()
+config.read("conf/applications.conf")
+
+
+# path
+base_path = config.get("path", 'base_path')
+sub_path = config.get("path", "sub_path")
+log_path = config.get("path", "log_path")
+
+# uvicorn
+uvicorn_port = config.get("uvicorn", "uvicorn_port")
+uvicorn_app = config.get("uvicorn", "uvicorn_app")
+
+# file name
+file_name_xlsx = config.get("file", "file_name_xlsx")
+full_path_xlsx = base_path + sub_path + file_name_xlsx
+
+# db connection
+client_connect = config.get("connection", "mongodb")
+
+# mongo
+projection = config.get("mongodb", "projection")
+db_name = config.get("mongodb", "db_name")
+collection_name = config.get("mongodb", "collection_name")
+
+# log
+formatter_time = config.get("log", "formatter_time")
+formatter_level = config.get("log", "formatter_level")

scripts/constants/api_routes_config.py

+# reading conf file
+import configparser
+
+
+config = configparser.RawConfigParser()
+config.read("conf/applications.conf")
+
+
+# api routes
+route_index = config.get("api_routes", 'route_index')
+route_submit = config.get("api_routes", 'route_submit')

scripts/core/database/mongodb_connect.py

+# connecting to mongodb server
+from pymongo import MongoClient
+
+from scripts.config.applications_config import client_connect
+
+# connect to server
+client = MongoClient(client_connect)

scripts/core/handlers/api_read.py

+from scripts.config.applications_config import full_path_xlsx
+from scripts.core.handlers.generate_excel_file import generate_excel
+from scripts.core.handlers.mongo_db_operations import MongoDbStart
+from scripts.core.handlers.uploaded_file_ import extract_uploaded_data
+from scripts.logging.loggers import logger
+
+
+def read_data_from_api(file_data):
+    try:
+        decoded_data = extract_uploaded_data(file_data)
+        db_class = MongoDbStart()
+        db_created = db_class.mongo_create()
+        db_class.mongo_insert(db_created, decoded_data)
+        print("\nBusiness Name with most Violations")
+        db_class.mongo_violation(db_created)
+        print("\nBusiness Name with No Violations")
+        db_class.mongo_no_violation(db_created)
+        print("\nGenerating Excel file")
+        cursor_data = db_class.mongo_view_results(db_created)
+        generate_excel(full_path_xlsx, cursor_data)
+    except Exception as e:
+        logger.error("Exception occurred while submitting: ", e)

scripts/core/handlers/data_insert.py

+# inserting data to mongo
+from scripts.logging.loggers import logger
+
+
+def insert_data_db(db, decoded):
+    try:
+        insert_res = db.insert_many(decoded)
+    except Exception as e:
+        logger.error("Exception occurred while inserting data: ", e)
+    else:
+        return insert_res

scripts/core/handlers/generate_excel_file.py

+# generating the Excel file
+import pandas as pd
+
+from scripts.logging.loggers import logger
+
+
+def generate_excel(path_xlsx, cursor_data):
+    try:
+        data_list = []
+        if cursor_data.alive > 0:
+            for business_names in cursor_data:
+                data_list.append(business_names)
+            data_frame = pd.DataFrame(data_list, columns=['result', 'business_name', 'date'])
+            print(data_frame)
+            data_frame.to_excel(path_xlsx, index=False)
+        else:
+            print("Error")
+    except Exception as e:
+        logger.error("Exception occurred while generating excel: ", e)

scripts/core/handlers/mongo_db_operations.py

+# class for mongodb operations
+from scripts.config.applications_config import db_name, collection_name
+from scripts.core.database.mongodb_connect import client
+from scripts.core.handlers.data_insert import insert_data_db
+from scripts.core.handlers.no_violations_data import db_select_no_violate
+from scripts.core.handlers.result_generate import results_data
+from scripts.core.handlers.violation_find import db_select_violate
+from scripts.logging.loggers import logger
+
+
+class MongoDbStart:
+    # create mongo table
+    @staticmethod
+    def mongo_create():
+        try:
+            client_con = client
+        except Exception as e:
+            logger.error("Exception occurred while connecting to mongo: ", e)
+        else:
+            db = client_con[db_name]
+            log_name = db[collection_name]
+            return log_name
+
+    # mongo data insert
+    @staticmethod
+    # getting input and insert into db
+    def mongo_insert(db, decoded):
+        try:
+            insert_check = insert_data_db(db, decoded)
+            if insert_check.acknowledged:
+                print("Data inserted....")
+            else:
+                print("Something went wrong data cannot be inserted....")
+        except Exception as e:
+            logger.error("Exception occurred while inserting to collection: ", e)
+
+    # mongo aggregate find businessman with most violations
+    @staticmethod
+    def mongo_violation(db):
+        try:
+            db_select_violate(db)
+        except Exception as e:
+            logger.error("Exception occurred while fetching violations: ", e)
+
+    # mongo aggregate find businessman with no violations
+    @staticmethod
+    def mongo_no_violation(db):
+        try:
+            db_select_no_violate(db)
+        except Exception as e:
+            logger.error("Exception occurred while fetching for no violations: ", e)
+
+    # mongo aggregate find
+    @staticmethod
+    def mongo_view_results(db):
+        try:
+            data = results_data(db)
+            return data
+        except Exception as e:
+            logger.error("Exception occurred while fetching all: ", e)

scripts/core/handlers/no_violations_data.py

+# business man with no violations
+from scripts.logging.loggers import logger
+from scripts.utils.mongo_operations import MongoDb
+
+
+def db_select_no_violate(db):
+    try:
+        mongo_obj = MongoDb()
+        cursor_data = mongo_obj.no_violations_db(db)
+        if cursor_data.alive > 0:
+            print("Data collected....")
+            for business_names in cursor_data:
+                print(business_names)
+        else:
+            print("Data fetching failed")
+    except Exception as e:
+        logger.error("Exception occurred while checking for violations: ", e)

scripts/core/handlers/result_generate.py

+# viewing the businessman and date based on results
+from scripts.logging.loggers import logger
+from scripts.utils.mongo_operations import MongoDb
+
+
+def results_data(db):
+    try:
+        mongo_obj = MongoDb()
+        cursor_data = mongo_obj.view_db_data(db)
+        if cursor_data.alive > 0:
+            print("Data collected....")
+            return cursor_data
+        else:
+            print("Data fetching failed")
+            return None
+    except Exception as e:
+        logger.error("Exception occurred while fetching results: ", e)

scripts/core/handlers/upload_file.html

+<!DOCTYPE html>
+<html lang="en" xml:lang="en">
+<head>
+    <title>Upload</title>
+</head>
+<body>
+    <!-- form upload -->
+    <form enctype = "multipart/form-data" action = "/submit_file/" method = "post">
+        <p>Upload File: <input type = "file" name = "file_data" /></p>
+        <p><input type = "submit" value = "Upload" /></p>
+    </form>
+</body>
+</html>

scripts/core/handlers/uploaded_file_.py

+# file upload to csv
+import json
+
+from scripts.logging.loggers import logger
+
+
+def extract_uploaded_data(file_data):
+    try:
+        decode = file_data.decode()
+        decoded_data_json = json.loads(decode)
+        return decoded_data_json
+    except Exception as e:
+        logger.error("Exception occurred while extracting uploaded data: ", e)

scripts/core/handlers/violation_find.py

+# businessman with violations
+from scripts.logging.loggers import logger
+from scripts.utils.mongo_operations import MongoDb
+
+
+def db_select_violate(db):
+    try:
+        mongo_obj = MongoDb()
+        cursor_data = mongo_obj.violations_db(db)
+        if cursor_data.alive > 0:
+            print("Data collected....")
+            print(cursor_data.next())
+        else:
+            print("Data fetching failed")
+    except Exception as e:
+        logger.error("Exception occurred while checking for violations: ", e)

scripts/logging/loggers.py

+import logging
+import os
+from logging.handlers import RotatingFileHandler
+
+from scripts.config import applications_config
+from scripts.config.applications_config import formatter_time, formatter_level
+
+
+def get_logger():
+    """
+     Creates a rotating log
+     """
+    __logger__ = logging.getLogger('')
+    __logger__.setLevel(logging.INFO)
+    log_formatter = f'%({formatter_time})s - %({formatter_level})-6s - %(message)s'
+    time_format = "%Y-%m-%d %H:%M:%S"
+    file_path = applications_config.log_path
+    formatter = logging.Formatter(log_formatter, time_format)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
+    log_file = os.path.join(f"{file_path}{applications_config.db_name}.log")
+    temp_handler = RotatingFileHandler(log_file, maxBytes=1)
+    temp_handler.setFormatter(formatter)
+    __logger__.addHandler(temp_handler)
+
+    return __logger__
+
+
+logger = get_logger()

scripts/services/app_services_run.py

+from fastapi import APIRouter, Request, UploadFile, File
+from fastapi.templating import Jinja2Templates
+from fastapi.responses import HTMLResponse
+
+from scripts.constants.api_routes_config import route_index, route_submit
+from scripts.core.handlers.api_read import read_data_from_api
+from scripts.logging.loggers import logger
+
+business_violate = APIRouter()
+templates = Jinja2Templates("scripts/core/handlers/")
+
+
+# html form access using api
+@business_violate.get(route_index, response_class=HTMLResponse)
+def template_render(request: Request):
+    try:
+        return templates.TemplateResponse("upload_file.html", {"request": request})
+    except Exception as e:
+        logger.error("Exception occurred while loading html: ", e)
+
+
+# handling the form submit using api and aggregations
+@business_violate.post(route_submit)
+async def submit_file_data(file_data: UploadFile = File(...)):
+    try:
+        data = await file_data.read()
+        read_data_from_api(data)
+    except Exception as e:
+        logger.error("Exception occurred while submitting: ", e)
+    else:
+        return {"msg": "file uploading successful..."}
+
+
+

scripts/temp/data.json

+[
+    {
+        "id": "100021-2015-ENFO",
+        "certificate_name": 9278806,
+        "business_name": "ATLIXCO DELI GROCERY INC.",
+        "date": "Feb 20 2015",
+        "result": "No Violation Issued",
+        "sector": "Cigaratte Retail Dealer - 127",
+        "address": {
+            "city": "RIDEWOOO",
+            "zip": 11385,
+            "street": "MENAHAN ST",
+            "number": 1712
+        }
+    },
+    {
+        "id": "100022-2015-ENFO",
+        "certificate_name": 5278806,
+        "business_name": "BIR FOR UIO.",
+        "date": "Feb 21 2015",
+        "result": "Violation Issued",
+        "sector": "Pillow Desinger",
+        "address": {
+            "city": "IKDK",
+            "zip": 1133,
+            "street": "KJFK ST",
+            "number": 1711
+        }
+    },
+    {
+        "id": "100023-2015-ENFO",
+        "certificate_name": 6278806,
+        "business_name": "HIO UEU JKI.",
+        "date": "Feb 22 2015",
+        "result": "No Violation Issued",
+        "sector": "Soap Manufacturer",
+        "address": {
+            "city": "IFKK",
+            "zip": 1139895,
+            "street": "IPOOEK",
+            "number": 17173
+        }
+    },
+    {
+        "id": "100024-2015-ENFO",
+        "certificate_name": 3278806,
+        "business_name": "KKD KJK KDM.",
+        "date": "Feb 20 2015",
+        "result": "No Violation Issued",
+        "sector": "Laptop dealer",
+        "address": {
+            "city": "JSJF",
+            "zip": 114985,
+            "street": "SBFJF ST",
+            "number": 1212
+        }
+    },
+    {
+        "id": "100025-2015-ENFO",
+        "certificate_name": 1278806,
+        "business_name": "KDK KJ KS.",
+        "date": "Feb 21 2015",
+        "result": "Violation Issued",
+        "sector": "Shoe Maker",
+        "address": {
+            "city": "IJFSIJ",
+            "zip": 113325,
+            "street": "KOKEO ST",
+            "number": 17122
+        }
+    },
+    {
+        "id": "100026-2015-ENFO",
+        "certificate_name": 8278806,
+        "business_name": "KJJK JNAS KJKQ.",
+        "date": "Feb 20 2015",
+        "result": "No Violation Issued",
+        "sector": "Phone Dealer",
+        "address": {
+            "city": "KKNJGG",
+            "zip": 11325,
+            "street": "OPEPM ST",
+            "number": 13912
+        }
+    },
+    {
+        "id": "100027-2015-ENFO",
+        "certificate_name": 79278806,
+        "business_name": "KJDK KADK KJW.",
+        "date": "Feb 23 2015",
+        "result": "Violation Issued",
+        "sector": "Potter",
+        "address": {
+            "city": "PLMLW",
+            "zip": 11385,
+            "street": "MENAHAN ST",
+            "number": 1712
+        }
+    },
+    {
+        "id": "100028-2015-ENFO",
+        "certificate_name": 9478806,
+        "business_name": "OII MNM KQW.",
+        "date": "Feb 23 2015",
+        "result": "Violation Issued",
+        "sector": "Switch manufacturer",
+        "address": {
+            "city": "IJKM",
+            "zip": 11335,
+            "street": "OKOWKL ST",
+            "number": 132
+        }
+    },
+    {
+        "id": "100029-2015-ENFO",
+        "certificate_name": 3978806,
+        "business_name": "WWKK KJKJ OQKW.",
+        "date": "Feb 21 2015",
+        "result": "No Violation Issued",
+        "sector": "Medicine Manufacturer",
+        "address": {
+            "city": "RIDEIIWWOOO",
+            "zip": 110,
+            "street": "KSFKK ST",
+            "number": 1312
+        }
+    },
+    {
+        "id": "100030-2015-ENFO",
+        "certificate_name": 3978878,
+        "business_name": "NSD KWW IJI.",
+        "date": "Feb 25 2015",
+        "result": "No Violation Issued",
+        "sector": "Pioker Oped",
+        "address": {
+            "city": "KMFK",
+            "zip": 118385,
+            "street": "KJFJ ST",
+            "number": 129
+        }
+    },
+    {
+        "id": "100021-2015-ENFO",
+        "certificate_name": 9478806,
+        "business_name": "IWI IJQ KDK.",
+        "date": "Feb 26 2015",
+        "result": "No Violation Issued",
+        "sector": "Kored Port",
+        "address": {
+            "city": "KJKIJIJ",
+            "zip": 9887,
+            "street": "WLKWEL ST",
+            "number": 1939
+        }
+    },
+    {
+        "id": "100021-2015-ENFO",
+        "certificate_name": 9278806,
+        "business_name": "ATLIXCO DELI GROCERY INC.",
+        "date": "Feb 21 2015",
+        "result": "Violation Issued",
+        "sector": "Cigaratte Retail Dealer - 127",
+        "address": {
+            "city": "RIDEWOOO",
+            "zip": 11385,
+            "street": "MENAHAN ST",
+            "number": 1712
+        }
+    },
+    {
+        "id": "100021-2015-ENFO",
+        "certificate_name": 9278806,
+        "business_name": "ATLIXCO DELI GROCERY INC.",
+        "date": "Feb 22 2015",
+        "result": "Violation Issued",
+        "sector": "Cigaratte Retail Dealer - 127",
+        "address": {
+            "city": "RIDEWOOO",
+            "zip": 11385,
+            "street": "MENAHAN ST",
+            "number": 1712
+        }
+    },
+    {
+        "id": "100021-2015-ENFO",
+        "certificate_name": 9278806,
+        "business_name": "ATLIXCO DELI GROCERY INC.",
+        "date": "Feb 23 2015",
+        "result": "Violation Issued",
+        "sector": "Cigaratte Retail Dealer - 127",
+        "address": {
+            "city": "RIDEWOOO",
+            "zip": 11385,
+            "street": "MENAHAN ST",
+            "number": 1712
+        }
+    },
+    {
+        "id": "100026-2015-ENFO",
+        "certificate_name": 8278806,
+        "business_name": "KJJK JNAS KJKQ.",
+        "date": "Feb 21 2015",
+        "result": "Violation Issued",
+        "sector": "Phone Dealer",
+        "address": {
+            "city": "KKNJGG",
+            "zip": 11325,
+            "street": "OPEPM ST",
+            "number": 13912
+        }
+    },
+    {
+        "id": "100026-2015-ENFO",
+        "certificate_name": 8278806,
+        "business_name": "KJJK JNAS KJKQ.",
+        "date": "Feb 23 2015",
+        "result": "Violation Issued",
+        "sector": "Phone Dealer",
+        "address": {
+            "city": "KKNJGG",
+            "zip": 11325,
+            "street": "OPEPM ST",
+            "number": 13912
+        }
+    }
+
+]
\ No newline at end of file

scripts/utils/mongo_operations.py

+# class for mongo operations
+from scripts.config.applications_config import projection
+
+
+class MongoDb:
+    # find business name with most violations
+    @staticmethod
+    def violations_db(db):
+        cursor_data = db.aggregate([
+            {
+                '$group': {
+                    '_id': '$business_name',
+                    'violations': {
+                        '$sum': {
+                            '$cond': [
+                                {
+                                    '$eq': [
+                                        '$result', 'Violation Issued'
+                                    ]
+                                }, 1, 0
+                            ]
+                        }
+                    }
+                }
+            }, {
+                '$sort': {
+                    'violations': -1
+                }
+            }, {
+                '$limit': 1
+            }, {
+                projection: {
+                    'business_name': '$_id',
+                    '_id': 0
+                }
+            }
+        ])
+        return cursor_data
+
+    # business names with no violations
+    @staticmethod
+    def no_violations_db(db):
+        cursor_data = db.aggregate([
+            {
+                '$group': {
+                    '_id': '$business_name',
+                    'violations': {
+                        '$sum': {
+                            '$cond': [
+                                {
+                                    '$eq': [
+                                        '$result', 'Violation Issued'
+                                    ]
+                                }, 1, 0
+                            ]
+                        }
+                    }
+                }
+            }, {
+                '$match': {
+                    'violations': {
+                        '$eq': 0
+                    }
+                }
+            }, {
+                projection: {
+                    'business_name': '$_id',
+                    '_id': 0
+                }
+            }
+        ])
+        return cursor_data
+
+    # business name and dates based on results
+    @staticmethod
+    def view_db_data(db):
+        cursor_data = db.aggregate([
+            {
+                projection: {
+                    '_id': 0,
+                    'result': 1,
+                    'business_name': 1,
+                    'date': 1
+                }
+            }
+        ])
+        return cursor_data