migrate to gitlab-pm

.env

+MONGO_URI = mongodb://localhost:27017
\ No newline at end of file

.idea/.gitignore

+# Default ignored files
+/shelf/
+/workspace.xml

.idea/.name

+task6.2_fast_api
\ No newline at end of file

.idea/inspectionProfiles/profiles_settings.xml

+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
\ No newline at end of file

.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (task6.2)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file

.idea/modules.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/task6.2_fast_api.iml" filepath="$PROJECT_DIR$/.idea/task6.2_fast_api.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

.idea/task6.2_fast_api.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.9 (task6.2)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file

.idea/vcs.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

app.py

+import uvicorn
+
+from scripts.config.applications_conf_read import uvicorn_port
+from scripts.logging.loggers import logger
+
+# starting the application
+if __name__ == "__main__":
+    try:
+        print("Fast API task")
+        uvicorn.run("main:app", port=int(uvicorn_port))
+    except Exception as e:
+        logger.error("Interruption occurred: ", e)
+

conf/applications.conf

+[path]
+base_path = scripts/
+sub_path = external/
+
+[file]
+csv_file = user_details_new.csv
+file_mode = wb
+
+[uvicorn]
+uvicorn_port = 8000
+uvicorn_app = scripts.services.app_service_run:app
+
+# database servers
+[connection]
+mongodb = mongodb://localhost:27017
+db_name = user_details_new
+collection = user
+
+[api_routes]
+route_index = /
+route_insert_csv = /insert-csv/
+route_insert_one = /insert-one/
+route_view_all = /view-all/
+route_update = /update/
+route_delete = /delete/
+
+[log]
+formatter_time = asctime
+formatter_level = levelname
\ No newline at end of file

main.py

+import uvicorn
+from fastapi import FastAPI
+
+from scripts.config.applications_conf_read import uvicorn_port
+from scripts.logging.loggers import logger
+from scripts.services.app_service_run import mongo_api
+
+app = FastAPI()
+app.include_router(mongo_api)
+
+# starting the application
+if __name__ == "__main__":
+    try:
+        print("Fast API task")
+        uvicorn.run(app, port=int(uvicorn_port))
+    except Exception as e:
+        logger.error("Interruption occurred: ", e)

scripts/config/applications_conf_read.py

+# reading from the applications.conf
+import configparser
+
+
+config = configparser.RawConfigParser()
+config.read("conf/applications.conf")
+
+
+# path
+base_path = config.get("path", 'base_path')
+sub_path = config.get("path", "sub_path")
+full_path = base_path + sub_path
+
+# file
+file_csv = config.get("file", "csv_file")
+csv_path = full_path + file_csv
+file_mode = config.get("file", "file_mode")
+
+# mongo server
+uri = config.get("connection", "mongodb")
+
+# log
+formatter_time = config.get("log", "formatter_time")
+formatter_level = config.get("log", "formatter_level")
+
+# uvicorn
+uvicorn_port = config.get("uvicorn", "uvicorn_port")
+uvicorn_app = config.get("uvicorn", "uvicorn_app")
+
+# mongodb conf
+db_name = config.get("connection", "db_name")
+collection = config.get("connection", "collection")

scripts/constants/api_path_config.py

+# reading conf file
+import configparser
+
+
+config = configparser.RawConfigParser()
+config.read("conf/applications.conf")
+
+# api paths
+route_index = config.get("api_routes", "route_index")
+route_insert_csv = config.get("api_routes", "route_insert_csv")
+route_insert_one = config.get("api_routes", "route_insert_one")
+route_view_all = config.get("api_routes", "route_view_all")
+route_update = config.get("api_routes", "route_update")
+route_delete = config.get("api_routes", "route_delete")
+
+
+

scripts/core/handlers/convert_to_json.py

+# convert csv to json
+import pandas as pd
+
+from scripts.logging.loggers import logger
+
+
+# convert csv to json
+def file_convert_json(csv_path):
+    try:
+        csv_file = pd.read_csv(csv_path)
+        dict_data = csv_file.to_dict(orient="records")
+    except Exception as e:
+        logger.error("Some exception occurred while reading csv: ", e)
+    else:
+        return dict_data

scripts/core/handlers/cursor_handle.py

+# cursor handler
+from scripts.logging.loggers import logger
+
+
+def cursor_to_dict(cursor):
+    try:
+        dict_data = {}
+        for i, dicts in enumerate(cursor):
+            dict_data.update({i: dicts})
+        return dict_data
+    except Exception as e:
+        logger.error("Some exception occurred while trying to run the cursor: ", e)

scripts/core/handlers/uploaded_file_.py

+# file upload to csv
+import os
+
+from scripts.config import applications_conf_read
+from scripts.config.applications_conf_read import file_mode
+from scripts.core.handlers.convert_to_json import file_convert_json
+from scripts.logging.loggers import logger
+
+
+def extract_uploaded_data(full_path_csv, file_data, db_obj, collection):
+    try:
+        filename, file_extension = os.path.splitext(file_data.filename)
+        if file_extension == ".csv":
+            with open(full_path_csv, file_mode) as file_object:
+                file_object.write(file_data.file.read())
+            csv_dict_data = file_convert_json(applications_conf_read.csv_path)
+            db_obj.insert_data_many(collection, csv_dict_data)
+        else:
+            logger.error("Invalid File format")
+    except Exception as e:
+        logger.error("Some exception occurred while extracting data: ", e)

scripts/databases/models/db_model_mongo.py

+from pydantic import BaseModel
+from pydantic.class_validators import Optional
+
+
+class User(BaseModel):
+    id: int
+    first_name: str
+    last_name: str
+    email: str
+    address: str
+
+
+class UpdateUser(BaseModel):
+    first_name: Optional[str] = None
+    last_name: Optional[str] = None
+    email: Optional[str] = None
+    address: Optional[str] = None

scripts/engine/connect_mongo.py

+# connect to mongodb
+
+from pymongo import MongoClient
+
+from scripts.config import applications_conf_read
+from scripts.logging.loggers import logger
+
+
+def connection_mongo_db():
+    try:
+        mongo_client = MongoClient(applications_conf_read.uri)
+        db = mongo_client[applications_conf_read.db_name]
+        collection = db[applications_conf_read.collection]
+        return collection
+    except Exception as e:
+        logger.exception("Some exception occurred while connecting to mongo: ", e)

scripts/external/user_details_new.csv

+id,first_name,last_name,email,Address
+1,Maurita,Helks,mhelks0@parallels.com,PO Box 3203
+2,Prisca,Baxstar,pbaxstar1@whitehouse.gov,20th Floor
+3,Camellia,Gallop,cgallop2@comsenz.com,5th Floor
+4,Kingston,Keightley,kkeightley3@rediff.com,Suite 35
+5,Suzie,Tredgold,stredgold4@myspace.com,PO Box 76402
+6,Ade,Coonihan,acoonihan5@stanford.edu,Apt 376
+7,Candida,Kobiela,ckobiela6@jiathis.com,10th Floor
+8,Karee,Marty,kmarty7@auda.org.au,Suite 75
+9,Granville,Weathey,gweathey8@bloglovin.com,2nd Floor
+10,Letty,Brecher,lbrecher9@posterous.com,3rd Floor
+11,Tam,Graal,tgraala@dmoz.org,Apt 525
+12,Dusty,Causbey,dcausbeyb@drupal.org,Apt 1003
+13,Kath,Chattey,kchatteyc@smugmug.com,Apt 1074
+14,Rutter,Wattins,rwattinsd@hhs.gov,Room 1920
+15,Simonette,Gledstane,sgledstanee@nytimes.com,20th Floor
+16,Cassey,Gerrelt,cgerreltf@scribd.com,12th Floor
+17,Paige,O'Henery,poheneryg@nature.com,Apt 1111
+18,Tabbie,Trever,ttreverh@ask.com,20th Floor
+19,Hoebart,Valder,hvalderi@tripod.com,5th Floor
+20,Margie,Stollenbecker,mstollenbeckerj@fema.gov,Apt 531
+21,Bobby,Beet,bbeetk@odnoklassniki.ru,20th Floor
+22,Obie,Le Blanc,oleblancl@bravesites.com,Room 832
+23,Alyosha,Ruffli,arufflim@bandcamp.com,Suite 14
+24,Syman,Granger,sgrangern@bbb.org,PO Box 19521
+25,Slade,Weller,swellero@livejournal.com,Room 1911
+26,Robina,Rate,rratep@ask.com,Apt 917
+27,Karon,Liveley,kliveleyq@tinypic.com,3rd Floor
+28,Patty,McGeouch,pmcgeouchr@paginegialle.it,11th Floor
+29,Osborn,Ferriday,oferridays@slashdot.org,Room 1474
+30,Ellsworth,Dundin,edundint@wikispaces.com,PO Box 17827
+31,Mitch,Stetson,mstetsonu@linkedin.com,Apt 603
+32,Hardy,Swansbury,hswansburyv@google.it,Apt 1802
+33,Nikolas,Telling,ntellingw@ocn.ne.jp,Room 148
+34,Gladi,Florey,gfloreyx@dropbox.com,Suite 55
+35,Adela,Overbury,aoverburyy@tinyurl.com,4th Floor
+36,Evie,McFfaden,emcffadenz@scribd.com,Suite 88
+37,Baldwin,Reaman,breaman10@booking.com,Suite 71
+38,Beckie,Leadley,bleadley11@blogs.com,Apt 1592
+39,Bendick,Kubes,bkubes12@latimes.com,Room 769
+40,Huntlee,Mazzeo,hmazzeo13@columbia.edu,Apt 1564
+41,Vally,Dzenisenka,vdzenisenka14@themeforest.net,PO Box 96195
+42,Lethia,Bolden,lbolden15@yandex.ru,Suite 80
+43,Odille,Chataignier,ochataignier16@slate.com,PO Box 73001
+44,Cy,Wescott,cwescott17@reuters.com,17th Floor
+45,Darnell,Jays,djays18@yellowbook.com,Room 1722
+46,Mikael,Wickwar,mwickwar19@bloglines.com,Room 1780
+47,Marguerite,McGahey,mmcgahey1a@1688.com,Room 243
+48,Jessie,Gaskill,jgaskill1b@angelfire.com,Suite 98
+49,Gardiner,Bestall,gbestall1c@cloudflare.com,17th Floor
+50,Jo-ann,Rickett,jrickett1d@live.com,Room 903
+51,Gavra,Moquin,gmoquin1e@quantcast.com,Room 1139
+52,Dylan,Slewcock,dslewcock1f@trellian.com,Apt 1256
+53,Danyette,Coolson,dcoolson1g@biglobe.ne.jp,Apt 452
+54,Roderich,Ashworth,rashworth1h@163.com,Apt 1416
+55,Hildagarde,Haucke,hhaucke1i@ox.ac.uk,Suite 7
+56,Darlleen,Bearsmore,dbearsmore1j@google.com,Suite 43
+57,Wilmette,Bedwell,wbedwell1k@weebly.com,Suite 65
+58,Neville,Swapp,nswapp1l@ocn.ne.jp,Apt 1054
+59,Norene,Kopacek,nkopacek1m@domainmarket.com,Suite 38
+60,Aubert,Streeting,astreeting1n@cocolog-nifty.com,Suite 82
+61,Katrine,Easson,keasson1o@bizjournals.com,Room 1306
+62,Celle,Elgie,celgie1p@sciencedirect.com,16th Floor
+63,Alvan,Curtin,acurtin1q@booking.com,16th Floor
+64,Cornela,Roder,croder1r@google.pl,2nd Floor
+65,Wanids,Dansie,wdansie1s@fc2.com,18th Floor
+66,Kip,Fillon,kfillon1t@imgur.com,Suite 82
+67,Holmes,Kidstoun,hkidstoun1u@usnews.com,Apt 17
+68,Freddy,Featherstone,ffeatherstone1v@parallels.com,13th Floor
+69,Gusella,Jodlkowski,gjodlkowski1w@chronoengine.com,Apt 515
+70,Marietta,Circuit,mcircuit1x@nydailynews.com,18th Floor
+71,Brander,Hackley,bhackley1y@drupal.org,Suite 91
+72,Nikola,Hughes,nhughes1z@elegantthemes.com,Suite 34
+73,Krisha,Aberkirder,kaberkirder20@blogs.com,Suite 70
+74,Shina,Meriott,smeriott21@rakuten.co.jp,18th Floor
+75,Debra,Sproat,dsproat22@wikia.com,Apt 1650
+76,Dunc,Wallworke,dwallworke23@devhub.com,PO Box 58145
+77,Rivalee,Gonnet,rgonnet24@oracle.com,Suite 97
+78,Regine,Keling,rkeling25@edublogs.org,Room 1519
+79,Ashla,Fair,afair26@ucla.edu,Apt 1213
+80,Arabela,Theakston,atheakston27@nsw.gov.au,Apt 1277
+81,Barney,Grimsdale,bgrimsdale28@reddit.com,PO Box 144
+82,Lucho,Braunlein,lbraunlein29@cisco.com,PO Box 26380
+83,Jania,Eldredge,jeldredge2a@bloglovin.com,Suite 3
+84,Hector,Remer,hremer2b@va.gov,Suite 51
+85,Hans,Cantrell,hcantrell2c@bbb.org,Apt 1834
+86,Jacquelynn,Kuhne,jkuhne2d@unc.edu,PO Box 70594
+87,Way,Moogan,wmoogan2e@chronoengine.com,PO Box 51805
+88,Janenna,Costa,jcosta2f@symantec.com,Room 1115
+89,Hermine,Bridywater,hbridywater2g@weibo.com,13th Floor
+90,Claudette,Highman,chighman2h@ftc.gov,Room 1359
+91,Reinhold,Bromehed,rbromehed2i@cargocollective.com,9th Floor
+92,Katleen,Dohmer,kdohmer2j@naver.com,PO Box 73746
+93,Kellen,Hull,khull2k@eepurl.com,Apt 1569
+94,Chrotoem,Lorincz,clorincz2l@behance.net,PO Box 8905
+95,Worth,Putton,wputton2m@qq.com,PO Box 91027
+96,Luce,Lyford,llyford2n@nps.gov,Room 739
+97,Van,Warre,vwarre2o@zimbio.com,Room 1771
+98,Kikelia,Benardette,kbenardette2p@vkontakte.ru,PO Box 74859
+99,Fabio,Grigoli,fgrigoli2q@plala.or.jp,Apt 1785
+100,Rhodia,Batch,rbatch2r@alibaba.com,Room 1597

scripts/external/user_details_new.log

+2023-02-03 18:07:29 - ERROR  - Invalid File format
+2023-02-03 18:07:53 - ERROR  - Invalid File format
+2023-02-03 18:08:38 - ERROR  - Invalid File format
+2023-02-03 18:11:43 - ERROR  - Invalid File format
+2023-02-03 18:19:05 - ERROR  - Invalid File format
+2023-02-03 18:20:24 - ERROR  - Invalid File format
+2023-02-03 18:20:57 - ERROR  - Invalid File format
+2023-02-03 18:25:40 - ERROR  - Invalid File format
+2023-02-03 18:26:29 - ERROR  - Invalid File format

scripts/logging/loggers.py

+import logging
+import os
+from logging.handlers import RotatingFileHandler
+
+from scripts.config import applications_conf_read
+from scripts.config.applications_conf_read import formatter_time, formatter_level
+
+
+def get_logger():
+    """
+     Creates a rotating log
+     """
+    __logger__ = logging.getLogger('')
+    __logger__.setLevel(logging.INFO)
+    log_formatter = f'%({formatter_time})s - %({formatter_level})-6s - %(message)s'
+    time_format = "%Y-%m-%d %H:%M:%S"
+    file_path = applications_conf_read.full_path
+    formatter = logging.Formatter(log_formatter, time_format)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
+    log_file = os.path.join(f"{file_path}{applications_conf_read.db_name}.log")
+    temp_handler = RotatingFileHandler(log_file, maxBytes=1)
+    temp_handler.setFormatter(formatter)
+    __logger__.addHandler(temp_handler)
+
+    return __logger__
+
+
+logger = get_logger()

scripts/services/app_service_run.py

+# api for convert, read, insert, update and delete
+from fastapi import UploadFile, File, APIRouter
+
+from scripts.config import applications_conf_read
+from scripts.constants import api_path_config
+from scripts.core.handlers.cursor_handle import cursor_to_dict
+from scripts.core.handlers.uploaded_file_ import extract_uploaded_data
+from scripts.engine.connect_mongo import connection_mongo_db
+from scripts.databases.models.db_model_mongo import User, UpdateUser
+from scripts.utils.mongo.mongo_utils import MongoDB
+from scripts.logging.loggers import logger
+
+mongo_api = APIRouter()
+
+collection = connection_mongo_db()
+db_obj = MongoDB()
+
+
+# index page
+@mongo_api.get(api_path_config.route_index)
+def start_app():
+    return "Starting the app"
+
+
+@mongo_api.post(api_path_config.route_insert_csv)
+def csv_file(file_data: UploadFile = File(...)):
+    try:
+        extract_uploaded_data(applications_conf_read.csv_path, file_data, db_obj, collection)
+    except Exception as e:
+        logger.error("Some exception occurred while running csv insertion: ", e)
+    else:
+        return {"msg": "Data added"}
+
+
+@mongo_api.post(api_path_config.route_insert_one)
+def insert_one(user_data: User):
+    try:
+        db_obj.insert_data_one(collection, user_data)
+    except Exception as e:
+        logger.error("Some exception occurred while running insertion: ", e)
+    else:
+        return {"msg": "Data inserted"}
+
+
+@mongo_api.post(api_path_config.route_view_all)
+def view_data_all():
+    try:
+        cursor = db_obj.find_data_all(collection)
+        data_fetch = cursor_to_dict(cursor)
+        return data_fetch
+    except Exception as e:
+        logger.error("Some exception occurred while running view all: ", e)
+
+
+@mongo_api.put(api_path_config.route_update)
+def update_data(d_id: int, user_data: UpdateUser):
+    try:
+        db_obj.update_data(collection, d_id, user_data)
+    except Exception as e:
+        logger.error("Some exception occurred while running update: ", e)
+    else:
+        return {"msg": "Data Updated"}
+
+
+@mongo_api.delete(api_path_config.route_delete + "{d_id}")
+def delete_data(d_id: int):
+    try:
+        db_obj.delete_data(collection, d_id)
+    except Exception as e:
+        logger.error("Some exception occurred while running delete: ", e)
+    else:
+        return {"msg": "Data deleted"}

scripts/utils/mongo/mongo_utils.py

+# mongo operations
+from scripts.logging.loggers import logger
+
+
+class MongoDB:
+
+    @staticmethod
+    def insert_data_many(collections, csv_dict_data):
+        try:
+            collections.insert_many(csv_dict_data)
+        except Exception as e:
+            logger.error("Some exception occurred while running insertion of csv: ", e)
+
+    @staticmethod
+    def insert_data_one(collections, user_data):
+        try:
+            collections.insert_one(dict(user_data))
+        except Exception as e:
+            logger.error("Some exception occurred while running insertion: ", e)
+
+    @staticmethod
+    def find_data_all(collections):
+        try:
+            cursor = collections.find({}, {"_id": 0})
+            return cursor
+        except Exception as e:
+            logger.error("Some exception occurred while fetching: ", e)
+
+    @staticmethod
+    def update_data(collections, d_id, user_data):
+        try:
+            collections.update_one({"id": d_id}, {"$set": dict(user_data)})
+        except Exception as e:
+            logger.error("Some exception occurred while updating: ", e)
+
+    @staticmethod
+    def delete_data(collections, d_id):
+        try:
+            collections.delete_one({"id": d_id})
+        except Exception as e:
+            logger.error("Some exception occurred while deleting: ", e)
+