Skip to content

D
dalmia-string-coefficient-db-script
Commit 14caa35f authored by aakash.bedi's avatar aakash.bedi
Browse files
Options

added mongo utility for coefficients

parent f6d98c15
Show whitespace changes
Inline Side-by-side
Showing with 299 additions and 0 deletions
.idea/.gitignore 0 → 100644
View file @ 14caa35f
# Default ignored files
/shelf/
/workspace.xml
# CodeStream ignored files
/../../dalmia-string-coefficient-db-script\.idea/codestream.xml
.idea/codestream.xml 0 → 100644
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CodeStream">
<option name="webViewContext" value="{&quot;chatProviderAccess&quot;:&quot;strict&quot;,&quot;currentTeamId&quot;:&quot;&quot;,&quot;currentStreamId&quot;:&quot;&quot;,&quot;pullRequestCheckoutBranch&quot;:false,&quot;isRepositioning&quot;:false,&quot;onboardStep&quot;:0,&quot;panelStack&quot;:[&quot;landing-redirect&quot;],&quot;hasFocus&quot;:false,&quot;channelFilter&quot;:&quot;all&quot;,&quot;channelsMuteAll&quot;:false,&quot;codemarkFileFilter&quot;:&quot;all&quot;,&quot;codemarkTypeFilter&quot;:&quot;all&quot;,&quot;codemarkTagFilter&quot;:&quot;all&quot;,&quot;codemarkBranchFilter&quot;:&quot;all&quot;,&quot;codemarkAuthorFilter&quot;:&quot;all&quot;,&quot;codemarksFileViewStyle&quot;:&quot;inline&quot;,&quot;codemarksShowArchived&quot;:false,&quot;codemarksShowResolved&quot;:false,&quot;codemarksWrapComments&quot;:false,&quot;showFeedbackSmiley&quot;:true,&quot;route&quot;:{&quot;name&quot;:&quot;newUserEntry&quot;,&quot;params&quot;:{}},&quot;spatialViewShowPRComments&quot;:false,&quot;currentPullRequestNeedsRefresh&quot;:{&quot;needsRefresh&quot;:false,&quot;providerId&quot;:&quot;&quot;,&quot;pullRequestId&quot;:&quot;&quot;},&quot;__teamless__&quot;:{&quot;selectedRegion&quot;:&quot;us&quot;},&quot;sessionStart&quot;:1677648871648}" />
</component>
</project>
\ No newline at end of file
.idea/dalmia-string-coefficient-db-script.iml 0 → 100644
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.9 (dalmia_forecasting)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
.idea/inspectionProfiles/Project_Default.xml 0 → 100644
View file @ 14caa35f
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="68">
<item index="0" class="java.lang.String" itemvalue="pymongo" />
<item index="1" class="java.lang.String" itemvalue="kafka-python" />
<item index="2" class="java.lang.String" itemvalue="sklearn" />
<item index="3" class="java.lang.String" itemvalue="python-dotenv" />
<item index="4" class="java.lang.String" itemvalue="mlflow" />
<item index="5" class="java.lang.String" itemvalue="scipy" />
<item index="6" class="java.lang.String" itemvalue="azure-storage-blob" />
<item index="7" class="java.lang.String" itemvalue="SQLAlchemy" />
<item index="8" class="java.lang.String" itemvalue="matplotlib" />
<item index="9" class="java.lang.String" itemvalue="PyYAML" />
<item index="10" class="java.lang.String" itemvalue="requests" />
<item index="11" class="java.lang.String" itemvalue="numpy" />
<item index="12" class="java.lang.String" itemvalue="pandas" />
<item index="13" class="java.lang.String" itemvalue="meteostat" />
<item index="14" class="java.lang.String" itemvalue="psycopg2" />
<item index="15" class="java.lang.String" itemvalue="lightgbm" />
<item index="16" class="java.lang.String" itemvalue="SVR" />
<item index="17" class="java.lang.String" itemvalue="protobuf" />
<item index="18" class="java.lang.String" itemvalue="simplejson" />
<item index="19" class="java.lang.String" itemvalue="sqlparse" />
<item index="20" class="java.lang.String" itemvalue="httpx" />
<item index="21" class="java.lang.String" itemvalue="python-dateutil" />
<item index="22" class="java.lang.String" itemvalue="psycopg2-binary" />
<item index="23" class="java.lang.String" itemvalue="boltons" />
<item index="24" class="java.lang.String" itemvalue="Jinja2" />
<item index="25" class="java.lang.String" itemvalue="redis" />
<item index="26" class="java.lang.String" itemvalue="jwt-signature-validator" />
<item index="27" class="java.lang.String" itemvalue="XlsxWriter" />
<item index="28" class="java.lang.String" itemvalue="celery" />
<item index="29" class="java.lang.String" itemvalue="pycryptodomex" />
<item index="30" class="java.lang.String" itemvalue="fdfgen" />
<item index="31" class="java.lang.String" itemvalue="shortuuid" />
<item index="32" class="java.lang.String" itemvalue="PyPDF2" />
<item index="33" class="java.lang.String" itemvalue="uvicorn" />
<item index="34" class="java.lang.String" itemvalue="pdfjinja" />
<item index="35" class="java.lang.String" itemvalue="pendulum" />
<item index="36" class="java.lang.String" itemvalue="pydantic" />
<item index="37" class="java.lang.String" itemvalue="Werkzeug" />
<item index="38" class="java.lang.String" itemvalue="aiofiles" />
<item index="39" class="java.lang.String" itemvalue="weasyprint" />
<item index="40" class="java.lang.String" itemvalue="cryptography" />
<item index="41" class="java.lang.String" itemvalue="reportlab" />
<item index="42" class="java.lang.String" itemvalue="orjson" />
<item index="43" class="java.lang.String" itemvalue="python-multipart" />
<item index="44" class="java.lang.String" itemvalue="pdfrw" />
<item index="45" class="java.lang.String" itemvalue="sqlalchemy" />
<item index="46" class="java.lang.String" itemvalue="docx" />
<item index="47" class="java.lang.String" itemvalue="openpyxl-image-loader" />
<item index="48" class="java.lang.String" itemvalue="pyyaml" />
<item index="49" class="java.lang.String" itemvalue="fastapi" />
<item index="50" class="java.lang.String" itemvalue="docxtpl" />
<item index="51" class="java.lang.String" itemvalue="pdf2image" />
<item index="52" class="java.lang.String" itemvalue="paho-mqtt" />
<item index="53" class="java.lang.String" itemvalue="aiohttp" />
<item index="54" class="java.lang.String" itemvalue="pycryptodome" />
<item index="55" class="java.lang.String" itemvalue="pytz" />
<item index="56" class="java.lang.String" itemvalue="pyjwt" />
<item index="57" class="java.lang.String" itemvalue="formio-data" />
<item index="58" class="java.lang.String" itemvalue="openpyxl" />
<item index="59" class="java.lang.String" itemvalue="ujson" />
<item index="60" class="java.lang.String" itemvalue="catboost" />
<item index="61" class="java.lang.String" itemvalue="loguru" />
<item index="62" class="java.lang.String" itemvalue="xgboost" />
<item index="63" class="java.lang.String" itemvalue="pycaret" />
<item index="64" class="java.lang.String" itemvalue="optuna" />
<item index="65" class="java.lang.String" itemvalue="prophet" />
<item index="66" class="java.lang.String" itemvalue="scikit-learn" />
<item index="67" class="java.lang.String" itemvalue="kfp" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="E122" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
View file @ 14caa35f
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
.idea/misc.xml 0 → 100644
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (dalmia_forecasting)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
.idea/modules.xml 0 → 100644
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/dalmia-string-coefficient-db-script.iml" filepath="$PROJECT_DIR$/.idea/dalmia-string-coefficient-db-script.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/vcs.xml 0 → 100644
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
app.py 0 → 100644
View file @ 14caa35f
if __name__=="__main__":
from dotenv import load_dotenv
load_dotenv(dotenv_path="config.env")
from loguru import logger
import warnings
from scripts.core.coefficients import get_coefficient, push_to_mongo
warnings.filterwarnings('ignore')
def orchestrator(city):
try:
df_coefficients = get_coefficient(city=city)
push_to_mongo(df=df_coefficients, city=city)
except Exception as e:
logger.exception(f'Exception - {e}')
if __name__ == "__main__":
orchestrator(city='ariyalur')
conf/application.conf 0 → 100644
View file @ 14caa35f
[MONGO]
mongo_uri=$MONGO_URI
project_id=$PROJECT_ID
query_filter=$QUERY_FILTER
config.env 0 → 100644
View file @ 14caa35f
MONGO_URI=mongodb://admin:iLensDevMongo783@192.168.0.220:2717/
PROJECT_ID=project_101
QUERY_FILTER=dalmia_string_level_tags
\ No newline at end of file
scripts/constants/app_configuration.py 0 → 100644
View file @ 14caa35f
if __name__ == "__main__":
from dotenv import load_dotenv
load_dotenv(dotenv_path='config.env')
import os
import os.path
import sys
from configparser import ConfigParser, BasicInterpolation
import yaml
from loguru import logger
# Configuring file constants
data_conf = "./conf/data.yml"
class EnvInterpolation(BasicInterpolation):
"""
Interpolation which expands environment variables in values.
"""
def before_get(self, parser, section, option, value, defaults):
value = super().before_get(parser, section, option, value, defaults)
if not os.path.expandvars(value).startswith("$"):
return os.path.expandvars(value)
else:
return
try:
config = ConfigParser(interpolation=EnvInterpolation())
config.read(f"conf/application.conf")
except Exception as e:
print(f"Error while loading the config: {e}")
print("Failed to Load Configuration. Exiting!!!")
sys.exit()
class Mongo:
mongo_uri = config["MONGO"]["mongo_uri"]
project_id = config["MONGO"]["project_id"]
query_filter = config["MONGO"]["query_filter"]
scripts/constants/app_constants.py 0 → 100644
View file @ 14caa35f
class MongoConstants:
# DB
db = "ilens_ai"
# collections
collection = "dalmiaStringTags"
scripts/core/coefficients.py 0 → 100644
View file @ 14caa35f
from loguru import logger
import warnings
import pandas as pd
from scripts.utils.mongo_utils import MongoConnect
from scripts.constants.app_configuration import Mongo
from scripts.constants.app_constants import MongoConstants
warnings.filterwarnings('ignore')
base_folder = 'data_folder'
def get_coefficient(city):
try:
df_coefficient = pd.read_excel(f'{base_folder}/coefficient.xlsx', sheet_name=city)
logger.info(f'shape of coefficient sheet for {city} - {df_coefficient.shape}')
df_coefficient.set_index('inv_id_mppt_id', inplace=True)
return df_coefficient
except Exception as e:
logger.exception(f'Exception - {e}')
def push_to_mongo(df, city):
try:
data_dict = df.to_dict(orient='index')
mongo_conn = MongoConnect(uri=Mongo.mongo_uri, database=MongoConstants.db,
collection=MongoConstants.collection)
response = mongo_conn.insert_one(data=data_dict, city=city)
logger.info(f'response - {response}')
except Exception as e:
logger.exception(f'Exception - {e}')
\ No newline at end of file
scripts/utils/mongo_utils.py 0 → 100644
View file @ 14caa35f
from typing import Dict
from loguru import logger
from pymongo import MongoClient
from scripts.constants.app_configuration import Mongo
class MongoConnect:
def __init__(self, uri, database, collection):
try:
self.uri = uri
self.client = MongoClient(self.uri, connect=False)
self.database = database
self.collection = collection
except Exception as e:
logger.exception(f'Exception - {e}')
@staticmethod
def data_dict(data, city):
try:
req_dict = dict()
req_dict['project_id'] = Mongo.project_id
req_dict['id'] = Mongo.query_filter
req_dict['city'] = city
req_dict['input_data'] = data
return req_dict
except Exception as e:
logger.exception(f'Exception - {e}')
def insert_one(self, data, city):
try:
db = self.client[self.database]
collection = db[self.collection]
req_dict = self.data_dict(data=data, city=city)
response = collection.insert_one(req_dict)
return response.inserted_id
except Exception as e:
logger.exception(f'Exception - {e}')
def find_one(self, query, filter_dict=None):
try:
if filter_dict is None:
filter_dict = {"_id": 0}
db = self.client[self.database]
collection = db[self.collection]
response = collection.find_one(query, filter_dict)
return response
except Exception as e:
logger.exception(f'Exception - {e}')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment