Skip to content

D
dalmia-string-coefficient-db-script
Commit e6e21b50 authored by aakash.bedi's avatar aakash.bedi
Browse files
Options

updated mongo utility for coefficients

parent 14caa35f
Hide whitespace changes
Inline Side-by-side
Showing with 9 additions and 139 deletions
.idea/.gitignore deleted 100644 → 0
View file @ 14caa35f
# Default ignored files
/shelf/
/workspace.xml
# CodeStream ignored files
/../../dalmia-string-coefficient-db-script\.idea/codestream.xml
.idea/codestream.xml deleted 100644 → 0
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CodeStream">
<option name="webViewContext" value="{&quot;chatProviderAccess&quot;:&quot;strict&quot;,&quot;currentTeamId&quot;:&quot;&quot;,&quot;currentStreamId&quot;:&quot;&quot;,&quot;pullRequestCheckoutBranch&quot;:false,&quot;isRepositioning&quot;:false,&quot;onboardStep&quot;:0,&quot;panelStack&quot;:[&quot;landing-redirect&quot;],&quot;hasFocus&quot;:false,&quot;channelFilter&quot;:&quot;all&quot;,&quot;channelsMuteAll&quot;:false,&quot;codemarkFileFilter&quot;:&quot;all&quot;,&quot;codemarkTypeFilter&quot;:&quot;all&quot;,&quot;codemarkTagFilter&quot;:&quot;all&quot;,&quot;codemarkBranchFilter&quot;:&quot;all&quot;,&quot;codemarkAuthorFilter&quot;:&quot;all&quot;,&quot;codemarksFileViewStyle&quot;:&quot;inline&quot;,&quot;codemarksShowArchived&quot;:false,&quot;codemarksShowResolved&quot;:false,&quot;codemarksWrapComments&quot;:false,&quot;showFeedbackSmiley&quot;:true,&quot;route&quot;:{&quot;name&quot;:&quot;newUserEntry&quot;,&quot;params&quot;:{}},&quot;spatialViewShowPRComments&quot;:false,&quot;currentPullRequestNeedsRefresh&quot;:{&quot;needsRefresh&quot;:false,&quot;providerId&quot;:&quot;&quot;,&quot;pullRequestId&quot;:&quot;&quot;},&quot;__teamless__&quot;:{&quot;selectedRegion&quot;:&quot;us&quot;},&quot;sessionStart&quot;:1677648871648}" />
</component>
</project>
\ No newline at end of file
.idea/dalmia-string-coefficient-db-script.iml deleted 100644 → 0
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.9 (dalmia_forecasting)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
.idea/inspectionProfiles/Project_Default.xml deleted 100644 → 0
View file @ 14caa35f
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="68">
<item index="0" class="java.lang.String" itemvalue="pymongo" />
<item index="1" class="java.lang.String" itemvalue="kafka-python" />
<item index="2" class="java.lang.String" itemvalue="sklearn" />
<item index="3" class="java.lang.String" itemvalue="python-dotenv" />
<item index="4" class="java.lang.String" itemvalue="mlflow" />
<item index="5" class="java.lang.String" itemvalue="scipy" />
<item index="6" class="java.lang.String" itemvalue="azure-storage-blob" />
<item index="7" class="java.lang.String" itemvalue="SQLAlchemy" />
<item index="8" class="java.lang.String" itemvalue="matplotlib" />
<item index="9" class="java.lang.String" itemvalue="PyYAML" />
<item index="10" class="java.lang.String" itemvalue="requests" />
<item index="11" class="java.lang.String" itemvalue="numpy" />
<item index="12" class="java.lang.String" itemvalue="pandas" />
<item index="13" class="java.lang.String" itemvalue="meteostat" />
<item index="14" class="java.lang.String" itemvalue="psycopg2" />
<item index="15" class="java.lang.String" itemvalue="lightgbm" />
<item index="16" class="java.lang.String" itemvalue="SVR" />
<item index="17" class="java.lang.String" itemvalue="protobuf" />
<item index="18" class="java.lang.String" itemvalue="simplejson" />
<item index="19" class="java.lang.String" itemvalue="sqlparse" />
<item index="20" class="java.lang.String" itemvalue="httpx" />
<item index="21" class="java.lang.String" itemvalue="python-dateutil" />
<item index="22" class="java.lang.String" itemvalue="psycopg2-binary" />
<item index="23" class="java.lang.String" itemvalue="boltons" />
<item index="24" class="java.lang.String" itemvalue="Jinja2" />
<item index="25" class="java.lang.String" itemvalue="redis" />
<item index="26" class="java.lang.String" itemvalue="jwt-signature-validator" />
<item index="27" class="java.lang.String" itemvalue="XlsxWriter" />
<item index="28" class="java.lang.String" itemvalue="celery" />
<item index="29" class="java.lang.String" itemvalue="pycryptodomex" />
<item index="30" class="java.lang.String" itemvalue="fdfgen" />
<item index="31" class="java.lang.String" itemvalue="shortuuid" />
<item index="32" class="java.lang.String" itemvalue="PyPDF2" />
<item index="33" class="java.lang.String" itemvalue="uvicorn" />
<item index="34" class="java.lang.String" itemvalue="pdfjinja" />
<item index="35" class="java.lang.String" itemvalue="pendulum" />
<item index="36" class="java.lang.String" itemvalue="pydantic" />
<item index="37" class="java.lang.String" itemvalue="Werkzeug" />
<item index="38" class="java.lang.String" itemvalue="aiofiles" />
<item index="39" class="java.lang.String" itemvalue="weasyprint" />
<item index="40" class="java.lang.String" itemvalue="cryptography" />
<item index="41" class="java.lang.String" itemvalue="reportlab" />
<item index="42" class="java.lang.String" itemvalue="orjson" />
<item index="43" class="java.lang.String" itemvalue="python-multipart" />
<item index="44" class="java.lang.String" itemvalue="pdfrw" />
<item index="45" class="java.lang.String" itemvalue="sqlalchemy" />
<item index="46" class="java.lang.String" itemvalue="docx" />
<item index="47" class="java.lang.String" itemvalue="openpyxl-image-loader" />
<item index="48" class="java.lang.String" itemvalue="pyyaml" />
<item index="49" class="java.lang.String" itemvalue="fastapi" />
<item index="50" class="java.lang.String" itemvalue="docxtpl" />
<item index="51" class="java.lang.String" itemvalue="pdf2image" />
<item index="52" class="java.lang.String" itemvalue="paho-mqtt" />
<item index="53" class="java.lang.String" itemvalue="aiohttp" />
<item index="54" class="java.lang.String" itemvalue="pycryptodome" />
<item index="55" class="java.lang.String" itemvalue="pytz" />
<item index="56" class="java.lang.String" itemvalue="pyjwt" />
<item index="57" class="java.lang.String" itemvalue="formio-data" />
<item index="58" class="java.lang.String" itemvalue="openpyxl" />
<item index="59" class="java.lang.String" itemvalue="ujson" />
<item index="60" class="java.lang.String" itemvalue="catboost" />
<item index="61" class="java.lang.String" itemvalue="loguru" />
<item index="62" class="java.lang.String" itemvalue="xgboost" />
<item index="63" class="java.lang.String" itemvalue="pycaret" />
<item index="64" class="java.lang.String" itemvalue="optuna" />
<item index="65" class="java.lang.String" itemvalue="prophet" />
<item index="66" class="java.lang.String" itemvalue="scikit-learn" />
<item index="67" class="java.lang.String" itemvalue="kfp" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="E122" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
.idea/inspectionProfiles/profiles_settings.xml deleted 100644 → 0
View file @ 14caa35f
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
.idea/misc.xml deleted 100644 → 0
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (dalmia_forecasting)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
.idea/modules.xml deleted 100644 → 0
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/dalmia-string-coefficient-db-script.iml" filepath="$PROJECT_DIR$/.idea/dalmia-string-coefficient-db-script.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/vcs.xml deleted 100644 → 0
View file @ 14caa35f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
app.py
View file @ e6e21b50
......@@ -7,15 +7,15 @@ from scripts.core.coefficients import get_coefficient, push_to_mongo
warnings.filterwarnings('ignore')
def orchestrator(city):
def orchestrator(city, tags_property):
try:
df_coefficients = get_coefficient(city=city)
push_to_mongo(df=df_coefficients, city=city)
push_to_mongo(df=df_coefficients, city=city, tags_property=tags_property)
except Exception as e:
logger.exception(f'Exception - {e}')
if __name__ == "__main__":
orchestrator(city='ariyalur')
orchestrator(city='ariyalur', tags_property='mppt_coefficients')
scripts/core/coefficients.py
View file @ e6e21b50
......@@ -19,12 +19,12 @@ def get_coefficient(city):
logger.exception(f'Exception - {e}')
def push_to_mongo(df, city):
def push_to_mongo(df, city, tags_property):
try:
data_dict = df.to_dict(orient='index')
mongo_conn = MongoConnect(uri=Mongo.mongo_uri, database=MongoConstants.db,
collection=MongoConstants.collection)
response = mongo_conn.insert_one(data=data_dict, city=city)
response = mongo_conn.insert_one(data=data_dict, city=city, tags_property=tags_property)
logger.info(f'response - {response}')
except Exception as e:
logger.exception(f'Exception - {e}')
\ No newline at end of file
scripts/utils/mongo_utils.py
View file @ e6e21b50
......@@ -15,22 +15,23 @@ class MongoConnect:
logger.exception(f'Exception - {e}')
@staticmethod
def data_dict(data, city):
def data_dict(data, city, tags_property):
try:
req_dict = dict()
req_dict['project_id'] = Mongo.project_id
req_dict['id'] = Mongo.query_filter
req_dict['city'] = city
req_dict['tags_property'] = tags_property
req_dict['input_data'] = data
return req_dict
except Exception as e:
logger.exception(f'Exception - {e}')
def insert_one(self, data, city):
def insert_one(self, data, city, tags_property):
try:
db = self.client[self.database]
collection = db[self.collection]
req_dict = self.data_dict(data=data, city=city)
req_dict = self.data_dict(data=data, city=city, tags_property=tags_property)
response = collection.insert_one(req_dict)
return response.inserted_id
except Exception as e:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment