master table create

fb84d673 · aakash.bedi · 3e0db800 · fb84d673 · fb84d673 · fb84d673
Commit fb84d673 authored Jan 20, 2023 by aakash.bedi
37 changed files
--- a/.gitignore
+++ b/.gitignore
+__pycache__/
\ No newline at end of file
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
+stages:
+  - auto-tagging
+  - deploy
+  - update
+variables:
+  MYSQL_CONNECTION: "mysql -h $MYSQL_HOST -u $MYSQL_USER -p$MYSQL_PASS "
+before_script:
+  - val=`echo $($MYSQL_CONNECTION -e "SELECT COUNT(*) FROM $VERSION_DB.$DB_TABLE WHERE category='Server' AND type='Service' AND os='docker' AND module_name='$CI_PROJECT_NAME' ") | cut -d " " -f2`
+  - if [ $val == 0 ]; then $MYSQL_CONNECTION -e "INSERT INTO $VERSION_DB.$DB_TABLE values('Server','Service','$CI_PROJECT_NAME','docker', '2', '0', '0', '0')";fi
+  - QA=$($MYSQL_CONNECTION -N -e "SELECT qa FROM $VERSION_DB.$DB_TABLE where module_name = '$CI_PROJECT_NAME' AND type = 'Service' AND category = 'Server' AND os = 'docker'")
+  - DEV=$($MYSQL_CONNECTION -N -e "SELECT dev FROM $VERSION_DB.$DB_TABLE where module_name = '$CI_PROJECT_NAME' AND type = 'Service' AND category = 'Server' AND os = 'docker'")
+  - UAT=$(mysql -h $MYSQL_HOST -u $MYSQL_USER -p$MYSQL_PASS -N -e "SELECT uat FROM $VERSION_DB.$DB_TABLE where module_name = '$CI_PROJECT_NAME' AND type = 'Service' AND category = 'Server' AND os = 'docker'")
+  - PROD=$($MYSQL_CONNECTION -N -e "SELECT prod FROM $VERSION_DB.$DB_TABLE where module_name = '$CI_PROJECT_NAME' AND type = 'Service' AND category = 'Server' AND os = 'docker'")
+auto-tagging:
+  stage: auto-tagging
+  before_script:
+    - val=`echo $($MYSQL_CONNECTION -e "SELECT COUNT(*) FROM $VERSION_DB.$VERSION_RELEASE_TABLE WHERE  module_name='$CI_PROJECT_NAME' ") | cut -d " " -f2`
+    - if [ $val == 0 ]; then $MYSQL_CONNECTION -N -e "INSERT INTO $VERSION_DB.$VERSION_RELEASE_TABLE values('$CI_PROJECT_NAME', 'iLens', '5', '13', '0', '0')";fi
+    - ILENS=$($MYSQL_CONNECTION -N -e "SELECT ilens_version FROM "$VERSION_DB.$VERSION_RELEASE_TABLE" where module_name = '$CI_PROJECT_NAME'")
+    - RELEASE=$($MYSQL_CONNECTION -N -e "SELECT release_version FROM "$VERSION_DB.$VERSION_RELEASE_TABLE" where module_name = '$CI_PROJECT_NAME'")
+    - FEATURE=$($MYSQL_CONNECTION -N -e "SELECT feature_version FROM "$VERSION_DB.$VERSION_RELEASE_TABLE" where module_name = '$CI_PROJECT_NAME'")
+    - PATCH=$($MYSQL_CONNECTION -N -e "SELECT patch_version FROM "$VERSION_DB.$VERSION_RELEASE_TABLE" where module_name = '$CI_PROJECT_NAME'")
+  script:
+    - SOURCE_BRANCH=$(echo $CI_COMMIT_TITLE | cut -f 3 -d " " | cut -f 1 -d "/" | cut -f 2 -d "'")
+    - >
+        if [ "$SOURCE_BRANCH" = "QA" ]; then
+          ((RELEASE=RELEASE+1)) && FEATURE=0 && PATCH=0;
+          TAG_NAME=v$ILENS.$RELEASE.$FEATURE
+          IMAGE_URL=azrilensprod.azurecr.io/ilens/release/versions/v"$ILENS.$RELEASE:$CI_PROJECT_NAME-$TAG_NAME"
+          PROD=$RELEASE; QA=0; DEV=0;
+          $MYSQL_CONNECTION -e "UPDATE $VERSION_DB.$DB_TABLE SET prod='$PROD' ,qa='$QA', dev='$DEV' WHERE module_name='$CI_PROJECT_NAME' AND type='Service' AND category='Server' AND os='docker'"         
+        elif [ $SOURCE_BRANCH == "feature" ]; then
+          ((FEATURE=FEATURE+1)) && PATCH=0;
+          TAG_NAME=v$ILENS.$RELEASE.$FEATURE
+          IMAGE_URL=azrilensprod.azurecr.io/ilens/release/versions/v"$ILENS.$RELEASE:$CI_PROJECT_NAME-$TAG_NAME"
+        elif [ $SOURCE_BRANCH == "patch" ]; then
+          ((PATCH=PATCH+1));
+          TAG_NAME=v$ILENS.$RELEASE.$FEATURE.$PATCH
+          IMAGE_URL=azrilensprod.azurecr.io/ilens/release/versions/v"$ILENS.$RELEASE:$CI_PROJECT_NAME-$TAG_NAME"
+        else
+          exit 1
+        fi
+    - echo -e "\n\nImage:" $IMAGE_URL >> ReleaseNote.txt
+    - sed -i "1s|^|Version":" $TAG_NAME\n|" ReleaseNote.txt
+    - sed -i "1s|^|Module Name":" $CI_PROJECT_NAME\n|" ReleaseNote.txt
+    - docker build -t $IMAGE_URL .
+    - docker push $IMAGE_URL
+    - docker rmi --force $IMAGE_URL
+    - URL=$(echo $CI_PROJECT_URL | sed 's|https://||')
+    - git remote set-url origin https://$GIT_USRNAME:$GIT_USRPASSWD@$URL
+    - git config user.email "devopsilens@gmail.com"
+    - git config user.name "$GIT_USRNAME"    
+    - git tag -a $TAG_NAME -F ReleaseNote.txt
+    - git push origin $TAG_NAME
+    - $MYSQL_CONNECTION -e "UPDATE $VERSION_DB.$VERSION_RELEASE_TABLE SET release_version='$RELEASE', feature_version='$FEATURE', patch_version='$PATCH'  WHERE module_name = '$CI_PROJECT_NAME' "
+    - $MYSQL_CONNECTION -e "INSERT INTO $HISTORY_DB.$VERSION_RELEASE_TABLE values('$CI_JOB_ID', '$CI_PROJECT_NAME','iLens', '$ILENS.$RELEASE.$FEATURE', '$CI_COMMIT_SHA', '$GITLAB_USER_NAME', '$CI_COMMIT_REF_NAME')"
+  tags:
+    - shell
+  only:
+    - master
+#~~~~~| QA K8 AKS |~~~~~#
+qa-aks-deployment:
+  stage: deploy
+  script:
+    - REGISTRY_URL=azacrknowledgelens.azurecr.io/knowledgelens/products/ilens/qa
+    - export KUBECONFIG=/home/gitlab-runner/.kube/config-aks-qa
+    - NAMESPACE=ilens-core
+    - QA=`expr $QA + 1` && DEV=0
+    - docker build -t $REGISTRY_URL/$CI_PROJECT_NAME:v$PROD.$QA.$DEV .
+    - docker push $REGISTRY_URL/$CI_PROJECT_NAME:v$PROD.$QA.$DEV
+  only:
+  - QA
+  tags:
+  - shell
+tag-update-qa:
+  stage: update
+  script:
+     - QA=`expr $QA + 1` && DEV=0
+     - REGISTRY_URL=azacrknowledgelens.azurecr.io/knowledgelens/products/ilens/qa
+     - docker rmi --force $REGISTRY_URL/$CI_PROJECT_NAME:v$PROD.$QA.$DEV
+     - $MYSQL_CONNECTION -e "INSERT INTO $HISTORY_DB.$DB_TABLE values('$CI_JOB_ID','Server','Service', '$CI_PROJECT_NAME','docker', '$PROD.$QA.$DEV', '$CI_COMMIT_SHA', '$GITLAB_USER_NAME', '$CI_COMMIT_REF_NAME')"
+     - $MYSQL_CONNECTION -e "UPDATE $VERSION_DB.$DB_TABLE SET prod='$PROD' ,qa='$QA', dev='$DEV' WHERE module_name = '$CI_PROJECT_NAME' AND type = 'Service' AND category = 'Server' AND os = 'docker'"
+  dependencies:
+    - qa-aks-deployment
+  only:
+  - QA
+  tags: 
+  - shell
--- a/.idea/.gitignore
+++ b/.idea/.gitignore
+# Default ignored files
+/shelf/
+/workspace.xml
+# CodeStream ignored files
+/../../batch_analytics\.idea/codestream.xml
--- a/.idea/batch_analytics.iml
+++ b/.idea/batch_analytics.iml
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>
\ No newline at end of file
--- a/.idea/codestream.xml
+++ b/.idea/codestream.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CodeStream">
+    <option name="webViewContext" value="{&quot;chatProviderAccess&quot;:&quot;strict&quot;,&quot;currentTeamId&quot;:&quot;&quot;,&quot;currentStreamId&quot;:&quot;&quot;,&quot;pullRequestCheckoutBranch&quot;:false,&quot;isRepositioning&quot;:false,&quot;onboardStep&quot;:0,&quot;panelStack&quot;:[&quot;landing-redirect&quot;],&quot;hasFocus&quot;:false,&quot;channelFilter&quot;:&quot;all&quot;,&quot;channelsMuteAll&quot;:false,&quot;codemarkFileFilter&quot;:&quot;all&quot;,&quot;codemarkTypeFilter&quot;:&quot;all&quot;,&quot;codemarkTagFilter&quot;:&quot;all&quot;,&quot;codemarkBranchFilter&quot;:&quot;all&quot;,&quot;codemarkAuthorFilter&quot;:&quot;all&quot;,&quot;codemarksFileViewStyle&quot;:&quot;inline&quot;,&quot;codemarksShowArchived&quot;:false,&quot;codemarksShowResolved&quot;:false,&quot;codemarksWrapComments&quot;:false,&quot;showFeedbackSmiley&quot;:true,&quot;route&quot;:{&quot;name&quot;:&quot;newUserEntry&quot;,&quot;params&quot;:{}},&quot;spatialViewShowPRComments&quot;:false,&quot;currentPullRequestNeedsRefresh&quot;:{&quot;needsRefresh&quot;:false,&quot;providerId&quot;:&quot;&quot;,&quot;pullRequestId&quot;:&quot;&quot;},&quot;__teamless__&quot;:{&quot;selectedRegion&quot;:&quot;us&quot;},&quot;sessionStart&quot;:1674199810926}" />
+  </component>
+</project>
\ No newline at end of file
--- a/.idea/inspectionProfiles/Project_Default.xml
+++ b/.idea/inspectionProfiles/Project_Default.xml
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="68">
+            <item index="0" class="java.lang.String" itemvalue="pymongo" />
+            <item index="1" class="java.lang.String" itemvalue="kafka-python" />
+            <item index="2" class="java.lang.String" itemvalue="sklearn" />
+            <item index="3" class="java.lang.String" itemvalue="python-dotenv" />
+            <item index="4" class="java.lang.String" itemvalue="mlflow" />
+            <item index="5" class="java.lang.String" itemvalue="scipy" />
+            <item index="6" class="java.lang.String" itemvalue="azure-storage-blob" />
+            <item index="7" class="java.lang.String" itemvalue="SQLAlchemy" />
+            <item index="8" class="java.lang.String" itemvalue="matplotlib" />
+            <item index="9" class="java.lang.String" itemvalue="PyYAML" />
+            <item index="10" class="java.lang.String" itemvalue="requests" />
+            <item index="11" class="java.lang.String" itemvalue="numpy" />
+            <item index="12" class="java.lang.String" itemvalue="pandas" />
+            <item index="13" class="java.lang.String" itemvalue="meteostat" />
+            <item index="14" class="java.lang.String" itemvalue="psycopg2" />
+            <item index="15" class="java.lang.String" itemvalue="lightgbm" />
+            <item index="16" class="java.lang.String" itemvalue="SVR" />
+            <item index="17" class="java.lang.String" itemvalue="protobuf" />
+            <item index="18" class="java.lang.String" itemvalue="simplejson" />
+            <item index="19" class="java.lang.String" itemvalue="sqlparse" />
+            <item index="20" class="java.lang.String" itemvalue="httpx" />
+            <item index="21" class="java.lang.String" itemvalue="python-dateutil" />
+            <item index="22" class="java.lang.String" itemvalue="psycopg2-binary" />
+            <item index="23" class="java.lang.String" itemvalue="boltons" />
+            <item index="24" class="java.lang.String" itemvalue="Jinja2" />
+            <item index="25" class="java.lang.String" itemvalue="redis" />
+            <item index="26" class="java.lang.String" itemvalue="jwt-signature-validator" />
+            <item index="27" class="java.lang.String" itemvalue="XlsxWriter" />
+            <item index="28" class="java.lang.String" itemvalue="celery" />
+            <item index="29" class="java.lang.String" itemvalue="pycryptodomex" />
+            <item index="30" class="java.lang.String" itemvalue="fdfgen" />
+            <item index="31" class="java.lang.String" itemvalue="shortuuid" />
+            <item index="32" class="java.lang.String" itemvalue="PyPDF2" />
+            <item index="33" class="java.lang.String" itemvalue="uvicorn" />
+            <item index="34" class="java.lang.String" itemvalue="pdfjinja" />
+            <item index="35" class="java.lang.String" itemvalue="pendulum" />
+            <item index="36" class="java.lang.String" itemvalue="pydantic" />
+            <item index="37" class="java.lang.String" itemvalue="Werkzeug" />
+            <item index="38" class="java.lang.String" itemvalue="aiofiles" />
+            <item index="39" class="java.lang.String" itemvalue="weasyprint" />
+            <item index="40" class="java.lang.String" itemvalue="cryptography" />
+            <item index="41" class="java.lang.String" itemvalue="reportlab" />
+            <item index="42" class="java.lang.String" itemvalue="orjson" />
+            <item index="43" class="java.lang.String" itemvalue="python-multipart" />
+            <item index="44" class="java.lang.String" itemvalue="pdfrw" />
+            <item index="45" class="java.lang.String" itemvalue="sqlalchemy" />
+            <item index="46" class="java.lang.String" itemvalue="docx" />
+            <item index="47" class="java.lang.String" itemvalue="openpyxl-image-loader" />
+            <item index="48" class="java.lang.String" itemvalue="pyyaml" />
+            <item index="49" class="java.lang.String" itemvalue="fastapi" />
+            <item index="50" class="java.lang.String" itemvalue="docxtpl" />
+            <item index="51" class="java.lang.String" itemvalue="pdf2image" />
+            <item index="52" class="java.lang.String" itemvalue="paho-mqtt" />
+            <item index="53" class="java.lang.String" itemvalue="aiohttp" />
+            <item index="54" class="java.lang.String" itemvalue="pycryptodome" />
+            <item index="55" class="java.lang.String" itemvalue="pytz" />
+            <item index="56" class="java.lang.String" itemvalue="pyjwt" />
+            <item index="57" class="java.lang.String" itemvalue="formio-data" />
+            <item index="58" class="java.lang.String" itemvalue="openpyxl" />
+            <item index="59" class="java.lang.String" itemvalue="ujson" />
+            <item index="60" class="java.lang.String" itemvalue="catboost" />
+            <item index="61" class="java.lang.String" itemvalue="loguru" />
+            <item index="62" class="java.lang.String" itemvalue="xgboost" />
+            <item index="63" class="java.lang.String" itemvalue="pycaret" />
+            <item index="64" class="java.lang.String" itemvalue="optuna" />
+            <item index="65" class="java.lang.String" itemvalue="prophet" />
+            <item index="66" class="java.lang.String" itemvalue="scikit-learn" />
+            <item index="67" class="java.lang.String" itemvalue="kfp" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="E122" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>
\ No newline at end of file
--- a/.idea/inspectionProfiles/profiles_settings.xml
+++ b/.idea/inspectionProfiles/profiles_settings.xml
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
\ No newline at end of file
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (batch_analytics)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file
--- a/.idea/modules.xml
+++ b/.idea/modules.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/batch_analytics.iml" filepath="$PROJECT_DIR$/.idea/batch_analytics.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file
--- a/Dockerfile
+++ b/Dockerfile
+FROM python:3.9-buster
+COPY . /code
+WORKDIR /code
+RUN pip install -r requirements.txt
+CMD [ "python","app.py" ]
\ No newline at end of file
--- a/README.md
+++ b/README.md
-# batch_analytics
+# Batch Analytics
--- a/app.py
+++ b/app.py
--- a/conf/application.conf
+++ b/conf/application.conf
+[KAIROS_DB]
+uri=$KAIROS_URI
+[KAFKA]
+kafka_host=$KAFKA_HOST
+kafka_port=$KAFKA_PORT
+kafka_topic=$KAFKA_TOPIC
+[TIMEZONE]
+default_tz=$DEFAULT_TZ
+required_tz=$REQUIRED_TZ
+[HISTORICAL_DATA]
+start_relative=$START_RELATIVE
+end_relative=$END_RELATIVE
+[FORECAST_DATA]
+days_of_degradation_prediction=$DAYS_OF_DEGRADATION_PREDICTION
--- a/conf/data.yml
+++ b/conf/data.yml
+plants:
+  plant_kadappa: ./conf/plant_info/kadapa_plant.yml
+#  plant_ariyalur: ./conf/plant_info/ariyalur_plant.yml
+#  plant_lanka: ./conf/plant_info/lanka_plant.yml
+#  plant_dalmiapuram: ./conf/plant_info/dalmiapuram_plant.yml
+#  plant_bcw : ./conf/plant_info/bcw_plant.yml
--- a/config.env
+++ b/config.env
+KAIROS_URI=https://iLens:iLensDAL$456@dalmia.ilens.io/kairos/
+KAFKA_HOST=192.168.0.220
+KAFKA_PORT=9092
+KAFKA_TOPIC=ilens_dev
+DEFAULT_TZ="UTC"
+REQUIRED_TZ="Asia/Kolkata"
+START_RELATIVE=90
+END_RELATIVE=0
+DAYS_OF_DEGRADATION_PREDICTION=7
--- a/data/MPP data model.xlsx
+++ b/data/MPP data model.xlsx
--- a/data/P2E Golden batch 05-01-2023.xlsx
+++ b/data/P2E Golden batch 05-01-2023.xlsx
--- a/data/batch_kpi_master.xlsx
+++ b/data/batch_kpi_master.xlsx
--- a/data/batch_master.xlsx
+++ b/data/batch_master.xlsx
--- a/data/batch_master_proto.xlsx
+++ b/data/batch_master_proto.xlsx
--- a/data/master_join.xlsx
+++ b/data/master_join.xlsx
--- a/data/~$P2E Golden batch 05-01-2023.xlsx
+++ b/data/~$P2E Golden batch 05-01-2023.xlsx
--- a/data/~$batch_master_proto.xlsx
+++ b/data/~$batch_master_proto.xlsx
--- a/default_env
+++ b/default_env
+APP_NAME=ebpr_report_engine
+MONGO_URI=mongodb://192.168.0.220:2717/
+KAIROS_URI= http://192.168.0.220:8080
+MQTT_URL=192.168.0.220
+MQTT_PORT=1883
+SCHEDULER_PROXY=http://192.168.0.220/ilens-scheduler
+ILENS_META_SERVICE_URL=http://192.168.0.220/ilens_api
+EBPR_DATA_ENGINE_PROXY=http://192.168.0.220/formde
+EBPR_PROXY=http://192.168.0.220/ebpr
+WORKFLOW_MGMT_PROXY = http://jubilant.ilens.io/workflow-mt
+REDIS_HOST=192.168.0.220
+REDIS_PORT=6379
+REDIS_BROKER_URI_CELERY = redis://192.168.0.220:6379/10
+SECURITY_IP_CHECK=false
+SECURITY_USER_CHECK=true
+SECURITY_AGENT_CHECK=true
+LOG_LEVEL=INFO
+LOG_TRACEBACK=true
+REPORT_DIRECTORY=reports
+BASE_DIRECTORY=data
+EBPR_REPORT_PROXY=http://192.168.0.220/ebpr_reports
+ENABLE_CELERY_WORKER=true
+HUB_NAME=
+CONNECTION_STRING=
+API_VERSION=
+MOBILE_PUSH_NOTIFICATION=false
+KAFKA_HOST=192.168.0.220
+KAFKA_PORT=9092
+KAFKA_TOPIC=ilens_dev
+START_DATE=1627776020
+END_DATE=1628035220
\ No newline at end of file
--- a/requirements.txt
+++ b/requirements.txt
+pytz==2021.3
+loguru==0.5.3
+scipy==1.7.1
+numpy==1.21.2
+pandas==1.3.3
+simplejson==3.17.5
+requests==2.26.0
+pydantic==1.8.2
+python-dotenv==0.19.2
+PyYAML==6.0
+kafka-python==1.4.7
+SQLAlchemy==1.3.20
+sqlparse==0.4.2
+psycopg2==2.9.1
+protobuf==3.20.*
+matplotlib==3.6.1
--- a/scripts/__init__.py
+++ b/scripts/__init__.py
--- a/scripts/constants/__init__.py
+++ b/scripts/constants/__init__.py
--- a/scripts/constants/app_configuration.py
+++ b/scripts/constants/app_configuration.py
+from dotenv import load_dotenv
+import os
+import sys
+from configparser import ConfigParser, BasicInterpolation
+import yaml
+from loguru import logger
+# Configuration File Constants
+_application_conf = f"./conf/application.conf"
+_default_conf = f"./config.env"
+data_conf = f"./conf/data.yml"
+training_data = f"./data/training.csv"
+load_dotenv(dotenv_path=_default_conf)
+class EnvInterpolation(BasicInterpolation):
+    """
+    Interpolation which expands environment variables in values.
+    """
+    def before_get(self, parser, section, option, value, defaults):
+        value = super().before_get(parser, section, option, value, defaults)
+        if not os.path.expandvars(value).startswith("$"):
+            return os.path.expandvars(value)
+        else:
+            return
+try:
+    config = ConfigParser(interpolation=EnvInterpolation())
+    config.read(_application_conf)
+except Exception as e:
+    print(f"Error while loading the config: {e}")
+    print("Failed to Load Configuration. Exiting!!!")
+    sys.exit()
+class Logging:
+    level = config.get("LOGGING", "level", fallback="INFO")
+    level = level if level else "INFO"
+    tb_flag = config.getboolean("LOGGING", "traceback", fallback=True)
+    tb_flag = tb_flag if tb_flag is not None else True
+class Kafka:
+    kairos_db_uri = config['KAIROS_DB']['uri']
+    kafka_host = config["KAFKA"]["kafka_host"]
+    kafka_port = config["KAFKA"]["kafka_port"]
+    kafka_topic = config["KAFKA"]["kafka_topic"]
+class HistoricalData:
+    start_relative = config['HISTORICAL_DATA']['start_relative']
+    end_relative = config['HISTORICAL_DATA']['end_relative']
+class ForecastData:
+    days_of_degradation_prediction = config['FORECAST_DATA']['days_of_degradation_prediction']
+class TimeZone:
+    default_tz = config["TIMEZONE"]["default_tz"]
+    required_tz = config["TIMEZONE"]["required_tz"]
+# Read the configuration file
+yml_file_path = "conf/"
+try:
+    with open(os.path.join(yml_file_path, 'data.yml'), 'r') as engine_yml_file:
+        config_engine = yaml.full_load(engine_yml_file)
+except Exception as e:
+    logger.exception(f"Exception occurred - {e}", exc_info=True)
+plants = config_engine["plants"]
--- a/scripts/constants/app_constants.py
+++ b/scripts/constants/app_constants.py
+class PostgresConstants:
+    upper_threshold_table = "solar_maintenance_plan_upper_band"
+    lower_threshold_table = "solar_maintenance_plan_lower_band"
+    threshold_table = "solar_maintenance_plan"
+    depletion_table = "solar_degradation_master"
--- a/scripts/core/__init__.py
+++ b/scripts/core/__init__.py
--- a/scripts/core/jupyter_notebooks/.ipynb_checkpoints/mpp_stages-checkpoint.ipynb
+++ b/scripts/core/jupyter_notebooks/.ipynb_checkpoints/mpp_stages-checkpoint.ipynb
--- a/scripts/core/jupyter_notebooks/MPP data model.xlsx
+++ b/scripts/core/jupyter_notebooks/MPP data model.xlsx
--- a/scripts/core/jupyter_notebooks/Postgres dataframe.xlsx
+++ b/scripts/core/jupyter_notebooks/Postgres dataframe.xlsx
--- a/scripts/core/jupyter_notebooks/mpp_stages.ipynb
+++ b/scripts/core/jupyter_notebooks/mpp_stages.ipynb
--- a/scripts/data_model/__init__.py
+++ b/scripts/data_model/__init__.py
--- a/scripts/data_model/egress_data_model.py
+++ b/scripts/data_model/egress_data_model.py
+from typing import Any, Dict
+from pydantic import BaseModel
+class AzureDataModel(BaseModel):
+    blob_name: str
+    data: Any
+class KafkaDataModel(BaseModel):
+    data: Dict[str, Any]
+    site_id: str
+    gw_id: str = ''
+    pd_id: str = ''
+    timestamp: int
+    msg_id: int = 0
+    partition: str = ''
+    retain_flag: bool = False