init

.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# pycharm
+.idea/
+
+# logs
+logs/
\ No newline at end of file

Dockerfile

+FROM python:3.7-slim
+ADD . /opt
+WORKDIR /opt
+RUN pip install -r requirements.txt
+CMD python main.py

conf/configuration.yml

+#---------------Service Configurations----------------#
+SERVICE_CONFIG:
+  LOG_LEVEL: info
+  LOG_HANDLER_NAME: MergeWeatherData
+  LOGSTASH_HOST: 192.168.1.47
+  LOGSTASH_PORT: 5000
+
+#--------------System Configurations--------------------#
+# for testing use temp/mnt/ilens/ai-jobs/
+#--------------System Configurations--------------------#
+SYSTEM_CONFIG:
+  shared_mount_base_ai_job: temp/mnt/ilens/ai-jobs/
+  default_run_id: default_run_id
+
+#----------------------If read conf from mongo------------#
+FOR_EACH_MONGO_CONFIG:
+  READ_FROM_MONGO: true
+  MONGO_URI: mongodb://192.168.0.210:27017
+  MONGO_DB: iLensAiPipeline
+  MONGO_RUN_COLL: runMetadata
+  MONGO_SITE_COLL: siteMetadata

main.py

+import traceback
+from scripts.common.config_parser import config
+from scripts.common.constants import MergeWeatherDataConstants, ComponentExceptions
+from scripts.common.logsetup import logger
+from datetime import date, datetime
+import uuid
+import json
+import os
+import pandas as pd
+
+
+class MergeWeatherData:
+    def __init__(self, query):
+        self.query = query
+        self.historical_data = self.query['historic_data']
+        self.component_input_dir = self.query['component_input_dir']
+        self.component_output_dir = self.query['component_output_dir']
+        self.energy_data_path = os.path.join(self.component_input_dir, os.listdir(self.component_input_dir)[0])
+        # "Epoch Time"
+        self.epoch_time = MergeWeatherDataConstants.EPOCH_TIME
+
+    def preprocess_energy_data(self, energy_df):
+        logger.info("Cleaning the energy data ..............")
+        energy_df["Date"] = energy_df[self.epoch_time].apply(self.fix_epoch_time)
+        energy_df["Date"] = pd.to_datetime(energy_df["Date"])
+        energy_df.drop(columns=[self.epoch_time], inplace=True)
+        return energy_df
+
+    def merge_data_frames(self, hist_df, energy_df):
+        logger.info("Merging the historical and energy data ............")
+        data_train_df = pd.merge(energy_df, hist_df, left_on="Date", right_on="Date", how="inner")
+        return data_train_df
+
+    def fix_epoch_time(self, x):
+        x = str(x)
+        return datetime.fromtimestamp(int(x[0:10])).date()
+
+    def preprocess_merged_data(self, data_train_df):
+        try:
+            logger.info("Cleaning the Merged data ..............")
+            data_train_df['month'] = pd.DatetimeIndex(data_train_df['Date']).month
+            data_train_df.drop(columns=["Date"], inplace=True)
+            return data_train_df
+        except Exception as e:
+            logger.info(traceback.format_exc())
+            logger.info(e)
+
+    def split_data(self, data_train_df):
+        try:
+            logger.info("Creating X train and Y train ..............")
+            x_train_df = data_train_df.drop(columns=["value"])
+            y_train_df = data_train_df[["value"]]
+            x_train_df.to_csv(os.path.join(self.component_output_dir,"X_train.csv"))
+            y_train_df.to_csv(os.path.join(self.component_output_dir, "Y_train.csv"))
+            return True
+        except Exception as e:
+            logger.info(traceback.format_exc())
+            logger.info(e)
+
+    def preprocess_historical_data(self, hist_df):
+        try:
+            logger.info("Cleaning the historical data ..............")
+            hist_df.reset_index(inplace=True)
+            hist_df["Date"] = pd.to_datetime(hist_df["Date time"])
+            hist_df.drop(
+                columns=["index", "Wind Chill", "Wind Gust", "Name", "Date time", "Snow Depth", "Snow", "Conditions",
+                         "Heat Index"], inplace=True)
+            return hist_df
+        except Exception as e:
+            logger.info(traceback.format_exc())
+            logger.info(e)
+
+    def run(self):
+        try:
+            logger.info("Reading historical data from {}".format(self.historical_data))
+            hist_df = pd.read_csv(self.historical_data)
+            hist_df = self.preprocess_historical_data(hist_df)
+            print(hist_df.head())
+            logger.info("Reading energy data from {}".format(self.energy_data_path))
+            energy_df = pd.read_csv(self.energy_data_path)[["timestamp", "value"]]
+            energy_df = self.preprocess_energy_data(energy_df)
+            print(energy_df.head())
+            data_train_df = self.merge_data_frames(hist_df, energy_df)
+            data_train_df = self.preprocess_merged_data(data_train_df)
+            print(data_train_df.head())
+            if self.split_data(data_train_df):
+                return True
+            else:
+                return False
+        except Exception as e:
+            logger.info(traceback.format_exc())
+            logger.info(e)
+            return False
+
+
+if __name__ == '__main__':
+    try:
+        obj = MergeWeatherData(config)
+        if obj.run():
+            logger.info("Component ran successfully")
+        else:
+            logger.info("Component Failed........")
+    except Exception as e:
+        logger.info("Merge Weather Data Component Failed")
+        logger.info(traceback.format_exc())

requirements.txt

+requests
+pyyaml
+python-logstash-async
+pymongo
+sklearn
+pandas

scripts/common/config_parser.py

+#!/usr/bin/env python
+import os
+import sys
+import yaml
+import json
+from pymongo import MongoClient, DESCENDING
+from scripts.common.constants import MergeWeatherDataConstants
+
+config_path = os.path.join(os.getcwd(), "conf", "configuration.yml")
+if os.path.exists(config_path):
+    sys.stderr.write("Reading config from --> {}".format(config_path))
+    sys.stderr.write("\n")
+
+    with open(config_path, 'r') as stream:
+        _config = yaml.safe_load(stream)
+else:
+    sys.stderr.write("Configuration not found...")
+    sys.stderr.write("Exiting....")
+    sys.exit(1)
+# uncomment for Testing
+os.environ['pipeline_id'] = "pipe1"
+# os.environ['artifact_base_path'] = "/data/ai-models"
+# os.environ['ilens_tag_hierarchy'] = 'site1'
+# os.environ['container_name'] = 'test'
+# ------------------------ Configurations -----------------------------------------------------------------------------
+pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
+shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job",
+                                          _config.get("SYSTEM_CONFIG", {}).get('shared_mount_base_ai_job'))
+# read from $shared_mount_base_ai_job/$pipeline_id/run.config
+# run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run_config.json"
+run_id_path = os.path.join(shared_mount_base_ai_job, pipeline_id, "run_config.json")
+
+try:
+    sys.stderr.write("Checking for run id parameters at path " + run_id_path + "\n")
+    with open(run_id_path) as f:
+        run_id_param = json.load(f)
+    run_id = run_id_param['run_id']
+except Exception as e:
+    sys.stderr.write("No run_config.json file is there so keeping run id as default_run_id  " + "\n")
+    run_id = _config.get("SYSTEM_CONFIG", {}).get('default_run_id')
+
+historic_data_path = os.path.join(shared_mount_base_ai_job, 'historical_weather_data')
+
+component_input_dir = os.path.join(shared_mount_base_ai_job, pipeline_id, run_id,
+                                   MergeWeatherDataConstants.COMPONENT_NAME, "input")
+
+component_output_dir = os.path.join(shared_mount_base_ai_job, pipeline_id, run_id,
+                                    MergeWeatherDataConstants.NEXT_COMPONENT, "input")
+component_parameter_path = os.path.join(shared_mount_base_ai_job, pipeline_id, run_id,
+                                        MergeWeatherDataConstants.COMPONENT_NAME, "param.json")
+try:
+    sys.stderr.write("Checking for component parameters at path " + component_parameter_path + "\n")
+    with open(component_parameter_path) as f:
+        component_parameter = json.load(f)
+    historic_file_name = component_parameter['historic_file_name']
+except Exception as e:
+    sys.stderr.write("No param.json file so trying to take from env" + "\n")
+    try:
+        historic_file_name = os.environ.get('historic_file_name')
+    except Exception as e:
+        raise KeyError(e)
+
+BASE_LOG_PATH = os.path.join(os.getcwd(), "logs")
+if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
+    os.mkdir(os.path.join(os.getcwd(), 'logs'))
+
+LOG_LEVEL = os.environ.get("LOG_LEVEL", _config.get('SERVICE_CONFIG', {}).get("LOG_LEVEL", "INFO")).upper()
+LOG_HANDLER_NAME = _config.get('SERVICE_CONFIG', {}).get("LOG_HANDLER_NAME", "MergeWeatherData")
+ENABLE_LOGSTASH_LOG = os.environ.get("ENABLE_LOGSTASH_LOG", 'False').lower()
+LOGSTASH_HOST = _config.get('SERVICE_CONFIG', {}).get('LOGSTASH_HOST')
+LOGSTASH_PORT = str(_config.get('SERVICE_CONFIG', {}).get('LOGSTASH_PORT'))
+
+config = {
+    'pipeline_id': pipeline_id,
+    'run_id': run_id,
+    'shared_mount_base_ai_job': shared_mount_base_ai_job,
+    'component_output_dir': component_output_dir,
+    'component_input_dir': component_input_dir,
+    'historic_data_path': historic_data_path,
+    'historic_file_name': historic_file_name,
+    'historic_data': os.path.join(historic_data_path, historic_file_name)
+}
+
+if not os.path.exists(config['shared_mount_base_ai_job']):
+    sys.stderr.write("Shared path does not exist!" + "\n")
+    sys.stderr.write("Creating path --> {}".format(config['shared_mount_base_ai_job'] + "\n"))
+    os.makedirs(config['shared_mount_base_ai_job'])
+
+if not os.path.exists(config['component_output_dir']):
+    sys.stderr.write("component_output_dir does not exist!" + "\n")
+    sys.stderr.write("Creating path --> {}".format(config['component_output_dir'] + "\n"))
+    os.makedirs(config['component_output_dir'])

scripts/common/constants.py

+#!/usr/bin/env python
+
+
+class MergeWeatherDataConstants:
+    NEXT_COMPONENT = "ModelFitting"
+    COMPONENT_NAME = "MergeWeatherData"
+    HTTP = "http://"
+    LOG_VAR_MESSAGE = "\n" + "#" * 25 + "\n" + "{}" + "\n" + "#" * 25 + "\n" + "{}\n"
+    EPOCH_TIME = "timestamp"
+
+
+class ComponentExceptions:
+    INVALID_AZURE_FILE_PATH_EXCEPTION = "AZURE PATH ERROR"
+    INVALID_LOCAL_FILE_PATH_EXCEPTION = "No File in the local path"
+    INVALID_ARTIFACT_BASE_PATH_EXCEPTION = "Artifact base path value is missing"
+    INVALID_AZURE_FILE_NAME_EXCEPTION = "Artifact name is missing"
+    INVALID_CONTAINER_NAME = "Container name is missing"
+    FAILED_UPLOAD_TO_BLOB = "Failed while uploading to blob"

scripts/common/logsetup.py

+import os
+import logging
+from logging.handlers import RotatingFileHandler
+from logstash_async.handler import AsynchronousLogstashHandler
+from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH
+from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH, LOGSTASH_HOST, LOGSTASH_PORT, ENABLE_LOGSTASH_LOG
+
+DEFAULT_FORMAT = '%(asctime)s %(levelname)5s %(name)s %(message)s'
+DEBUG_FORMAT = '%(asctime)s %(levelname)5s %(name)s [%(threadName)5s:%(filename)5s:%(funcName)5s():%(lineno)s] %(' \
+               'message)s '
+EXTRA = {}
+FORMATTER = DEFAULT_FORMAT
+if LOG_LEVEL.strip() == "DEBUG":
+    FORMATTER = DEBUG_FORMAT
+logging.trace = logging.DEBUG - 5
+logging.addLevelName(logging.DEBUG - 5, 'TRACE')
+
+
+class ILensLogger(logging.getLoggerClass()):
+    def __init__(self, name):
+        super().__init__(name)
+
+    def trace(self, msg, *args, **kwargs):
+        if self.isEnabledFor(logging.trace):
+            self._log(logging.trace, msg, args, **kwargs)
+
+
+def get_logger(log_handler_name):
+    """
+    Purpose : To create logger .
+    :param log_handler_name: Name of the log handler.
+    :return: logger object.
+    """
+    log_path = os.path.join(BASE_LOG_PATH, log_handler_name + ".log")
+    logging.setLoggerClass(ILensLogger)
+    _logger = logging.getLogger(log_handler_name)
+    _logger.setLevel(LOG_LEVEL.strip().upper())
+    log_handler = logging.StreamHandler()
+    log_handler.setLevel(LOG_LEVEL)
+    formatter = logging.Formatter(FORMATTER)
+    log_handler.setFormatter(formatter)
+    handler = RotatingFileHandler(log_path, maxBytes=10485760,
+                                  backupCount=5)
+    handler.setFormatter(formatter)
+    _logger.addHandler(log_handler)
+    _logger.addHandler(handler)
+    if ENABLE_LOGSTASH_LOG == 'true' and LOGSTASH_PORT is not None and LOGSTASH_HOST is not None and LOGSTASH_PORT.isdigit():
+        _logger.addHandler(AsynchronousLogstashHandler(LOGSTASH_HOST, int(LOGSTASH_PORT), database_path=None))
+
+    return _logger
+
+
+logger = get_logger(LOG_HANDLER_NAME)

temp/mnt/ilens/ai-jobs/historical_weather_data/data1.csv

+Name,Date time,Maximum Temperature,Minimum Temperature,Temperature,Wind Chill,Heat Index,Precipitation,Snow,Snow Depth,Wind Speed,Wind Direction,Wind Gust,Visibility,Cloud Cover,Relative Humidity,Conditions
+"Aluva, Kerala, India",12/01/2019,30,25,27.5,,36.1,44.8,0,0,5.4,26.76,,3.8,44.3,86.46,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/02/2019,32.8,24,27.5,,39.1,18.03,0,0,16.3,99.09,,3.7,40.1,85.35,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/03/2019,31,24,27.4,,36.4,42.56,0,0,11.2,84.48,,4.1,32.3,79.52,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/04/2019,33.8,24,28.4,,37.8,0,0,0,11.7,96.38,,5,31.1,72.24,Partially cloudy
+"Aluva, Kerala, India",12/05/2019,32,23,27.9,,34.8,0,0,0,14.8,124.7,,4.4,27.9,68.83,Partially cloudy
+"Aluva, Kerala, India",12/06/2019,33,23,27.7,,36.9,0,0,0,16,74,,4.2,29.4,71.77,Partially cloudy
+"Aluva, Kerala, India",12/07/2019,34,24,28.4,,38,0,0,0,11.2,110.29,,4.6,37,69.89,Partially cloudy
+"Aluva, Kerala, India",12/08/2019,34,24,28.5,,38.2,0,0,0,11.8,68.21,,4.6,31,71.52,Partially cloudy
+"Aluva, Kerala, India",12/09/2019,34,23,27.8,,34.5,0,0,0,14.8,75.57,,4.7,24.2,66.08,Clear
+"Aluva, Kerala, India",12/10/2019,35,23,27.8,,35.7,0,0,0,14.8,109.04,,4.9,18.1,68.74,Clear
+"Aluva, Kerala, India",12/11/2019,33.8,24.9,28.9,,36.9,0,0,0,16.6,130.79,,4.8,27.5,67.35,Partially cloudy
+"Aluva, Kerala, India",12/12/2019,34,24,28,,39.2,1,0,0,21.7,127.88,,4.2,27.8,76.68,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/13/2019,34,23,28.2,,37,0,0,0,12.9,87.83,,4.5,30.1,72.56,Partially cloudy
+"Aluva, Kerala, India",12/14/2019,32.8,24,26.7,,36.8,38.98,0,0,20.5,82.96,,4.3,37.8,83.37,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/15/2019,32,23.9,27.2,,35,31,0,0,14.8,71.91,,4.5,29.3,76.1,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/16/2019,33,21,26.8,,33.9,0,0,0,13,75.96,,5,33.9,69.64,Partially cloudy
+"Aluva, Kerala, India",12/17/2019,33,23,27.5,,36,0,0,0,19.9,33.43,,4.7,29.3,70.92,Partially cloudy
+"Aluva, Kerala, India",12/18/2019,33,23,27.6,,36,0,0,0,16,48.42,,4.4,33.7,68.73,Partially cloudy
+"Aluva, Kerala, India",12/19/2019,34,23,28.2,,37.2,0,0,0,16.6,59.33,,4.8,26.1,69.76,Partially cloudy
+"Aluva, Kerala, India",12/20/2019,32,24,27.5,,35.7,0,0,0,14.8,130.29,,4.9,29.5,73.58,Partially cloudy
+"Aluva, Kerala, India",12/21/2019,31,25,27.4,,34.4,0,0,0,18.4,80.58,,3.9,32.8,73.91,Partially cloudy
+"Aluva, Kerala, India",12/22/2019,35,23,28.3,,39.4,0,0,0,13,98.52,,4.6,26.2,69.11,Partially cloudy
+"Aluva, Kerala, India",12/23/2019,32,24,27.7,,35.7,0,0,0,12.8,133.91,,4.1,39.5,74.76,Partially cloudy
+"Aluva, Kerala, India",12/24/2019,34,23,27.9,,38.2,0,0,0,13,100,,4.1,36.7,74.27,Partially cloudy
+"Aluva, Kerala, India",12/25/2019,33.8,24,28.6,,38,0,0,0,13,76.09,,4.6,38.8,72.14,Partially cloudy
+"Aluva, Kerala, India",12/26/2019,33,25,28.3,,37,0,0,0,16.6,99.38,,4.6,27.5,73.7,Partially cloudy
+"Aluva, Kerala, India",12/27/2019,34,24,28.2,,39.1,0,0,0,14.8,121.08,,4.4,36.3,75.48,Partially cloudy
+"Aluva, Kerala, India",12/28/2019,33.7,25,28.3,,37.1,0,0,0,18.4,130.04,,3.8,33.4,76.65,Partially cloudy
+"Aluva, Kerala, India",12/29/2019,34,24,28.5,,36.9,0,0,0,13.6,96.3,,4.4,28.3,69.59,Partially cloudy
+"Aluva, Kerala, India",12/30/2019,33,23,27.9,,36.9,0,0,0,18.4,119.04,,4.6,24.8,71.51,Clear
+"Aluva, Kerala, India",12/31/2019,33,24,27.8,,35.9,0,0,0,16.6,128,,4.2,37.3,73.04,Partially cloudy

temp/mnt/ilens/ai-jobs/historical_weather_data/data2.csv

+Name,Date time,Maximum Temperature,Minimum Temperature,Temperature,Wind Chill,Heat Index,Precipitation,Snow,Snow Depth,Wind Speed,Wind Direction,Wind Gust,Visibility,Cloud Cover,Relative Humidity,Conditions
+"Aluva, Kerala, India",12/01/2018,34,24.7,28.8,,37.9,34.8,0,0,13,69.67,,4.2,41.8,74.57,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/02/2018,33,24,28.1,,36.9,0,0,0,18.4,88.45,,4.3,36.8,71.04,Partially cloudy
+"Aluva, Kerala, India",12/03/2018,33,24,27.8,,35.3,0,0,0,18.4,75.92,,4.2,31.9,73.84,Partially cloudy
+"Aluva, Kerala, India",12/04/2018,33,24,27.4,,36.9,17,0,0,10.2,70.17,,4,66.3,80.61,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/05/2018,33.8,24.9,28.4,,39.1,40,0,0,16.6,54.55,,4.3,71.5,78.2,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/06/2018,33,25,28.8,,39.1,3,0,0,13.9,88.65,,4.5,52.9,76.17,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/07/2018,33,25,28.5,,37,0,0,0,14.8,122.96,,4.4,45.8,75.38,Partially cloudy
+"Aluva, Kerala, India",12/08/2018,34,24,28.5,,36.4,3,0,0,14,91.45,,4.4,45.5,71.85,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/09/2018,33,23,27.7,,35.7,0,0,0,14.8,85.73,,4.3,29.5,74.55,Partially cloudy
+"Aluva, Kerala, India",12/10/2018,33,24,27.7,,48.4,0,0,0,13,56.88,,3.3,50.5,79.23,Partially cloudy
+"Aluva, Kerala, India",12/11/2018,33,23,27.4,,36,0,0,0,13,64.48,,3.3,30.2,75.17,Partially cloudy
+"Aluva, Kerala, India",12/12/2018,32.8,24,27,,37.9,0,0,0,15.5,141.1,,3.7,50.1,75.94,Partially cloudy
+"Aluva, Kerala, India",12/13/2018,31.2,24,27.6,,36.5,0,0,0,18.1,151.41,,3.7,46.7,78.05,Partially cloudy
+"Aluva, Kerala, India",12/14/2018,33,24,27.7,,37.2,1,0,0,14.8,84.5,,4,43.8,75.64,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/15/2018,34,22,27,,36.4,0,0,0,12.6,71.29,,4.3,36.5,72.72,Partially cloudy
+"Aluva, Kerala, India",12/16/2018,32,23,27,,35.7,0,0,0,9.4,64,,4.1,43.2,72.9,Partially cloudy
+"Aluva, Kerala, India",12/17/2018,32,23,26.8,,35.7,0,0,0,16.6,103.83,,4.1,42.7,77.33,Partially cloudy
+"Aluva, Kerala, India",12/18/2018,32.7,21,26.4,,35.4,0,0,0,16.6,109.58,,4,35.2,73.73,Partially cloudy
+"Aluva, Kerala, India",12/19/2018,32,23,27,,36.7,0,0,0,14.8,119.26,,4.2,28.9,77.28,Partially cloudy
+"Aluva, Kerala, India",12/20/2018,32,24.9,27.2,,35.7,5.2,0,0,16.6,111.26,,3.8,69.4,79.04,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/21/2018,33,23,27.5,,36,0,0,0,16.6,89.48,,3.8,42.6,77.2,Partially cloudy
+"Aluva, Kerala, India",12/22/2018,32,23,27.3,,35.7,0,0,0,7.6,100.76,,3.7,45.7,76.22,Partially cloudy
+"Aluva, Kerala, India",12/23/2018,33.7,23,28.2,,38,0,0,0,11.2,124.67,,4.2,45.5,74.02,Partially cloudy
+"Aluva, Kerala, India",12/24/2018,34,24,27.9,,37.2,3.4,0,0,22.3,106.79,,3.8,57,77.59,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/25/2018,33,23,27.7,,38.9,36.84,0,0,11.3,104.19,,4.1,64.5,78.75,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/26/2018,34,24,28,,37.2,8.15,0,0,9.8,180,,4,53.3,75.06,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/27/2018,33,24,28.1,,36,0,0,0,14.8,154.52,,4.4,46.3,73.99,Partially cloudy
+"Aluva, Kerala, India",12/28/2018,32,24,27.5,,35.7,26,0,0,11.2,109.52,,4.1,42,74.73,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/29/2018,29,23,26.5,,32.8,3.49,0,0,10.2,133,,3.6,79.1,82.31,"Rain, Overcast"
+"Aluva, Kerala, India",12/30/2018,32,24,27.5,,34.8,0.1,0,0,13.8,101.71,,4.4,43.7,74.38,"Rain, Partially cloudy"
+"Aluva, Kerala, India",12/31/2018,34,23,27,,34,0,0,0,14.8,119.58,,4.2,26.8,68.52,Partially cloudy
+"Aluva, Kerala, India",01/01/2019,32,22,26.6,,32.4,0,0,0,16.6,75.92,,4.1,20.4,62.89,Clear
+"Aluva, Kerala, India",01/02/2019,32,20,25.6,,32.3,0,0,0,11.5,92.43,,4.5,15.9,63.17,Clear
+"Aluva, Kerala, India",01/03/2019,35,19,25.9,,35.1,0,0,0,11.2,79.52,,4.8,12.9,57.01,Clear
+"Aluva, Kerala, India",01/04/2019,33,18,25.4,,32.2,0,0,0,89.9,62.29,,4.7,9.9,57.2,Clear
+"Aluva, Kerala, India",01/05/2019,33,18,24.6,,32.2,0,0,0,14.8,125.14,,4.5,28.7,61.5,Partially cloudy
+"Aluva, Kerala, India",01/06/2019,31,19,24.8,,31.3,0,0,0,14.3,140.05,,4.4,30.9,66.8,Partially cloudy
+"Aluva, Kerala, India",01/07/2019,33,19,25.1,,35.4,0,0,0,14.8,121.38,,4.3,18.9,66.14,Clear
+"Aluva, Kerala, India",01/08/2019,34,20,25.7,,34.5,0,0,0,11.3,85.17,,4.3,9.8,64.22,Clear
+"Aluva, Kerala, India",01/09/2019,34,20,26.1,,34.5,0,0,0,11.2,102.83,,4.3,2.7,65.46,Clear
+"Aluva, Kerala, India",01/10/2019,34,19,26.2,,33.2,0,0,0,13,131.67,,4.9,12.6,62.24,Clear
+"Aluva, Kerala, India",01/11/2019,34,20,26.4,,33.6,0,0,0,15.5,124.29,,4.4,10.2,64.86,Clear
+"Aluva, Kerala, India",01/12/2019,33.7,21,26.7,,35.2,0,0,0,16.6,120.45,,4.1,29,67.94,Partially cloudy
+"Aluva, Kerala, India",01/13/2019,32,21,26.2,,32.3,0,0,0,14.8,139.76,,4.2,31.1,63.67,Partially cloudy
+"Aluva, Kerala, India",01/14/2019,33,20,26.3,,33.9,0,0,0,16.6,116.57,,4.2,27.1,65.35,Partially cloudy
+"Aluva, Kerala, India",01/15/2019,33,21,26.6,,33.9,0,0,0,13,155.79,,4.2,31.8,68.69,Partially cloudy
+"Aluva, Kerala, India",01/16/2019,35,20,26.6,,34.5,0,0,0,14.8,120.95,,4.6,28.4,58.37,Partially cloudy
+"Aluva, Kerala, India",01/17/2019,34,20,26.2,,34,0,0,0,13,92.5,,4.3,30,61.64,Partially cloudy
+"Aluva, Kerala, India",01/18/2019,32,20,25.4,,32.3,0,0,0,11.6,100.24,,3.9,30.7,68.66,Partially cloudy
+"Aluva, Kerala, India",01/19/2019,33,20,25.7,,33.9,0,0,0,18.4,124.2,,4.3,30.4,68.49,Partially cloudy
+"Aluva, Kerala, India",01/20/2019,32,20,25.2,,31.9,0,0,0,10.3,93.64,,4.1,61.8,65.48,Partially cloudy
+"Aluva, Kerala, India",01/21/2019,31,18,23.8,,31.3,0,0,0,20.5,124.95,,4.4,35.7,64.96,Partially cloudy
+"Aluva, Kerala, India",01/22/2019,32,19,25,,31.5,0,0,0,14.8,159.91,,4.7,21.3,61.92,Clear
+"Aluva, Kerala, India",01/23/2019,33,20,25.8,,34.1,0,0,0,18.4,120.96,,4.7,17.8,64.76,Clear
+"Aluva, Kerala, India",01/24/2019,33,22,27,,34.5,0,0,0,18.4,146,,4.5,22,72.42,Clear
+"Aluva, Kerala, India",01/25/2019,30,23,26.3,,34,0,0,0,18.4,161.59,,3.4,30.3,79.3,Partially cloudy
+"Aluva, Kerala, India",01/26/2019,32.8,23,27.3,,36.6,0,0,0,24.1,135.61,,3.7,27.2,73.61,Partially cloudy
+"Aluva, Kerala, India",01/27/2019,33,23,27.1,,36.7,0,0,0,22.3,147.71,,4.8,25.4,73.38,Partially cloudy
+"Aluva, Kerala, India",01/28/2019,33,23,27.5,,36.6,0,0,0,23.2,177,,4.4,25.8,73.89,Partially cloudy
+"Aluva, Kerala, India",01/29/2019,33.6,24,27.8,,38,0,0,0,16.6,131.74,,4.4,35.4,73.3,Partially cloudy
+"Aluva, Kerala, India",01/30/2019,34.7,23,27.7,,38.2,0,0,0,18.4,106.96,,4.3,34.3,69.71,Partially cloudy
+"Aluva, Kerala, India",01/31/2019,34,23,27.9,,34.5,0,0,0,14.4,143.45,,4.1,30.3,67.17,Partially cloudy
+"Aluva, Kerala, India",02/01/2019,34,22,27.2,,34,0,0,0,18.4,133.19,,4.1,26.2,64.31,Partially cloudy
+"Aluva, Kerala, India",02/02/2019,34,22,27,,35.2,0,0,0,16.6,107.55,,4.3,9.9,68.88,Clear
+"Aluva, Kerala, India",02/03/2019,33.6,23,27.6,,38,0,0,0,16.6,138.35,,4.2,14.9,71.27,Clear
+"Aluva, Kerala, India",02/04/2019,36,23,28.5,,37.3,0,0,0,18.4,99.79,,4.1,24.1,65.95,Clear
+"Aluva, Kerala, India",02/05/2019,33,25,28.4,,36.2,0,0,0,14.8,91.91,,4,29.5,70.8,Partially cloudy
+"Aluva, Kerala, India",02/06/2019,34,25,29.1,,37.2,18,0,0,18.4,80.48,,4.1,27,69.68,"Rain, Partially cloudy"
+"Aluva, Kerala, India",02/07/2019,33,25,28.8,,37.7,21.32,0,0,20.5,145.5,,4,27.2,72.55,"Rain, Partially cloudy"
+"Aluva, Kerala, India",02/08/2019,34,25,29,,39.2,0,0,0,22.3,206.33,,4.2,36.1,73.13,Partially cloudy
+"Aluva, Kerala, India",02/09/2019,33,25,28.3,,38.7,0,0,0,18.4,139.42,,4,40.1,75.84,Partially cloudy
+"Aluva, Kerala, India",02/10/2019,33,24.1,27.3,,39.1,0.2,0,0,25.9,73.38,,4.4,37.5,77.08,"Rain, Partially cloudy"
+"Aluva, Kerala, India",02/11/2019,35,23,28.3,,38.4,15.51,0,0,18.4,136.61,,4.3,32.6,72.64,"Rain, Partially cloudy"
+"Aluva, Kerala, India",02/12/2019,35,23,29.1,,36.2,0,0,0,11.2,66.83,,4.3,27.6,67.93,Partially cloudy
+"Aluva, Kerala, India",02/13/2019,36,23.1,28.7,,36.2,0,0,0,16.6,76.3,,4.4,18.4,62.95,Clear
+"Aluva, Kerala, India",02/14/2019,36,22,28.4,,38,0,0,0,18.4,125.37,,4.4,27.1,62.52,Partially cloudy
+"Aluva, Kerala, India",02/15/2019,34,24,28.9,,39.1,0,0,0,22.3,127.67,,4.2,30.5,69.88,Partially cloudy
+"Aluva, Kerala, India",02/16/2019,35,25,29.2,,41.8,0,0,0,20.5,123.78,,4.5,31.7,73.54,Partially cloudy
+"Aluva, Kerala, India",02/17/2019,34,25.9,29,,39.2,5,0,0,22.3,150.21,,4.4,36.5,73.6,"Rain, Partially cloudy"
+"Aluva, Kerala, India",02/18/2019,34,24,28.3,,37.2,0,0,0,22.3,154.13,,4.6,29.8,70.69,Partially cloudy
+"Aluva, Kerala, India",02/19/2019,34,24,28.2,,39.1,0,0,0,20.5,203.05,,4.3,31,73.36,Partially cloudy
+"Aluva, Kerala, India",02/20/2019,35,25,29.5,,40.2,0,0,0,18.7,105.57,,4.1,45.6,72.37,Partially cloudy
+"Aluva, Kerala, India",02/21/2019,34.7,25,29,,37.2,0,0,0,14.8,114.87,,4.4,26.2,67.96,Partially cloudy
+"Aluva, Kerala, India",02/22/2019,36,22,28.3,,34.7,0,0,0,14.8,89.46,,4.4,19.6,58.72,Clear
+"Aluva, Kerala, India",02/23/2019,34,22,27.7,,35.2,0,0,0,18.4,138.54,,4.1,23.6,65.38,Clear
+"Aluva, Kerala, India",02/24/2019,36.7,23,28.3,,37.6,0,0,0,18.4,117.59,,4.3,26.2,68.27,Partially cloudy
+"Aluva, Kerala, India",02/25/2019,34,23,27.8,,37,0,0,0,22,136.65,,4.3,22.7,71.49,Clear
+"Aluva, Kerala, India",02/26/2019,33,24.1,28.4,,38,0,0,0,20.5,147.89,,4.2,30.7,73.73,Partially cloudy
+"Aluva, Kerala, India",02/27/2019,32,25.9,28.7,,36.7,0,0,0,22.3,167.57,,4.3,36.7,73.67,Partially cloudy
+"Aluva, Kerala, India",02/28/2019,34,26,29.2,,39,0,0,0,18.4,235.63,,4.6,30.7,69.09,Partially cloudy

temp/mnt/ilens/ai-jobs/pipe1/default_run_id/MergeWeatherData/param.json

+{
+  "historic_file_name": "data1.csv"
+}
\ No newline at end of file

temp/mnt/ilens/ai-jobs/pipe1/default_run_id/ModelFitting/input/X_train.csv

+,Maximum Temperature,Minimum Temperature,Temperature,Precipitation,Wind Speed,Wind Direction,Visibility,Cloud Cover,Relative Humidity,month
+0,30.0,25.0,27.5,44.8,5.4,26.76,3.8,44.3,86.46,12
+1,32.8,24.0,27.5,18.03,16.3,99.09,3.7,40.1,85.35,12
+2,31.0,24.0,27.4,42.56,11.2,84.48,4.1,32.3,79.52,12
+3,33.8,24.0,28.4,0.0,11.7,96.38,5.0,31.1,72.24,12
+4,32.0,23.0,27.9,0.0,14.8,124.7,4.4,27.9,68.83,12
+5,33.0,23.0,27.7,0.0,16.0,74.0,4.2,29.4,71.77,12
+6,34.0,24.0,28.4,0.0,11.2,110.29,4.6,37.0,69.89,12
+7,34.0,24.0,28.5,0.0,11.8,68.21,4.6,31.0,71.52,12
+8,34.0,23.0,27.8,0.0,14.8,75.57,4.7,24.2,66.08,12
+9,35.0,23.0,27.8,0.0,14.8,109.04,4.9,18.1,68.74,12
+10,33.8,24.9,28.9,0.0,16.6,130.79,4.8,27.5,67.35,12
+11,34.0,24.0,28.0,1.0,21.7,127.88,4.2,27.8,76.68,12
+12,34.0,23.0,28.2,0.0,12.9,87.83,4.5,30.1,72.56,12
+13,32.8,24.0,26.7,38.98,20.5,82.96,4.3,37.8,83.37,12
+14,32.0,23.9,27.2,31.0,14.8,71.91,4.5,29.3,76.1,12
+15,33.0,21.0,26.8,0.0,13.0,75.96,5.0,33.9,69.64,12
+16,33.0,23.0,27.5,0.0,19.9,33.43,4.7,29.3,70.92,12
+17,33.0,23.0,27.6,0.0,16.0,48.42,4.4,33.7,68.73,12
+18,34.0,23.0,28.2,0.0,16.6,59.33,4.8,26.1,69.76,12
+19,32.0,24.0,27.5,0.0,14.8,130.29,4.9,29.5,73.58,12
+20,31.0,25.0,27.4,0.0,18.4,80.58,3.9,32.8,73.91,12
+21,35.0,23.0,28.3,0.0,13.0,98.52,4.6,26.2,69.11,12
+22,32.0,24.0,27.7,0.0,12.8,133.91,4.1,39.5,74.76,12
+23,34.0,23.0,27.9,0.0,13.0,100.0,4.1,36.7,74.27,12
+24,33.8,24.0,28.6,0.0,13.0,76.09,4.6,38.8,72.14,12
+25,33.0,25.0,28.3,0.0,16.6,99.38,4.6,27.5,73.7,12
+26,34.0,24.0,28.2,0.0,14.8,121.08,4.4,36.3,75.48,12
+27,33.7,25.0,28.3,0.0,18.4,130.04,3.8,33.4,76.65,12
+28,34.0,24.0,28.5,0.0,13.6,96.3,4.4,28.3,69.59,12
+29,33.0,23.0,27.9,0.0,18.4,119.04,4.6,24.8,71.51,12
+30,33.0,24.0,27.8,0.0,16.6,128.0,4.2,37.3,73.04,12

temp/mnt/ilens/ai-jobs/pipe1/default_run_id/ModelFitting/input/Y_train.csv

+,value
+0,553000
+1,834000
+2,626000
+3,961000
+4,976000
+5,913000
+6,921000
+7,934000
+8,999000
+9,988000
+10,955000
+11,858000
+12,934000
+13,891000
+14,680000
+15,957000
+16,961000
+17,920000
+18,957000
+19,817000
+20,486000
+21,928000
+22,458000
+23,542000
+24,778000
+25,536000
+26,708000
+27,706000
+28,763000
+29,742000
+30,662000

train.csv

+,value,Maximum Temperature,Minimum Temperature,Temperature,Precipitation,Wind Speed,Wind Direction,Visibility,Cloud Cover,Relative Humidity,month
+0,553000,30.0,25.0,27.5,44.8,5.4,26.76,3.8,44.3,86.46,12
+1,834000,32.8,24.0,27.5,18.03,16.3,99.09,3.7,40.1,85.35,12
+2,626000,31.0,24.0,27.4,42.56,11.2,84.48,4.1,32.3,79.52,12
+3,961000,33.8,24.0,28.4,0.0,11.7,96.38,5.0,31.1,72.24,12
+4,976000,32.0,23.0,27.9,0.0,14.8,124.7,4.4,27.9,68.83,12
+5,913000,33.0,23.0,27.7,0.0,16.0,74.0,4.2,29.4,71.77,12
+6,921000,34.0,24.0,28.4,0.0,11.2,110.29,4.6,37.0,69.89,12
+7,934000,34.0,24.0,28.5,0.0,11.8,68.21,4.6,31.0,71.52,12
+8,999000,34.0,23.0,27.8,0.0,14.8,75.57,4.7,24.2,66.08,12
+9,988000,35.0,23.0,27.8,0.0,14.8,109.04,4.9,18.1,68.74,12
+10,955000,33.8,24.9,28.9,0.0,16.6,130.79,4.8,27.5,67.35,12
+11,858000,34.0,24.0,28.0,1.0,21.7,127.88,4.2,27.8,76.68,12
+12,934000,34.0,23.0,28.2,0.0,12.9,87.83,4.5,30.1,72.56,12
+13,891000,32.8,24.0,26.7,38.98,20.5,82.96,4.3,37.8,83.37,12
+14,680000,32.0,23.9,27.2,31.0,14.8,71.91,4.5,29.3,76.1,12
+15,957000,33.0,21.0,26.8,0.0,13.0,75.96,5.0,33.9,69.64,12
+16,961000,33.0,23.0,27.5,0.0,19.9,33.43,4.7,29.3,70.92,12
+17,920000,33.0,23.0,27.6,0.0,16.0,48.42,4.4,33.7,68.73,12
+18,957000,34.0,23.0,28.2,0.0,16.6,59.33,4.8,26.1,69.76,12
+19,817000,32.0,24.0,27.5,0.0,14.8,130.29,4.9,29.5,73.58,12
+20,486000,31.0,25.0,27.4,0.0,18.4,80.58,3.9,32.8,73.91,12
+21,928000,35.0,23.0,28.3,0.0,13.0,98.52,4.6,26.2,69.11,12
+22,458000,32.0,24.0,27.7,0.0,12.8,133.91,4.1,39.5,74.76,12
+23,542000,34.0,23.0,27.9,0.0,13.0,100.0,4.1,36.7,74.27,12
+24,778000,33.8,24.0,28.6,0.0,13.0,76.09,4.6,38.8,72.14,12
+25,536000,33.0,25.0,28.3,0.0,16.6,99.38,4.6,27.5,73.7,12
+26,708000,34.0,24.0,28.2,0.0,14.8,121.08,4.4,36.3,75.48,12
+27,706000,33.7,25.0,28.3,0.0,18.4,130.04,3.8,33.4,76.65,12
+28,763000,34.0,24.0,28.5,0.0,13.6,96.3,4.4,28.3,69.59,12
+29,742000,33.0,23.0,27.9,0.0,18.4,119.04,4.6,24.8,71.51,12
+30,662000,33.0,24.0,27.8,0.0,16.6,128.0,4.2,37.3,73.04,12