Skip to content

P
PreprocessWeatherData
Commit e1e6add9 authored by dasharatha.vamshi's avatar dasharatha.vamshi
Browse files
Options

init

parent 03121e35
Expand all Show whitespace changes
Inline Side-by-side
Showing with 10461 additions and 0 deletions
.idea/.gitignore 0 → 100644
View file @ e1e6add9
# Default ignored files
/shelf/
/workspace.xml
.idea/inspectionProfiles/Project_Default.xml 0 → 100644
View file @ e1e6add9
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyChainedComparisonsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoreConstantInTheMiddle" value="true" />
</inspection_tool>
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="55">
<item index="0" class="java.lang.String" itemvalue="scikit-image" />
<item index="1" class="java.lang.String" itemvalue="scipy" />
<item index="2" class="java.lang.String" itemvalue="opencv-python" />
<item index="3" class="java.lang.String" itemvalue="torch" />
<item index="4" class="java.lang.String" itemvalue="torchvision" />
<item index="5" class="java.lang.String" itemvalue="absl-py" />
<item index="6" class="java.lang.String" itemvalue="protobuf" />
<item index="7" class="java.lang.String" itemvalue="rsa" />
<item index="8" class="java.lang.String" itemvalue="tensorflow-estimator" />
<item index="9" class="java.lang.String" itemvalue="opt-einsum" />
<item index="10" class="java.lang.String" itemvalue="python-dateutil" />
<item index="11" class="java.lang.String" itemvalue="cycler" />
<item index="12" class="java.lang.String" itemvalue="gast" />
<item index="13" class="java.lang.String" itemvalue="numpy" />
<item index="14" class="java.lang.String" itemvalue="pyasn1" />
<item index="15" class="java.lang.String" itemvalue="requests" />
<item index="16" class="java.lang.String" itemvalue="importlib-metadata" />
<item index="17" class="java.lang.String" itemvalue="pyasn1-modules" />
<item index="18" class="java.lang.String" itemvalue="requests-oauthlib" />
<item index="19" class="java.lang.String" itemvalue="tensorflow" />
<item index="20" class="java.lang.String" itemvalue="tensorboard-plugin-wit" />
<item index="21" class="java.lang.String" itemvalue="zipp" />
<item index="22" class="java.lang.String" itemvalue="oauthlib" />
<item index="23" class="java.lang.String" itemvalue="astunparse" />
<item index="24" class="java.lang.String" itemvalue="urllib3" />
<item index="25" class="java.lang.String" itemvalue="pyparsing" />
<item index="26" class="java.lang.String" itemvalue="Cython" />
<item index="27" class="java.lang.String" itemvalue="Markdown" />
<item index="28" class="java.lang.String" itemvalue="google-auth-oauthlib" />
<item index="29" class="java.lang.String" itemvalue="Werkzeug" />
<item index="30" class="java.lang.String" itemvalue="kiwisolver" />
<item index="31" class="java.lang.String" itemvalue="tqdm" />
<item index="32" class="java.lang.String" itemvalue="yolov5processor" />
<item index="33" class="java.lang.String" itemvalue="tensorboard" />
<item index="34" class="java.lang.String" itemvalue="future" />
<item index="35" class="java.lang.String" itemvalue="matplotlib" />
<item index="36" class="java.lang.String" itemvalue="cachetools" />
<item index="37" class="java.lang.String" itemvalue="grpcio" />
<item index="38" class="java.lang.String" itemvalue="Keras" />
<item index="39" class="java.lang.String" itemvalue="google-auth" />
<item index="40" class="java.lang.String" itemvalue="idna" />
<item index="41" class="java.lang.String" itemvalue="Pillow" />
<item index="42" class="java.lang.String" itemvalue="cython" />
<item index="43" class="java.lang.String" itemvalue="keras" />
<item index="44" class="java.lang.String" itemvalue="imgaug" />
<item index="45" class="java.lang.String" itemvalue="opencv-contrib-python" />
<item index="46" class="java.lang.String" itemvalue="paho-mqtt" />
<item index="47" class="java.lang.String" itemvalue="pymongo" />
<item index="48" class="java.lang.String" itemvalue="fbprophet" />
<item index="49" class="java.lang.String" itemvalue="scikit-learn" />
<item index="50" class="java.lang.String" itemvalue="sklearn" />
<item index="51" class="java.lang.String" itemvalue="statsmodels" />
<item index="52" class="java.lang.String" itemvalue="python-dotenv" />
<item index="53" class="java.lang.String" itemvalue="pystan" />
<item index="54" class="java.lang.String" itemvalue="Cerberus" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N802" />
<option value="N806" />
<option value="N801" />
<option value="N803" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="cv2.imresize" />
<option value="detectron2.model_zoo.get_config_file" />
<option value="detectron2.model_zoo.get_checkpoint_url" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
View file @ e1e6add9
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
.idea/misc.xml 0 → 100644
View file @ e1e6add9
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (welspun-defects) (2)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
.idea/modules.xml 0 → 100644
View file @ e1e6add9
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/preprocessweatherdata.iml" filepath="$PROJECT_DIR$/.idea/preprocessweatherdata.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/preprocessweatherdata.iml 0 → 100644
View file @ e1e6add9
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.7 (welspun-defects) (2)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="pytest" />
</component>
</module>
\ No newline at end of file
.idea/vcs.xml 0 → 100644
View file @ e1e6add9
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
Dockerfile 0 → 100644
View file @ e1e6add9
FROM python:3.7-slim
ADD . /opt
WORKDIR /opt
RUN pip install -r requirements.txt
CMD python main.py
conf/configuration.yml 0 → 100644
View file @ e1e6add9
#---------------Service Configurations----------------#
SERVICE_CONFIG:
LOG_LEVEL: info
LOG_HANDLER_NAME: PreprocessWeatherData
LOGSTASH_HOST: 192.168.1.47
LOGSTASH_PORT: 5000
#--------------System Configurations--------------------#
SYSTEM_CONFIG:
shared_mount_base_ai_job: /mnt/ilens/ai-job/
#----------------------If read conf from mongo------------#
FOR_EACH_MONGO_CONFIG:
READ_FROM_MONGO: true
MONGO_URI: mongodb://192.168.0.210:27017
MONGO_DB: iLensAiPipeline
MONGO_RUN_COLL: runMetadata
MONGO_SITE_COLL: siteMetadata
logs/PreprocessWeatherData.log 0 → 100644
View file @ e1e6add9
2021-03-02 19:09:57,986 INFO PreprocessWeatherData Traceback (most recent call last):
File "E:/iLens-AI/preprocessweatherdata/main.py", line 62, in <module>
json_path = os.path.join(json_path_dir,os.listdir(json_path_dir)[0])
FileNotFoundError: [WinError 3] The system cannot find the path specified: 'temp/mnt/ilens/ai-job//pipe1/run1PreprocessWeatherData/input'
2021-03-02 19:11:35,286 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:11:35,301 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input/data.csv
2021-03-02 19:13:19,263 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:13:19,272 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input/data.csv
2021-03-02 19:14:37,570 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:14:37,580 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:14:37,584 INFO PreprocessWeatherData Component executed Successfully
2021-03-02 19:15:18,736 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:15:18,745 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:15:18,749 INFO PreprocessWeatherData Component executed Successfully
2021-03-02 19:17:47,686 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:17:47,696 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:17:47,701 INFO PreprocessWeatherData Component executed Successfully
main.py 0 → 100644
View file @ e1e6add9
import json
import pickle
import traceback
import os
import pandas as pd
from sklearn.preprocessing import StandardScaler
from scripts.common.config_parser import *
from scripts.common.constants import PreprocessWeatherDataConstants, ComponentExceptions
from scripts.common.logsetup import logger
class PreProcessComponent:
def __init__(self):
self.ds, self.tempmax, self.tempmin, self.temp, self.humidity, self.precip, self.winddir, self.cloudcover, self.visibility, self.date, self.windspeed = (
[] for i in range(11))
def read_pickle_file(self, file):
try:
logger.info("Parsing the pickle file at location " + file + " ......")
return pickle.load(open(file, "rb"))
except:
logger.info("Failed to read pickle file")
logger.info("----Traceback----")
logger.info(traceback.format_exc())
def preprocess(self, forcast_json, null_value_method):
try:
logger.info("Reading Json file")
with open(forcast_json) as f:
data = json.load(f)
except:
logger.info("Failed reading Json File")
logger.info(traceback.format_exc())
for i in data[PreprocessWeatherDataConstants.DAYS_KEY]:
self.date.append(i[PreprocessWeatherDataConstants.DATETIME_KEY])
self.ds.append(int(i[PreprocessWeatherDataConstants.DATETIME_KEY].split("-")[1]))
self.tempmax.append(i[PreprocessWeatherDataConstants.TEMPMAX_KEY])
self.tempmin.append(i[PreprocessWeatherDataConstants.TEMPMIN_KEY])
self.temp.append(i[PreprocessWeatherDataConstants.TEMP_KEY])
self.humidity.append(i[PreprocessWeatherDataConstants.HUMIDITY_KEY])
self.precip.append(i[PreprocessWeatherDataConstants.PRECIP_KEY])
self.windspeed.append(i[PreprocessWeatherDataConstants.WINDSPEED_KEY])
self.winddir.append(i[PreprocessWeatherDataConstants.WINDDIR_KEY])
self.cloudcover.append(i[PreprocessWeatherDataConstants.CLOUDCOVER_KEY])
self.visibility.append(i[PreprocessWeatherDataConstants.VISIBILITY_KEY])
forcast_data = pd.DataFrame(
{"Maximum Temperature": self.tempmax, "Minimum Temperature": self.tempmin, "Temperature": self.temp,
"Precipitation": self.precip, "Wind Speed": self.windspeed, "Wind Direction": self.winddir,
"Visibility": self.visibility, "Cloud Cover": self.cloudcover,
"Relative Humidity": self.humidity, "month": self.ds})
forcast_data.fillna(method=null_value_method, inplace=True)
forcast_data.index = self.date
return forcast_data
if __name__ == '__main__':
try:
# Checking json path
json_path_dir = config[PreprocessWeatherDataConstants.COMPONENT_INPUT_DIR]
json_path = os.path.join(json_path_dir, os.listdir(json_path_dir)[0])
except Exception as e:
logger.info(traceback.format_exc())
raise Exception(e)
# Checking Fill NA method
if PreprocessWeatherDataConstants.FILLNA_METHOD in config.keys():
fillna_method = config[PreprocessWeatherDataConstants.FILLNA_METHOD]
else:
raise Exception(ComponentExceptions.INVALID_Fillna_Method)
obj = PreProcessComponent()
forcast_df = obj.preprocess(json_path, fillna_method)
try:
logger.info("Writing forcast data to csv file at " + os.path.join(config['component_output_dir'], 'data.csv'))
output2 = os.path.join(config['component_output_dir'], 'data.csv')
forcast_df.to_csv(output2, index=True)
logger.info("Component executed Successfully")
except Exception as e:
raise Exception(e)
scripts/common/config_parser.py 0 → 100644
View file @ e1e6add9
#!/usr/bin/env python
import os
import sys
import yaml
import json
from pymongo import MongoClient, DESCENDING
from scripts.common.constants import PreprocessWeatherDataConstants
config_path = os.path.join(os.getcwd(), "conf", "configuration.yml")
if os.path.exists(config_path):
sys.stderr.write("Reading config from --> {}".format(config_path))
sys.stderr.write("\n")
with open(config_path, 'r') as stream:
_config = yaml.safe_load(stream)
else:
sys.stderr.write("Configuration not found...")
sys.stderr.write("Exiting....")
sys.exit(1)
# For testing uncomment this
# os.environ['pipeline_id'] = "pipe1"
# ------------------------ Configurations -----------------------------------------------------------------------------
pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job",
_config.get("SYSTEM_CONFIG", {}).get('shared_mount_base_ai_job'))
# read from $shared_mount_base_ai_job/$pipeline_id/run.config
run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run.json"
try:
sys.stderr.write("Checking for run id parameters at path " + run_id_path + "\n")
with open(run_id_path) as f:
run_id_param = json.load(f)
except Exception as e:
raise Exception(e)
run_id = run_id_param['run_id']
fillna_method = os.environ.get("fillna_method", default="ffill")
# shared_mount_base_ai_job/$pipeline_id/$run_id/PreprocessWeatherData/input/$.json
component_input = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + PreprocessWeatherDataConstants.COMPONENT_NAME + "/input"
# shared_mount_base_ai_job/$pipeline_id/$run_id/RunModel/input/data.csv
component_output_dir = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + PreprocessWeatherDataConstants.NEXT_COMPONENT + "/input"
BASE_LOG_PATH = os.path.join(os.getcwd(), "logs")
if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
os.mkdir(os.path.join(os.getcwd(), 'logs'))
LOG_LEVEL = os.environ.get("LOG_LEVEL", _config.get('SERVICE_CONFIG', {}).get("LOG_LEVEL", "INFO")).upper()
LOG_HANDLER_NAME = _config.get('SERVICE_CONFIG', {}).get("LOG_HANDLER_NAME", "PreprocessWeatherData")
ENABLE_LOGSTASH_LOG = os.environ.get("ENABLE_LOGSTASH_LOG", 'False').lower()
LOGSTASH_HOST = _config.get('SERVICE_CONFIG', {}).get('LOGSTASH_HOST')
LOGSTASH_PORT = str(_config.get('SERVICE_CONFIG', {}).get('LOGSTASH_PORT'))
# os.environ["shared_volume"] = "test"
# os.environ["json_path"] = r"E:\welspun-defects\preprocess_data-ilendev\response.json"
# os.environ["fillna_method"] = "ffill"
# os.environ["standard_scalar_path"] = r"E:\welspun-defects\preprocess_data-ilendev\StandardScaler.pkl"
# print(os.getcwd())
config = {
'pipeline_id': pipeline_id,
'run_id': run_id,
'shared_mount_base_ai_job': shared_mount_base_ai_job,
'fillna_method': fillna_method,
'component_input': component_input,
'component_output_dir': component_output_dir
}
if not os.path.exists(config['shared_mount_base_ai_job']):
sys.stderr.write("Shared path does not exist!" + "\n")
sys.stderr.write("Creating path --> {}".format(config['shared_mount_base_ai_job'] + "\n"))
os.makedirs(config['shared_mount_base_ai_job'])
if not os.path.exists(config['component_output_dir']):
sys.stderr.write("component_output_dir does not exist!" + "\n")
sys.stderr.write("Creating path --> {}".format(config['component_output_dir'] + "\n"))
os.makedirs(config['component_output_dir'])
scripts/common/constants.py 0 → 100644
View file @ e1e6add9
#!/usr/bin/env python
class PreprocessWeatherDataConstants:
COMPONENT_INPUT_DIR = "component_input"
COMPONENT_NAME = "PreprocessWeatherData"
NEXT_COMPONENT = "RunModel"
SHARED_VOLUME = "shared_volume"
JSON_PATH = "json_path"
FILLNA_METHOD = "fillna_method"
STANDARD_SCALAR_PATH = "standard_scalar_path"
TEMPMAX_KEY = "tempmax"
TEMPMIN_KEY = "tempmin"
TEMP_KEY = "temp"
HUMIDITY_KEY = "humidity"
PRECIP_KEY = "precip"
WINDSPEED_KEY = "windspeed"
WINDDIR_KEY = "winddir"
CLOUDCOVER_KEY = "cloudcover"
VISIBILITY_KEY = "visibility"
DAYS_KEY = "days"
DATETIME_KEY = "datetime"
REQUEST_SUCCESS_CODE = 200
ERRORS_KEY = "errors"
Default_headers = {'content-type': 'application/json'}
HTTP = "http://"
LOG_VAR_MESSAGE = "\n" + "#" * 25 + "\n" + "{}" + "\n" + "#" * 25 + "\n" + "{}\n"
class ComponentExceptions:
INVALID_SHARED_VOLUME = "Shared Volume is required"
INVALID_JSON_PATH = "Path to Json file is required"
INVALID_Standard_Scalar_Pkl_Path = "Path to Standard Scalar Pkl file is needed"
INVALID_Fillna_Method = "Method to fill NA values is required"
scripts/common/logsetup.py 0 → 100644
View file @ e1e6add9
import os
import logging
from logging.handlers import RotatingFileHandler
from logstash_async.handler import AsynchronousLogstashHandler
from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH
from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH, LOGSTASH_HOST, LOGSTASH_PORT, ENABLE_LOGSTASH_LOG
DEFAULT_FORMAT = '%(asctime)s %(levelname)5s %(name)s %(message)s'
DEBUG_FORMAT = '%(asctime)s %(levelname)5s %(name)s [%(threadName)5s:%(filename)5s:%(funcName)5s():%(lineno)s] %(' \
'message)s '
EXTRA = {}
FORMATTER = DEFAULT_FORMAT
if LOG_LEVEL.strip() == "DEBUG":
FORMATTER = DEBUG_FORMAT
logging.trace = logging.DEBUG - 5
logging.addLevelName(logging.DEBUG - 5, 'TRACE')
class ILensLogger(logging.getLoggerClass()):
def __init__(self, name):
super().__init__(name)
def trace(self, msg, *args, **kwargs):
if self.isEnabledFor(logging.trace):
self._log(logging.trace, msg, args, **kwargs)
def get_logger(log_handler_name):
"""
Purpose : To create logger .
:param log_handler_name: Name of the log handler.
:return: logger object.
"""
log_path = os.path.join(BASE_LOG_PATH, log_handler_name + ".log")
logging.setLoggerClass(ILensLogger)
_logger = logging.getLogger(log_handler_name)
_logger.setLevel(LOG_LEVEL.strip().upper())
log_handler = logging.StreamHandler()
log_handler.setLevel(LOG_LEVEL)
formatter = logging.Formatter(FORMATTER)
log_handler.setFormatter(formatter)
handler = RotatingFileHandler(log_path, maxBytes=10485760,
backupCount=5)
handler.setFormatter(formatter)
_logger.addHandler(log_handler)
_logger.addHandler(handler)
if ENABLE_LOGSTASH_LOG == 'true' and LOGSTASH_PORT is not None and LOGSTASH_HOST is not None and LOGSTASH_PORT.isdigit():
_logger.addHandler(AsynchronousLogstashHandler(LOGSTASH_HOST, int(LOGSTASH_PORT), database_path=None))
return _logger
logger = get_logger(LOG_HANDLER_NAME)
temp/mnt/ilens/ai-job/pipe1/run.json 0 → 100644
View file @ e1e6add9
{
"run_id" : "run1"
}
\ No newline at end of file
temp/mnt/ilens/ai-job/pipe1/run1/GetDataFromStore/param.json 0 → 100644
View file @ e1e6add9
{
"artifact_name": "response.json"
}
\ No newline at end of file
temp/mnt/ilens/ai-job/pipe1/run1/RunModel/input/data.csv 0 → 100644
View file @ e1e6add9
,Maximum Temperature,Minimum Temperature,Temperature,Precipitation,Wind Speed,Wind Direction,Visibility,Cloud Cover,Relative Humidity,month
2021-03-02,36.0,24.0,28.4,0.0,18.4,126.4,9.8,7.6,71.7,3
2021-03-03,30.4,23.0,26.9,0.0,19.1,68.0,24.1,0.3,73.3,3
2021-03-04,29.8,23.1,26.5,0.1,21.2,54.0,24.1,17.3,72.0,3
2021-03-05,30.6,22.3,26.3,0.0,19.4,66.9,24.1,0.3,71.9,3
2021-03-06,30.5,22.9,26.7,0.0,20.9,51.9,24.1,0.1,73.9,3
2021-03-07,30.4,23.9,27.2,0.09,20.2,59.2,24.1,19.0,75.9,3
2021-03-08,31.3,24.8,27.7,0.35,20.9,52.7,24.1,20.9,75.0,3
2021-03-09,31.5,24.4,27.6,4.46,22.3,58.5,22.6,28.5,75.9,3
2021-03-10,31.9,24.2,27.8,0.14,21.2,66.2,24.1,5.0,76.6,3
2021-03-11,30.8,24.1,27.5,0.0,21.2,56.9,24.1,7.9,75.5,3
2021-03-12,31.2,24.1,27.7,0.51,21.6,65.7,24.1,28.6,74.0,3
2021-03-13,31.0,24.8,27.6,0.71,20.9,55.5,24.1,7.2,76.2,3
2021-03-14,31.9,23.7,27.4,0.0,22.0,64.0,24.1,2.9,74.6,3
2021-03-15,31.1,23.6,27.3,0.0,20.5,60.0,24.1,9.3,75.5,3
2021-03-16,30.6,23.5,27.2,0.0,21.2,59.6,24.1,10.7,72.7,3
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment