Commit e1e6add9 authored by dasharatha.vamshi's avatar dasharatha.vamshi

init

parent 03121e35
# Default ignored files
/shelf/
/workspace.xml
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyChainedComparisonsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoreConstantInTheMiddle" value="true" />
</inspection_tool>
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="55">
<item index="0" class="java.lang.String" itemvalue="scikit-image" />
<item index="1" class="java.lang.String" itemvalue="scipy" />
<item index="2" class="java.lang.String" itemvalue="opencv-python" />
<item index="3" class="java.lang.String" itemvalue="torch" />
<item index="4" class="java.lang.String" itemvalue="torchvision" />
<item index="5" class="java.lang.String" itemvalue="absl-py" />
<item index="6" class="java.lang.String" itemvalue="protobuf" />
<item index="7" class="java.lang.String" itemvalue="rsa" />
<item index="8" class="java.lang.String" itemvalue="tensorflow-estimator" />
<item index="9" class="java.lang.String" itemvalue="opt-einsum" />
<item index="10" class="java.lang.String" itemvalue="python-dateutil" />
<item index="11" class="java.lang.String" itemvalue="cycler" />
<item index="12" class="java.lang.String" itemvalue="gast" />
<item index="13" class="java.lang.String" itemvalue="numpy" />
<item index="14" class="java.lang.String" itemvalue="pyasn1" />
<item index="15" class="java.lang.String" itemvalue="requests" />
<item index="16" class="java.lang.String" itemvalue="importlib-metadata" />
<item index="17" class="java.lang.String" itemvalue="pyasn1-modules" />
<item index="18" class="java.lang.String" itemvalue="requests-oauthlib" />
<item index="19" class="java.lang.String" itemvalue="tensorflow" />
<item index="20" class="java.lang.String" itemvalue="tensorboard-plugin-wit" />
<item index="21" class="java.lang.String" itemvalue="zipp" />
<item index="22" class="java.lang.String" itemvalue="oauthlib" />
<item index="23" class="java.lang.String" itemvalue="astunparse" />
<item index="24" class="java.lang.String" itemvalue="urllib3" />
<item index="25" class="java.lang.String" itemvalue="pyparsing" />
<item index="26" class="java.lang.String" itemvalue="Cython" />
<item index="27" class="java.lang.String" itemvalue="Markdown" />
<item index="28" class="java.lang.String" itemvalue="google-auth-oauthlib" />
<item index="29" class="java.lang.String" itemvalue="Werkzeug" />
<item index="30" class="java.lang.String" itemvalue="kiwisolver" />
<item index="31" class="java.lang.String" itemvalue="tqdm" />
<item index="32" class="java.lang.String" itemvalue="yolov5processor" />
<item index="33" class="java.lang.String" itemvalue="tensorboard" />
<item index="34" class="java.lang.String" itemvalue="future" />
<item index="35" class="java.lang.String" itemvalue="matplotlib" />
<item index="36" class="java.lang.String" itemvalue="cachetools" />
<item index="37" class="java.lang.String" itemvalue="grpcio" />
<item index="38" class="java.lang.String" itemvalue="Keras" />
<item index="39" class="java.lang.String" itemvalue="google-auth" />
<item index="40" class="java.lang.String" itemvalue="idna" />
<item index="41" class="java.lang.String" itemvalue="Pillow" />
<item index="42" class="java.lang.String" itemvalue="cython" />
<item index="43" class="java.lang.String" itemvalue="keras" />
<item index="44" class="java.lang.String" itemvalue="imgaug" />
<item index="45" class="java.lang.String" itemvalue="opencv-contrib-python" />
<item index="46" class="java.lang.String" itemvalue="paho-mqtt" />
<item index="47" class="java.lang.String" itemvalue="pymongo" />
<item index="48" class="java.lang.String" itemvalue="fbprophet" />
<item index="49" class="java.lang.String" itemvalue="scikit-learn" />
<item index="50" class="java.lang.String" itemvalue="sklearn" />
<item index="51" class="java.lang.String" itemvalue="statsmodels" />
<item index="52" class="java.lang.String" itemvalue="python-dotenv" />
<item index="53" class="java.lang.String" itemvalue="pystan" />
<item index="54" class="java.lang.String" itemvalue="Cerberus" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N802" />
<option value="N806" />
<option value="N801" />
<option value="N803" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="cv2.imresize" />
<option value="detectron2.model_zoo.get_config_file" />
<option value="detectron2.model_zoo.get_checkpoint_url" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (welspun-defects) (2)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/preprocessweatherdata.iml" filepath="$PROJECT_DIR$/.idea/preprocessweatherdata.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.7 (welspun-defects) (2)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="pytest" />
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
FROM python:3.7-slim
ADD . /opt
WORKDIR /opt
RUN pip install -r requirements.txt
CMD python main.py
#---------------Service Configurations----------------#
SERVICE_CONFIG:
LOG_LEVEL: info
LOG_HANDLER_NAME: PreprocessWeatherData
LOGSTASH_HOST: 192.168.1.47
LOGSTASH_PORT: 5000
#--------------System Configurations--------------------#
SYSTEM_CONFIG:
shared_mount_base_ai_job: /mnt/ilens/ai-job/
#----------------------If read conf from mongo------------#
FOR_EACH_MONGO_CONFIG:
READ_FROM_MONGO: true
MONGO_URI: mongodb://192.168.0.210:27017
MONGO_DB: iLensAiPipeline
MONGO_RUN_COLL: runMetadata
MONGO_SITE_COLL: siteMetadata
2021-03-02 19:09:57,986 INFO PreprocessWeatherData Traceback (most recent call last):
File "E:/iLens-AI/preprocessweatherdata/main.py", line 62, in <module>
json_path = os.path.join(json_path_dir,os.listdir(json_path_dir)[0])
FileNotFoundError: [WinError 3] The system cannot find the path specified: 'temp/mnt/ilens/ai-job//pipe1/run1PreprocessWeatherData/input'
2021-03-02 19:11:35,286 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:11:35,301 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input/data.csv
2021-03-02 19:13:19,263 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:13:19,272 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input/data.csv
2021-03-02 19:14:37,570 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:14:37,580 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:14:37,584 INFO PreprocessWeatherData Component executed Successfully
2021-03-02 19:15:18,736 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:15:18,745 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:15:18,749 INFO PreprocessWeatherData Component executed Successfully
2021-03-02 19:17:47,686 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:17:47,696 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:17:47,701 INFO PreprocessWeatherData Component executed Successfully
import json
import pickle
import traceback
import os
import pandas as pd
from sklearn.preprocessing import StandardScaler
from scripts.common.config_parser import *
from scripts.common.constants import PreprocessWeatherDataConstants, ComponentExceptions
from scripts.common.logsetup import logger
class PreProcessComponent:
def __init__(self):
self.ds, self.tempmax, self.tempmin, self.temp, self.humidity, self.precip, self.winddir, self.cloudcover, self.visibility, self.date, self.windspeed = (
[] for i in range(11))
def read_pickle_file(self, file):
try:
logger.info("Parsing the pickle file at location " + file + " ......")
return pickle.load(open(file, "rb"))
except:
logger.info("Failed to read pickle file")
logger.info("----Traceback----")
logger.info(traceback.format_exc())
def preprocess(self, forcast_json, null_value_method):
try:
logger.info("Reading Json file")
with open(forcast_json) as f:
data = json.load(f)
except:
logger.info("Failed reading Json File")
logger.info(traceback.format_exc())
for i in data[PreprocessWeatherDataConstants.DAYS_KEY]:
self.date.append(i[PreprocessWeatherDataConstants.DATETIME_KEY])
self.ds.append(int(i[PreprocessWeatherDataConstants.DATETIME_KEY].split("-")[1]))
self.tempmax.append(i[PreprocessWeatherDataConstants.TEMPMAX_KEY])
self.tempmin.append(i[PreprocessWeatherDataConstants.TEMPMIN_KEY])
self.temp.append(i[PreprocessWeatherDataConstants.TEMP_KEY])
self.humidity.append(i[PreprocessWeatherDataConstants.HUMIDITY_KEY])
self.precip.append(i[PreprocessWeatherDataConstants.PRECIP_KEY])
self.windspeed.append(i[PreprocessWeatherDataConstants.WINDSPEED_KEY])
self.winddir.append(i[PreprocessWeatherDataConstants.WINDDIR_KEY])
self.cloudcover.append(i[PreprocessWeatherDataConstants.CLOUDCOVER_KEY])
self.visibility.append(i[PreprocessWeatherDataConstants.VISIBILITY_KEY])
forcast_data = pd.DataFrame(
{"Maximum Temperature": self.tempmax, "Minimum Temperature": self.tempmin, "Temperature": self.temp,
"Precipitation": self.precip, "Wind Speed": self.windspeed, "Wind Direction": self.winddir,
"Visibility": self.visibility, "Cloud Cover": self.cloudcover,
"Relative Humidity": self.humidity, "month": self.ds})
forcast_data.fillna(method=null_value_method, inplace=True)
forcast_data.index = self.date
return forcast_data
if __name__ == '__main__':
try:
# Checking json path
json_path_dir = config[PreprocessWeatherDataConstants.COMPONENT_INPUT_DIR]
json_path = os.path.join(json_path_dir, os.listdir(json_path_dir)[0])
except Exception as e:
logger.info(traceback.format_exc())
raise Exception(e)
# Checking Fill NA method
if PreprocessWeatherDataConstants.FILLNA_METHOD in config.keys():
fillna_method = config[PreprocessWeatherDataConstants.FILLNA_METHOD]
else:
raise Exception(ComponentExceptions.INVALID_Fillna_Method)
obj = PreProcessComponent()
forcast_df = obj.preprocess(json_path, fillna_method)
try:
logger.info("Writing forcast data to csv file at " + os.path.join(config['component_output_dir'], 'data.csv'))
output2 = os.path.join(config['component_output_dir'], 'data.csv')
forcast_df.to_csv(output2, index=True)
logger.info("Component executed Successfully")
except Exception as e:
raise Exception(e)
#!/usr/bin/env python
import os
import sys
import yaml
import json
from pymongo import MongoClient, DESCENDING
from scripts.common.constants import PreprocessWeatherDataConstants
config_path = os.path.join(os.getcwd(), "conf", "configuration.yml")
if os.path.exists(config_path):
sys.stderr.write("Reading config from --> {}".format(config_path))
sys.stderr.write("\n")
with open(config_path, 'r') as stream:
_config = yaml.safe_load(stream)
else:
sys.stderr.write("Configuration not found...")
sys.stderr.write("Exiting....")
sys.exit(1)
# For testing uncomment this
# os.environ['pipeline_id'] = "pipe1"
# ------------------------ Configurations -----------------------------------------------------------------------------
pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job",
_config.get("SYSTEM_CONFIG", {}).get('shared_mount_base_ai_job'))
# read from $shared_mount_base_ai_job/$pipeline_id/run.config
run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run.json"
try:
sys.stderr.write("Checking for run id parameters at path " + run_id_path + "\n")
with open(run_id_path) as f:
run_id_param = json.load(f)
except Exception as e:
raise Exception(e)
run_id = run_id_param['run_id']
fillna_method = os.environ.get("fillna_method", default="ffill")
# shared_mount_base_ai_job/$pipeline_id/$run_id/PreprocessWeatherData/input/$.json
component_input = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + PreprocessWeatherDataConstants.COMPONENT_NAME + "/input"
# shared_mount_base_ai_job/$pipeline_id/$run_id/RunModel/input/data.csv
component_output_dir = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + PreprocessWeatherDataConstants.NEXT_COMPONENT + "/input"
BASE_LOG_PATH = os.path.join(os.getcwd(), "logs")
if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
os.mkdir(os.path.join(os.getcwd(), 'logs'))
LOG_LEVEL = os.environ.get("LOG_LEVEL", _config.get('SERVICE_CONFIG', {}).get("LOG_LEVEL", "INFO")).upper()
LOG_HANDLER_NAME = _config.get('SERVICE_CONFIG', {}).get("LOG_HANDLER_NAME", "PreprocessWeatherData")
ENABLE_LOGSTASH_LOG = os.environ.get("ENABLE_LOGSTASH_LOG", 'False').lower()
LOGSTASH_HOST = _config.get('SERVICE_CONFIG', {}).get('LOGSTASH_HOST')
LOGSTASH_PORT = str(_config.get('SERVICE_CONFIG', {}).get('LOGSTASH_PORT'))
# os.environ["shared_volume"] = "test"
# os.environ["json_path"] = r"E:\welspun-defects\preprocess_data-ilendev\response.json"
# os.environ["fillna_method"] = "ffill"
# os.environ["standard_scalar_path"] = r"E:\welspun-defects\preprocess_data-ilendev\StandardScaler.pkl"
# print(os.getcwd())
config = {
'pipeline_id': pipeline_id,
'run_id': run_id,
'shared_mount_base_ai_job': shared_mount_base_ai_job,
'fillna_method': fillna_method,
'component_input': component_input,
'component_output_dir': component_output_dir
}
if not os.path.exists(config['shared_mount_base_ai_job']):
sys.stderr.write("Shared path does not exist!" + "\n")
sys.stderr.write("Creating path --> {}".format(config['shared_mount_base_ai_job'] + "\n"))
os.makedirs(config['shared_mount_base_ai_job'])
if not os.path.exists(config['component_output_dir']):
sys.stderr.write("component_output_dir does not exist!" + "\n")
sys.stderr.write("Creating path --> {}".format(config['component_output_dir'] + "\n"))
os.makedirs(config['component_output_dir'])
#!/usr/bin/env python
class PreprocessWeatherDataConstants:
COMPONENT_INPUT_DIR = "component_input"
COMPONENT_NAME = "PreprocessWeatherData"
NEXT_COMPONENT = "RunModel"
SHARED_VOLUME = "shared_volume"
JSON_PATH = "json_path"
FILLNA_METHOD = "fillna_method"
STANDARD_SCALAR_PATH = "standard_scalar_path"
TEMPMAX_KEY = "tempmax"
TEMPMIN_KEY = "tempmin"
TEMP_KEY = "temp"
HUMIDITY_KEY = "humidity"
PRECIP_KEY = "precip"
WINDSPEED_KEY = "windspeed"
WINDDIR_KEY = "winddir"
CLOUDCOVER_KEY = "cloudcover"
VISIBILITY_KEY = "visibility"
DAYS_KEY = "days"
DATETIME_KEY = "datetime"
REQUEST_SUCCESS_CODE = 200
ERRORS_KEY = "errors"
Default_headers = {'content-type': 'application/json'}
HTTP = "http://"
LOG_VAR_MESSAGE = "\n" + "#" * 25 + "\n" + "{}" + "\n" + "#" * 25 + "\n" + "{}\n"
class ComponentExceptions:
INVALID_SHARED_VOLUME = "Shared Volume is required"
INVALID_JSON_PATH = "Path to Json file is required"
INVALID_Standard_Scalar_Pkl_Path = "Path to Standard Scalar Pkl file is needed"
INVALID_Fillna_Method = "Method to fill NA values is required"
import os
import logging
from logging.handlers import RotatingFileHandler
from logstash_async.handler import AsynchronousLogstashHandler
from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH
from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH, LOGSTASH_HOST, LOGSTASH_PORT, ENABLE_LOGSTASH_LOG
DEFAULT_FORMAT = '%(asctime)s %(levelname)5s %(name)s %(message)s'
DEBUG_FORMAT = '%(asctime)s %(levelname)5s %(name)s [%(threadName)5s:%(filename)5s:%(funcName)5s():%(lineno)s] %(' \
'message)s '
EXTRA = {}
FORMATTER = DEFAULT_FORMAT
if LOG_LEVEL.strip() == "DEBUG":
FORMATTER = DEBUG_FORMAT
logging.trace = logging.DEBUG - 5
logging.addLevelName(logging.DEBUG - 5, 'TRACE')
class ILensLogger(logging.getLoggerClass()):
def __init__(self, name):
super().__init__(name)
def trace(self, msg, *args, **kwargs):
if self.isEnabledFor(logging.trace):
self._log(logging.trace, msg, args, **kwargs)
def get_logger(log_handler_name):
"""
Purpose : To create logger .
:param log_handler_name: Name of the log handler.
:return: logger object.
"""
log_path = os.path.join(BASE_LOG_PATH, log_handler_name + ".log")
logging.setLoggerClass(ILensLogger)
_logger = logging.getLogger(log_handler_name)
_logger.setLevel(LOG_LEVEL.strip().upper())
log_handler = logging.StreamHandler()
log_handler.setLevel(LOG_LEVEL)
formatter = logging.Formatter(FORMATTER)
log_handler.setFormatter(formatter)
handler = RotatingFileHandler(log_path, maxBytes=10485760,
backupCount=5)
handler.setFormatter(formatter)
_logger.addHandler(log_handler)
_logger.addHandler(handler)
if ENABLE_LOGSTASH_LOG == 'true' and LOGSTASH_PORT is not None and LOGSTASH_HOST is not None and LOGSTASH_PORT.isdigit():
_logger.addHandler(AsynchronousLogstashHandler(LOGSTASH_HOST, int(LOGSTASH_PORT), database_path=None))
return _logger
logger = get_logger(LOG_HANDLER_NAME)
{
"run_id" : "run1"
}
\ No newline at end of file
{
"artifact_name": "response.json"
}
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
,Maximum Temperature,Minimum Temperature,Temperature,Precipitation,Wind Speed,Wind Direction,Visibility,Cloud Cover,Relative Humidity,month
2021-03-02,36.0,24.0,28.4,0.0,18.4,126.4,9.8,7.6,71.7,3
2021-03-03,30.4,23.0,26.9,0.0,19.1,68.0,24.1,0.3,73.3,3
2021-03-04,29.8,23.1,26.5,0.1,21.2,54.0,24.1,17.3,72.0,3
2021-03-05,30.6,22.3,26.3,0.0,19.4,66.9,24.1,0.3,71.9,3
2021-03-06,30.5,22.9,26.7,0.0,20.9,51.9,24.1,0.1,73.9,3
2021-03-07,30.4,23.9,27.2,0.09,20.2,59.2,24.1,19.0,75.9,3
2021-03-08,31.3,24.8,27.7,0.35,20.9,52.7,24.1,20.9,75.0,3
2021-03-09,31.5,24.4,27.6,4.46,22.3,58.5,22.6,28.5,75.9,3
2021-03-10,31.9,24.2,27.8,0.14,21.2,66.2,24.1,5.0,76.6,3
2021-03-11,30.8,24.1,27.5,0.0,21.2,56.9,24.1,7.9,75.5,3
2021-03-12,31.2,24.1,27.7,0.51,21.6,65.7,24.1,28.6,74.0,3
2021-03-13,31.0,24.8,27.6,0.71,20.9,55.5,24.1,7.2,76.2,3
2021-03-14,31.9,23.7,27.4,0.0,22.0,64.0,24.1,2.9,74.6,3
2021-03-15,31.1,23.6,27.3,0.0,20.5,60.0,24.1,9.3,75.5,3
2021-03-16,30.6,23.5,27.2,0.0,21.2,59.6,24.1,10.7,72.7,3
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment