Skip to content

G
GetDataFromStore
Commit 9ffe5e99 authored by dasharatha.vamshi's avatar dasharatha.vamshi
Browse files
Options

init

parent 359799d3
Hide whitespace changes
Inline Side-by-side
Showing with 394 additions and 0 deletions
.idea/.gitignore 0 → 100644
View file @ 9ffe5e99
# Default ignored files
/shelf/
/workspace.xml
.idea/getdatafromstore.iml 0 → 100644
View file @ 9ffe5e99
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.7 (welspun-defects) (2)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
.idea/inspectionProfiles/Project_Default.xml 0 → 100644
View file @ 9ffe5e99
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyChainedComparisonsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoreConstantInTheMiddle" value="true" />
</inspection_tool>
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="55">
<item index="0" class="java.lang.String" itemvalue="scikit-image" />
<item index="1" class="java.lang.String" itemvalue="scipy" />
<item index="2" class="java.lang.String" itemvalue="opencv-python" />
<item index="3" class="java.lang.String" itemvalue="torch" />
<item index="4" class="java.lang.String" itemvalue="torchvision" />
<item index="5" class="java.lang.String" itemvalue="absl-py" />
<item index="6" class="java.lang.String" itemvalue="protobuf" />
<item index="7" class="java.lang.String" itemvalue="rsa" />
<item index="8" class="java.lang.String" itemvalue="tensorflow-estimator" />
<item index="9" class="java.lang.String" itemvalue="opt-einsum" />
<item index="10" class="java.lang.String" itemvalue="python-dateutil" />
<item index="11" class="java.lang.String" itemvalue="cycler" />
<item index="12" class="java.lang.String" itemvalue="gast" />
<item index="13" class="java.lang.String" itemvalue="numpy" />
<item index="14" class="java.lang.String" itemvalue="pyasn1" />
<item index="15" class="java.lang.String" itemvalue="requests" />
<item index="16" class="java.lang.String" itemvalue="importlib-metadata" />
<item index="17" class="java.lang.String" itemvalue="pyasn1-modules" />
<item index="18" class="java.lang.String" itemvalue="requests-oauthlib" />
<item index="19" class="java.lang.String" itemvalue="tensorflow" />
<item index="20" class="java.lang.String" itemvalue="tensorboard-plugin-wit" />
<item index="21" class="java.lang.String" itemvalue="zipp" />
<item index="22" class="java.lang.String" itemvalue="oauthlib" />
<item index="23" class="java.lang.String" itemvalue="astunparse" />
<item index="24" class="java.lang.String" itemvalue="urllib3" />
<item index="25" class="java.lang.String" itemvalue="pyparsing" />
<item index="26" class="java.lang.String" itemvalue="Cython" />
<item index="27" class="java.lang.String" itemvalue="Markdown" />
<item index="28" class="java.lang.String" itemvalue="google-auth-oauthlib" />
<item index="29" class="java.lang.String" itemvalue="Werkzeug" />
<item index="30" class="java.lang.String" itemvalue="kiwisolver" />
<item index="31" class="java.lang.String" itemvalue="tqdm" />
<item index="32" class="java.lang.String" itemvalue="yolov5processor" />
<item index="33" class="java.lang.String" itemvalue="tensorboard" />
<item index="34" class="java.lang.String" itemvalue="future" />
<item index="35" class="java.lang.String" itemvalue="matplotlib" />
<item index="36" class="java.lang.String" itemvalue="cachetools" />
<item index="37" class="java.lang.String" itemvalue="grpcio" />
<item index="38" class="java.lang.String" itemvalue="Keras" />
<item index="39" class="java.lang.String" itemvalue="google-auth" />
<item index="40" class="java.lang.String" itemvalue="idna" />
<item index="41" class="java.lang.String" itemvalue="Pillow" />
<item index="42" class="java.lang.String" itemvalue="cython" />
<item index="43" class="java.lang.String" itemvalue="keras" />
<item index="44" class="java.lang.String" itemvalue="imgaug" />
<item index="45" class="java.lang.String" itemvalue="opencv-contrib-python" />
<item index="46" class="java.lang.String" itemvalue="paho-mqtt" />
<item index="47" class="java.lang.String" itemvalue="pymongo" />
<item index="48" class="java.lang.String" itemvalue="fbprophet" />
<item index="49" class="java.lang.String" itemvalue="scikit-learn" />
<item index="50" class="java.lang.String" itemvalue="sklearn" />
<item index="51" class="java.lang.String" itemvalue="statsmodels" />
<item index="52" class="java.lang.String" itemvalue="python-dotenv" />
<item index="53" class="java.lang.String" itemvalue="pystan" />
<item index="54" class="java.lang.String" itemvalue="Cerberus" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N802" />
<option value="N806" />
<option value="N801" />
<option value="N803" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="cv2.imresize" />
<option value="detectron2.model_zoo.get_config_file" />
<option value="detectron2.model_zoo.get_checkpoint_url" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
View file @ 9ffe5e99
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
.idea/misc.xml 0 → 100644
View file @ 9ffe5e99
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (welspun-defects) (2)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
.idea/modules.xml 0 → 100644
View file @ 9ffe5e99
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/getdatafromstore.iml" filepath="$PROJECT_DIR$/.idea/getdatafromstore.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/vcs.xml 0 → 100644
View file @ 9ffe5e99
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
Dockerfile 0 → 100644
View file @ 9ffe5e99
FROM python:3.7-slim
ADD . /opt
WORKDIR /opt
RUN pip install -r requirements.txt
CMD python main.py
conf/configuration.yml 0 → 100644
View file @ 9ffe5e99
#---------------Service Configurations----------------#
SERVICE_CONFIG:
LOG_LEVEL: info
LOG_HANDLER_NAME: GetDataFromStore
LOGSTASH_HOST: 192.168.1.47
LOGSTASH_PORT: 5000
#--------------System Configurations--------------------#
SYSTEM_CONFIG:
shared_mount_base_ai_job: temp/mnt/ilens/ai-job/
connection_string: DefaultEndpointsProtocol=https;AccountName=azrabsilensqa01;AccountKey=DN6q6kX98JM8yUwtuJh2bAaXUGFo1zRS5HJSsa/ZA+MlmctjC000eHP7bdXiQqkI/MVtADhS8c9E88LI5T4UHw==;EndpointSuffix=core.windows.net
#----------------------If read conf from mongo------------#
FOR_EACH_MONGO_CONFIG:
READ_FROM_MONGO: true
MONGO_URI: mongodb://192.168.0.210:27017
MONGO_DB: iLensAiPipeline
MONGO_RUN_COLL: runMetadata
MONGO_SITE_COLL: siteMetadata
logs/GetDataFromStore.log 0 → 100644
View file @ 9ffe5e99
2021-03-02 17:36:37,055 INFO GetDataFromStore {'pipeline_id': 'pipe1', 'run_id': 'run1', 'shared_mount_base_ai_job': 'temp/mnt/ilens/ai-job/', 'data_store': 'Azure', 'container_name': 'ilensqa', 'artifact_base_path': '/data/model/tested/', 'artifact_name': 'test1.pkl', 'connection_string': 'DefaultEndpointsProtocol=https;AccountName=azrabsilensqa01;AccountKey=DN6q6kX98JM8yUwtuJh2bAaXUGFo1zRS5HJSsa/ZA+MlmctjC000eHP7bdXiQqkI/MVtADhS8c9E88LI5T4UHw==;EndpointSuffix=core.windows.net', 'component_output_dir': 'temp/mnt/ilens/ai-job//pipe1/run1/PreprocessWeatherData/input'}
2021-03-02 17:36:37,057 INFO GetDataFromStore Creating Connection........
2021-03-02 17:36:39,368 INFO GetDataFromStore Started downloading file to path temp/mnt/ilens/ai-job//pipe1/run1/PreprocessWeatherData/input\test1.pkl from path /data/model/tested/test1.pkl on azure
2021-03-02 17:36:39,368 INFO GetDataFromStore File Downloaded Successfully
2021-03-02 17:39:37,849 INFO GetDataFromStore {'pipeline_id': 'pipe1', 'run_id': 'run1', 'shared_mount_base_ai_job': 'temp/mnt/ilens/ai-job/', 'data_store': 'Azure', 'container_name': 'ilensqa', 'artifact_base_path': '/data/model/tested/', 'artifact_name': 'test1.pkl', 'connection_string': 'DefaultEndpointsProtocol=https;AccountName=azrabsilensqa01;AccountKey=DN6q6kX98JM8yUwtuJh2bAaXUGFo1zRS5HJSsa/ZA+MlmctjC000eHP7bdXiQqkI/MVtADhS8c9E88LI5T4UHw==;EndpointSuffix=core.windows.net', 'component_output_dir': 'temp/mnt/ilens/ai-job//pipe1/run1/PreprocessWeatherData/input'}
2021-03-02 17:39:37,850 INFO GetDataFromStore Creating Connection........
2021-03-02 17:39:40,100 INFO GetDataFromStore Started downloading file to path temp/mnt/ilens/ai-job//pipe1/run1/PreprocessWeatherData/input\test1.pkl from path /data/model/tested/test1.pkl on azure
2021-03-02 17:39:40,101 INFO GetDataFromStore File Downloaded Successfully
main.py 0 → 100644
View file @ 9ffe5e99
import traceback
from azure.storage.blob import BlobServiceClient
from scripts.common.config_parser import *
from scripts.common.constants import GetDataFromStoreConstants, ComponentExceptions
from scripts.common.logsetup import logger
class AzureDownload:
def __init__(self,query):
self.query = query
self.container = self.query[GetDataFromStoreConstants.CONTAINER_NAME]
self.connection_string = self.query['connection_string']
self.blob_service_client = BlobServiceClient.from_connection_string(self.connection_string)
self.container_client = self.blob_service_client.get_container_client(self.container)
def download_from_blob(self):
try:
logger.info("Creating Connection........")
blob_client = self.blob_service_client.get_blob_client(container=self.container,
blob=self.query[
GetDataFromStoreConstants.AZURE_FILE_PATH])
with open(self.query[GetDataFromStoreConstants.LOCAL_FILE_PATH], "wb") as download_file:
download_file.write(blob_client.download_blob().readall())
logger.info(
"Started downloading file to path " + self.query[GetDataFromStoreConstants.LOCAL_FILE_PATH] + " from path " +
self.query[GetDataFromStoreConstants.AZURE_FILE_PATH] + " on azure")
return True
except Exception as e:
raise Exception(e)
if __name__ == '__main__':
try:
if GetDataFromStoreConstants.ARTIFACT_BASE_PATH in config.keys():
azure_file_path = config[GetDataFromStoreConstants.ARTIFACT_BASE_PATH]
else:
raise Exception(ComponentExceptions.INVALID_ARTIFACT_BASE_PATH_EXCEPTION)
if GetDataFromStoreConstants.ARTIFACT_NAME in config.keys():
azure_file_name = config[GetDataFromStoreConstants.ARTIFACT_NAME]
else:
raise Exception(ComponentExceptions.INVALID_AZURE_FILE_NAME_EXCEPTION)
if GetDataFromStoreConstants.CONTAINER_NAME in config.keys():
azure_container_name = config[GetDataFromStoreConstants.CONTAINER_NAME]
else:
raise Exception(ComponentExceptions.INVALID_CONTAINER_NAME)
logger.info(config)
mydict = {
"azure_file_path": os.path.join(azure_file_path, azure_file_name),
"local_file_path": os.path.join(config['component_output_dir'], azure_file_name),
"container_name": config['container_name'],
"connection_string": config['connection_string']
}
obj = AzureDownload(mydict)
val = obj.download_from_blob()
if val:
logger.info("File Downloaded Successfully")
except:
logger.info("Azure File Download Component Failed")
logger.info(traceback.format_exc())
scripts/common/config_parser.py 0 → 100644
View file @ 9ffe5e99
#!/usr/bin/env python
import os
import sys
import yaml
import json
from pymongo import MongoClient, DESCENDING
from scripts.common.constants import GetDataFromStoreConstants
config_path = os.path.join(os.getcwd(), "conf", "configuration.yml")
if os.path.exists(config_path):
sys.stderr.write("Reading config from --> {}".format(config_path))
sys.stderr.write("\n")
with open(config_path, 'r') as stream:
_config = yaml.safe_load(stream)
else:
sys.stderr.write("Configuration not found...")
sys.stderr.write("Exiting....")
sys.exit(1)
# os.environ['pipeline_id'] = "pipe1"
# os.environ['component_output_dir'] = "test"
# os.environ['artifact_base_path'] = "/data/model/tested/"
# os.environ['artifact_name'] = "test1.pkl"
# ------------------------ Configurations -----------------------------------------------------------------------------
pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job",
_config.get("SYSTEM_CONFIG", {}).get('shared_mount_base_ai_job'))
# read from $shared_mount_base_ai_job/$pipeline_id/run.config
run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run.json"
try:
sys.stderr.write("Checking for run id parameters at path " + run_id_path + "\n")
with open(run_id_path) as f:
run_id_param = json.load(f)
except Exception as e:
raise Exception(e)
run_id = run_id_param['run_id']
data_store = os.environ.get('data_store', default="Azure")
container_name = os.environ.get('container_name', default=GetDataFromStoreConstants.CONTAINER)
connection_string = os.environ.get('connection_string', _config.get("SYSTEM_CONFIG", {}).get('connection_string'))
artifact_base_path = os.environ.get('artifact_base_path')
# Component Parameter:Read from $shared_mount_base_ai_job/$pipeline_id/$run_id/GetDataFromStore/param.json
component_parameter_path = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + GetDataFromStoreConstants.COMPONENT_NAME + "/param.json"
try:
sys.stderr.write("Checking for component parameters at path " + component_parameter_path + "\n")
with open(component_parameter_path) as f:
component_parameter = json.load(f)
except Exception as e:
raise Exception(e)
artifact_name = component_parameter['artifact_name']
# $shared_mount_base_ai_job/$pipeline_id/$run_id/PreprocessWeatherData/input/$artifact_Name
component_output_dir = shared_mount_base_ai_job + "/" + pipeline_id + "/" + run_id + "/" + GetDataFromStoreConstants.NEXT_COMPONENT + "/input"
BASE_LOG_PATH = os.path.join(os.getcwd(), "logs")
if not os.path.exists(os.path.join(os.getcwd(), 'logs')):
os.mkdir(os.path.join(os.getcwd(), 'logs'))
LOG_LEVEL = os.environ.get("LOG_LEVEL", _config.get('SERVICE_CONFIG', {}).get("LOG_LEVEL", "INFO")).upper()
LOG_HANDLER_NAME = _config.get('SERVICE_CONFIG', {}).get("LOG_HANDLER_NAME", "GetDataFromStore")
ENABLE_LOGSTASH_LOG = os.environ.get("ENABLE_LOGSTASH_LOG", 'False').lower()
LOGSTASH_HOST = _config.get('SERVICE_CONFIG', {}).get('LOGSTASH_HOST')
LOGSTASH_PORT = str(_config.get('SERVICE_CONFIG', {}).get('LOGSTASH_PORT'))
# os.environ['azure_file_path'] = '/data/model/tested/test1.pkl'
# os.environ['local_file_path'] = r'E:\iLens-AI\azure-file-download\StandardScaler.pkl'
config = {
'pipeline_id': pipeline_id,
'run_id': run_id,
'shared_mount_base_ai_job': shared_mount_base_ai_job,
'data_store': data_store,
'container_name': container_name,
'artifact_base_path': artifact_base_path,
'artifact_name': artifact_name,
'connection_string': connection_string,
'component_output_dir': component_output_dir,
# 'run_id_path': run_id_path,
# 'component_parameter_path': component_parameter_path
}
if not os.path.exists(config['shared_mount_base_ai_job']):
sys.stderr.write("Shared path does not exist!" + "\n")
sys.stderr.write("Creating path --> {}".format(config['shared_mount_base_ai_job'] + "\n"))
os.makedirs(config['shared_mount_base_ai_job'])
if not os.path.exists(config['component_output_dir']):
sys.stderr.write("component_output_dir does not exist!" + "\n")
sys.stderr.write("Creating path --> {}".format(config['component_output_dir'] + "\n"))
os.makedirs(config['component_output_dir'])
scripts/common/constants.py 0 → 100644
View file @ 9ffe5e99
#!/usr/bin/env python
class GetDataFromStoreConstants:
ARTIFACT_BASE_PATH = "artifact_base_path"
ARTIFACT_NAME = "artifact_name"
CONTAINER_NAME = "container_name"
NEXT_COMPONENT = "PreprocessWeatherData"
CONTAINER = "ilensqa"
CONNECTION_STRING = "DefaultEndpointsProtocol=https;AccountName=azrabsilensqa01;AccountKey=DN6q6kX98JM8yUwtuJh2bAaXUGFo1zRS5HJSsa/ZA+MlmctjC000eHP7bdXiQqkI/MVtADhS8c9E88LI5T4UHw==;EndpointSuffix=core.windows.net"
COMPONENT_NAME = "GetDataFromStore"
AZURE_FILE_PATH = "azure_file_path"
LOCAL_FILE_PATH = "local_file_path"
HTTP = "http://"
LOG_VAR_MESSAGE = "\n" + "#" * 25 + "\n" + "{}" + "\n" + "#" * 25 + "\n" + "{}\n"
class ComponentExceptions:
INVALID_AZURE_FILE_PATH_EXCEPTION = "AZURE PATH ERROR"
INVALID_LOCAL_FILE_PATH_EXCEPTION = "No File in the local path"
INVALID_ARTIFACT_BASE_PATH_EXCEPTION = "Artifact base path value is missing"
INVALID_AZURE_FILE_NAME_EXCEPTION = "Artifact name is missing"
INVALID_CONTAINER_NAME = "Container name is missing"
scripts/common/logsetup.py 0 → 100644
View file @ 9ffe5e99
import os
import logging
from logging.handlers import RotatingFileHandler
from logstash_async.handler import AsynchronousLogstashHandler
from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH
from scripts.common.config_parser import LOG_LEVEL, LOG_HANDLER_NAME, BASE_LOG_PATH, LOGSTASH_HOST, LOGSTASH_PORT, ENABLE_LOGSTASH_LOG
DEFAULT_FORMAT = '%(asctime)s %(levelname)5s %(name)s %(message)s'
DEBUG_FORMAT = '%(asctime)s %(levelname)5s %(name)s [%(threadName)5s:%(filename)5s:%(funcName)5s():%(lineno)s] %(' \
'message)s '
EXTRA = {}
FORMATTER = DEFAULT_FORMAT
if LOG_LEVEL.strip() == "DEBUG":
FORMATTER = DEBUG_FORMAT
logging.trace = logging.DEBUG - 5
logging.addLevelName(logging.DEBUG - 5, 'TRACE')
class ILensLogger(logging.getLoggerClass()):
def __init__(self, name):
super().__init__(name)
def trace(self, msg, *args, **kwargs):
if self.isEnabledFor(logging.trace):
self._log(logging.trace, msg, args, **kwargs)
def get_logger(log_handler_name):
"""
Purpose : To create logger .
:param log_handler_name: Name of the log handler.
:return: logger object.
"""
log_path = os.path.join(BASE_LOG_PATH, log_handler_name + ".log")
logging.setLoggerClass(ILensLogger)
_logger = logging.getLogger(log_handler_name)
_logger.setLevel(LOG_LEVEL.strip().upper())
log_handler = logging.StreamHandler()
log_handler.setLevel(LOG_LEVEL)
formatter = logging.Formatter(FORMATTER)
log_handler.setFormatter(formatter)
handler = RotatingFileHandler(log_path, maxBytes=10485760,
backupCount=5)
handler.setFormatter(formatter)
_logger.addHandler(log_handler)
_logger.addHandler(handler)
if ENABLE_LOGSTASH_LOG == 'true' and LOGSTASH_PORT is not None and LOGSTASH_HOST is not None and LOGSTASH_PORT.isdigit():
_logger.addHandler(AsynchronousLogstashHandler(LOGSTASH_HOST, int(LOGSTASH_PORT), database_path=None))
return _logger
logger = get_logger(LOG_HANDLER_NAME)
temp/mnt/ilens/ai-job/pipe1/run.json 0 → 100644
View file @ 9ffe5e99
{
"run_id" : "run1"
}
\ No newline at end of file
temp/mnt/ilens/ai-job/pipe1/run1/GetDataFromStore/param.json 0 → 100644
View file @ 9ffe5e99
{
"artifact_name": "test1.pkl"
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment