Skip to content

P
PreprocessWeatherData
Commit b2488cf4 authored by dasharatha.vamshi's avatar dasharatha.vamshi
Browse files
Options

git ignore

parent 5c2d105f
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 145 additions and 10194 deletions
.gitignore 0 → 100644
View file @ b2488cf4
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# pycharm
.idea/
# logs
logs/
\ No newline at end of file
.idea/.gitignore deleted 100644 → 0
View file @ 5c2d105f
# Default ignored files
/shelf/
/workspace.xml
.idea/inspectionProfiles/Project_Default.xml deleted 100644 → 0
View file @ 5c2d105f
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyChainedComparisonsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoreConstantInTheMiddle" value="true" />
</inspection_tool>
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="55">
<item index="0" class="java.lang.String" itemvalue="scikit-image" />
<item index="1" class="java.lang.String" itemvalue="scipy" />
<item index="2" class="java.lang.String" itemvalue="opencv-python" />
<item index="3" class="java.lang.String" itemvalue="torch" />
<item index="4" class="java.lang.String" itemvalue="torchvision" />
<item index="5" class="java.lang.String" itemvalue="absl-py" />
<item index="6" class="java.lang.String" itemvalue="protobuf" />
<item index="7" class="java.lang.String" itemvalue="rsa" />
<item index="8" class="java.lang.String" itemvalue="tensorflow-estimator" />
<item index="9" class="java.lang.String" itemvalue="opt-einsum" />
<item index="10" class="java.lang.String" itemvalue="python-dateutil" />
<item index="11" class="java.lang.String" itemvalue="cycler" />
<item index="12" class="java.lang.String" itemvalue="gast" />
<item index="13" class="java.lang.String" itemvalue="numpy" />
<item index="14" class="java.lang.String" itemvalue="pyasn1" />
<item index="15" class="java.lang.String" itemvalue="requests" />
<item index="16" class="java.lang.String" itemvalue="importlib-metadata" />
<item index="17" class="java.lang.String" itemvalue="pyasn1-modules" />
<item index="18" class="java.lang.String" itemvalue="requests-oauthlib" />
<item index="19" class="java.lang.String" itemvalue="tensorflow" />
<item index="20" class="java.lang.String" itemvalue="tensorboard-plugin-wit" />
<item index="21" class="java.lang.String" itemvalue="zipp" />
<item index="22" class="java.lang.String" itemvalue="oauthlib" />
<item index="23" class="java.lang.String" itemvalue="astunparse" />
<item index="24" class="java.lang.String" itemvalue="urllib3" />
<item index="25" class="java.lang.String" itemvalue="pyparsing" />
<item index="26" class="java.lang.String" itemvalue="Cython" />
<item index="27" class="java.lang.String" itemvalue="Markdown" />
<item index="28" class="java.lang.String" itemvalue="google-auth-oauthlib" />
<item index="29" class="java.lang.String" itemvalue="Werkzeug" />
<item index="30" class="java.lang.String" itemvalue="kiwisolver" />
<item index="31" class="java.lang.String" itemvalue="tqdm" />
<item index="32" class="java.lang.String" itemvalue="yolov5processor" />
<item index="33" class="java.lang.String" itemvalue="tensorboard" />
<item index="34" class="java.lang.String" itemvalue="future" />
<item index="35" class="java.lang.String" itemvalue="matplotlib" />
<item index="36" class="java.lang.String" itemvalue="cachetools" />
<item index="37" class="java.lang.String" itemvalue="grpcio" />
<item index="38" class="java.lang.String" itemvalue="Keras" />
<item index="39" class="java.lang.String" itemvalue="google-auth" />
<item index="40" class="java.lang.String" itemvalue="idna" />
<item index="41" class="java.lang.String" itemvalue="Pillow" />
<item index="42" class="java.lang.String" itemvalue="cython" />
<item index="43" class="java.lang.String" itemvalue="keras" />
<item index="44" class="java.lang.String" itemvalue="imgaug" />
<item index="45" class="java.lang.String" itemvalue="opencv-contrib-python" />
<item index="46" class="java.lang.String" itemvalue="paho-mqtt" />
<item index="47" class="java.lang.String" itemvalue="pymongo" />
<item index="48" class="java.lang.String" itemvalue="fbprophet" />
<item index="49" class="java.lang.String" itemvalue="scikit-learn" />
<item index="50" class="java.lang.String" itemvalue="sklearn" />
<item index="51" class="java.lang.String" itemvalue="statsmodels" />
<item index="52" class="java.lang.String" itemvalue="python-dotenv" />
<item index="53" class="java.lang.String" itemvalue="pystan" />
<item index="54" class="java.lang.String" itemvalue="Cerberus" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N802" />
<option value="N806" />
<option value="N801" />
<option value="N803" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="cv2.imresize" />
<option value="detectron2.model_zoo.get_config_file" />
<option value="detectron2.model_zoo.get_checkpoint_url" />
</list>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
.idea/inspectionProfiles/profiles_settings.xml deleted 100644 → 0
View file @ 5c2d105f
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
.idea/misc.xml deleted 100644 → 0
View file @ 5c2d105f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (welspun-defects) (2)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
.idea/modules.xml deleted 100644 → 0
View file @ 5c2d105f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/preprocessweatherdata.iml" filepath="$PROJECT_DIR$/.idea/preprocessweatherdata.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/preprocessweatherdata.iml deleted 100644 → 0
View file @ 5c2d105f
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.7 (welspun-defects) (2)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="pytest" />
</component>
</module>
\ No newline at end of file
.idea/vcs.xml deleted 100644 → 0
View file @ 5c2d105f
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
logs/PreprocessWeatherData.log deleted 100644 → 0
View file @ 5c2d105f
2021-03-02 19:09:57,986 INFO PreprocessWeatherData Traceback (most recent call last):
File "E:/iLens-AI/preprocessweatherdata/main.py", line 62, in <module>
json_path = os.path.join(json_path_dir,os.listdir(json_path_dir)[0])
FileNotFoundError: [WinError 3] The system cannot find the path specified: 'temp/mnt/ilens/ai-job//pipe1/run1PreprocessWeatherData/input'
2021-03-02 19:11:35,286 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:11:35,301 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input/data.csv
2021-03-02 19:13:19,263 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:13:19,272 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input/data.csv
2021-03-02 19:14:37,570 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:14:37,580 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:14:37,584 INFO PreprocessWeatherData Component executed Successfully
2021-03-02 19:15:18,736 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:15:18,745 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:15:18,749 INFO PreprocessWeatherData Component executed Successfully
2021-03-02 19:17:47,686 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:17:47,696 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:17:47,701 INFO PreprocessWeatherData Component executed Successfully
2021-03-02 19:23:52,747 INFO PreprocessWeatherData Reading Json file
2021-03-02 19:23:52,761 INFO PreprocessWeatherData Writing forcast data to csv file at temp/mnt/ilens/ai-job//pipe1/run1/RunModel/input\data.csv
2021-03-02 19:23:52,766 INFO PreprocessWeatherData Component executed Successfully
scripts/common/config_parser.py
View file @ b2488cf4
...@@ -24,7 +24,7 @@ pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313") ...@@ -24,7 +24,7 @@ pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job", shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job",
_config.get("SYSTEM_CONFIG", {}).get('shared_mount_base_ai_job')) _config.get("SYSTEM_CONFIG", {}).get('shared_mount_base_ai_job'))
# read from $shared_mount_base_ai_job/$pipeline_id/run.config # read from $shared_mount_base_ai_job/$pipeline_id/run.config
run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run.json" run_id_path = shared_mount_base_ai_job + "/" + pipeline_id + "/run_config.json"
try: try:
sys.stderr.write("Checking for run id parameters at path " + run_id_path + "\n") sys.stderr.write("Checking for run id parameters at path " + run_id_path + "\n")
with open(run_id_path) as f: with open(run_id_path) as f:
......
temp/mnt/ilens/ai-job/pipe1/run.json deleted 100644 → 0
View file @ 5c2d105f
{
"run_id" : "run1"
}
\ No newline at end of file
temp/mnt/ilens/ai-job/pipe1/run1/GetDataFromStore/param.json deleted 100644 → 0
View file @ 5c2d105f
{
"artifact_name": "response.json"
}
\ No newline at end of file
temp/mnt/ilens/ai-job/pipe1/run1/RunModel/input/data.csv deleted 100644 → 0
View file @ 5c2d105f
,Maximum Temperature,Minimum Temperature,Temperature,Precipitation,Wind Speed,Wind Direction,Visibility,Cloud Cover,Relative Humidity,month
2021-03-02,36.0,24.0,28.4,0.0,18.4,126.4,9.8,7.6,71.7,3
2021-03-03,30.4,23.0,26.9,0.0,19.1,68.0,24.1,0.3,73.3,3
2021-03-04,29.8,23.1,26.5,0.1,21.2,54.0,24.1,17.3,72.0,3
2021-03-05,30.6,22.3,26.3,0.0,19.4,66.9,24.1,0.3,71.9,3
2021-03-06,30.5,22.9,26.7,0.0,20.9,51.9,24.1,0.1,73.9,3
2021-03-07,30.4,23.9,27.2,0.09,20.2,59.2,24.1,19.0,75.9,3
2021-03-08,31.3,24.8,27.7,0.35,20.9,52.7,24.1,20.9,75.0,3
2021-03-09,31.5,24.4,27.6,4.46,22.3,58.5,22.6,28.5,75.9,3
2021-03-10,31.9,24.2,27.8,0.14,21.2,66.2,24.1,5.0,76.6,3
2021-03-11,30.8,24.1,27.5,0.0,21.2,56.9,24.1,7.9,75.5,3
2021-03-12,31.2,24.1,27.7,0.51,21.6,65.7,24.1,28.6,74.0,3
2021-03-13,31.0,24.8,27.6,0.71,20.9,55.5,24.1,7.2,76.2,3
2021-03-14,31.9,23.7,27.4,0.0,22.0,64.0,24.1,2.9,74.6,3
2021-03-15,31.1,23.6,27.3,0.0,20.5,60.0,24.1,9.3,75.5,3
2021-03-16,30.6,23.5,27.2,0.0,21.2,59.6,24.1,10.7,72.7,3
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment