Commit 647d193e authored by dasharatha.vamshi's avatar dasharatha.vamshi

completed the flow

parent ad5678ea
# Get start and end time
## Overview
- **Component Name** : Get start and end time
- **Component Description** :
- **Component Type** : Input type
## Component Param
Variable Name |Datatype |Required/Optional |Default Value |Type |Description |Example
--- |--- |--- |--- |--- |--- |--- |
START_RELATIVE |String |Required |None |env | |
START_HOUR_RELATIVE |String |Required |None |env | |
DAY_STARTING_HOUR |String |Required |None |env | |
START_MINUTE_RELATIVE |String |Required |None |env | |
START_SECOND_RELATIVE |String |Required |None |env | |
END_RELATIVE |String |Required |None |env | |
END_HOUR_RELATIVE |String |Required |None |env | |
END_MINUTE_RELATIVE |String |Required |None |env | |
END_SECOND_RELATIVE |String |Required |None |env | |
REQUIRED_TZ |String |Required |None |env | |
Output |JsonObject |Required |None |outputPath | |
> Note 1 : Available Component types are: Input, Transform, Output.
> Note 2 : Available Environment types are: env, InputValues, InputPath, OutputValues, OutputPath, PipelineParm
\ No newline at end of file
import kfp
from loguru import logger
from src import program
import yaml
import inspect
import os
function = \
[func[1] for func in inspect.getmembers(program, inspect.isfunction) if inspect.getmodule(func[1]) == program][0]
def read_data_from_yaml(path):
"""
It opens the file at the given path, reads the contents, and then parses the contents as YAML
:param path: The path to the YAML file
:return: A dictionary
"""
with open(path, "r") as stream:
return yaml.load(stream, Loader=yaml.FullLoader)
def get_component_yml():
"""
:param file_name:
:return:
"""
try:
requirements = list()
with open('requirements.txt', 'r') as file:
for line in file:
if "=" in line and "#" not in line:
requirements.append(line.strip())
elif "#" in line:
...
else:
logger.exception(f"Mentioned package does not have version {line.strip()}")
date_function_yml = kfp.components.func_to_component_text(
function, packages_to_install=requirements)
variables_path = "variables.yml"
if os.path.exists(variables_path):
yaml_data: dict = read_data_from_yaml(variables_path)
if yaml_data:
envs: dict = yaml_data.get("deployment", {}).get("environmentVar", [])
python_version: str = yaml_data.get("deployment", {}).get("pythonVersion", None)
if python_version is not None and python_version in ["3.7", "3.8", "3.9", "3.10"]:
date_function_yml = kfp.components.func_to_component_text(
function, packages_to_install=requirements, base_image=f"python:{python_version}")
date_function = date_function_yml + f" env:\n"
for env_var in envs:
date_function += f" {env_var['name']}: '{env_var['value']}'\n"
with open('component.yml', 'w') as file:
file.write(date_function)
else:
with open('component.yml', 'w') as file:
file.write(date_function_yml)
else:
with open('component.yml', 'w') as file:
file.write(date_function_yml)
except Exception as e:
logger.exception(f"Unable to get the component yml {e}")
def create_table(data, key):
"""
:return:
"""
try:
rows_list = list()
for each_input in data.get(key, []):
rows_dict = dict()
rows_dict['name'] = each_input.get("name", '')
rows_dict['data_type'] = each_input.get('type', 'String')
if each_input.get('optional'):
req_opt = "Optional"
default_value = each_input.get('default', '')
else:
req_opt = "Required"
default_value = "None"
rows_dict['req_opt'] = req_opt
rows_dict['default_value'] = default_value
for each_arg in data.get('implementation', {}).get('container', {}).get('args', []):
if type(each_arg) == dict and rows_dict['name'] in each_arg.values():
rows_dict['Type'] = list(each_arg.keys())[0]
rows_dict['Description'] = each_input.get('description', '')
rows_dict['Example'] = ''
rows_list.append(list(rows_dict.values()))
if key == "inputs" and os.path.exists("variables.yml"):
yaml_data: dict = read_data_from_yaml("variables.yml")
if yaml_data:
env_var = yaml_data.get("deployment", {}).get("environmentVar", [])
for each in env_var:
env_dict = dict()
env_dict['name'] = each.get("name")
env_dict['data_type'] = "String"
env_dict['req_opt'] = "Required"
env_dict['default_value'] = "None"
env_dict['Type'] = "env"
env_dict['description'] = ""
env_dict['example'] = ""
rows_list.append(list(env_dict.values()))
return rows_list
except Exception as e:
logger.exception(f"Unable to create the table for README.MD file {e}")
def create_readme():
"""
Function is to create the readme file for the given components details
:return: Create the README.MD file in the given path
"""
try:
note_1 = "Note 1 : Available Component types are: Input, Transform, Output."
note_2 = "Note 2 : Available Environment types are: env, InputValues, InputPath, OutputValues," \
" OutputPath, PipelineParm"
column_list = ["Variable Name", "Datatype", "Required/Optional", "Default Value", "Type", "Description",
"Example"]
with open("component.yml", "r") as file:
data = yaml.safe_load(file)
if "inputs" in list(data.keys()) and "outputs" in list(data.keys()):
component_type = "Transform type"
elif "inputs" not in data:
component_type = "Input type"
else:
component_type = "Output type"
component_overview_json = dict()
component_overview_json['Component Name'] = data.get("name", " ")
component_overview_json['Component Description'] = data.get("description", " ")
component_overview_json['Component Type'] = component_type
rows_list_input = create_table(data, "inputs")
rows_list_output = create_table(data, "outputs")
rows_list = rows_list_input + rows_list_output
header = component_overview_json.get("Component Name")
table_header = " |".join(column_list) + "\n"
table_line = "--- |" * len(column_list) + "\n"
table_body = "\n".join(map(lambda x: " |".join(x), rows_list))
table = table_header + table_line + table_body
readme = f"""
# {header}
## {"Overview"}
- **Component Name** : {component_overview_json.get("Component Name")}
- **Component Description** : {component_overview_json.get("Component Description")}
- **Component Type** : {component_overview_json.get("Component Type")}
## Component Param
{table}
> {note_1}
> {note_2}
"""
with open('README.md', 'w') as f:
f.write(readme)
except Exception as e:
logger.exception(f"Unable to create the README.MD file {e}")
if __name__ == "__main__":
get_component_yml()
create_readme()
name: Get start and end time
outputs:
- {name: Output, type: JsonObject}
implementation:
container:
image: python:3.9
command:
- sh
- -c
- (PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location
'loguru==0.5.3' 'pytz==2021.3' 'pymongo~=3.12.1' 'pandas==1.3.*' || PIP_DISABLE_PIP_VERSION_CHECK=1
python3 -m pip install --quiet --no-warn-script-location 'loguru==0.5.3' 'pytz==2021.3'
'pymongo~=3.12.1' 'pandas==1.3.*' --user) && "$0" "$@"
- sh
- -ec
- |
program_path=$(mktemp)
printf "%s" "$0" > "$program_path"
python3 -u "$program_path" "$@"
- |
def get_start_and_end_time():
from loguru import logger
from datetime import datetime, timedelta
import pytz
import os
class DateRange:
start_relative_days = int(os.getenv("START_RELATIVE"))
start_hour_relative = int(os.getenv("START_HOUR_RELATIVE"))
day_starting_hour = int(os.getenv("DAY_STARTING_HOUR"))
start_minute_relative = int(os.getenv("START_MINUTE_RELATIVE"))
start_second_relative = int(os.getenv("START_SECOND_RELATIVE"))
end_relative_days = int(os.getenv("END_RELATIVE"))
end_hour_relative = int(os.getenv("END_HOUR_RELATIVE"))
end_minute_relative = int(os.getenv("END_MINUTE_RELATIVE"))
end_second_relative = int(os.getenv("END_SECOND_RELATIVE"))
class ReqTimeZone:
required_tz = os.getenv("REQUIRED_TZ")
try:
start_date = (datetime.now(pytz.utc) - timedelta(days=int(DateRange.start_relative_days),
hours=int(DateRange.start_hour_relative))). \
astimezone(pytz.timezone(ReqTimeZone.required_tz)).replace(hour=int(DateRange.day_starting_hour),
minute=int(DateRange.start_minute_relative),
second=int(DateRange.start_second_relative),
microsecond=0)
end_date = (datetime.now(pytz.utc) - timedelta(days=int(DateRange.end_relative_days),
hours=int(DateRange.end_hour_relative))). \
astimezone(pytz.timezone(ReqTimeZone.required_tz)).replace(minute=int(DateRange.end_minute_relative),
second=int(DateRange.end_second_relative),
microsecond=0)
start_timestamp = int(start_date.timestamp()) * 1000
end_timestamp = int(end_date.timestamp()) * 1000
final_dict = {
"start_timestamp": start_timestamp,
"end_timestamp": end_timestamp}
print(final_dict)
return final_dict
except Exception as e:
logger.exception(f'Exception - {e}')
def _serialize_json(obj) -> str:
if isinstance(obj, str):
return obj
import json
def default_serializer(obj):
if hasattr(obj, 'to_struct'):
return obj.to_struct()
else:
raise TypeError(
"Object of type '%s' is not JSON serializable and does not have .to_struct() method."
% obj.__class__.__name__)
return json.dumps(obj, default=default_serializer, sort_keys=True)
import argparse
_parser = argparse.ArgumentParser(prog='Get start and end time', description='')
_parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1)
_parsed_args = vars(_parser.parse_args())
_output_files = _parsed_args.pop("_output_paths", [])
_outputs = get_start_and_end_time(**_parsed_args)
_outputs = [_outputs]
_output_serializers = [
_serialize_json,
]
import os
for idx, output_file in enumerate(_output_files):
try:
os.makedirs(os.path.dirname(output_file))
except OSError:
pass
with open(output_file, 'w') as f:
f.write(_output_serializers[idx](_outputs[idx]))
args:
- '----output-paths'
- {outputPath: Output}
env:
START_RELATIVE: '0'
START_HOUR_RELATIVE: '1'
DAY_STARTING_HOUR: '5'
START_MINUTE_RELATIVE: '0'
START_SECOND_RELATIVE: '0'
END_RELATIVE: '0'
END_HOUR_RELATIVE: '1'
END_MINUTE_RELATIVE: '59'
END_SECOND_RELATIVE: '59'
REQUIRED_TZ: 'Asia/Bangkok'
loguru==0.5.3
pytz==2021.3
pymongo~=3.12.1
pandas==1.3.*
\ No newline at end of file
def get_start_and_end_time() -> dict:
from loguru import logger
from datetime import datetime, timedelta
import pytz
import os
class DateRange:
start_relative_days = int(os.getenv("START_RELATIVE"))
start_hour_relative = int(os.getenv("START_HOUR_RELATIVE"))
day_starting_hour = int(os.getenv("DAY_STARTING_HOUR"))
start_minute_relative = int(os.getenv("START_MINUTE_RELATIVE"))
start_second_relative = int(os.getenv("START_SECOND_RELATIVE"))
end_relative_days = int(os.getenv("END_RELATIVE"))
end_hour_relative = int(os.getenv("END_HOUR_RELATIVE"))
end_minute_relative = int(os.getenv("END_MINUTE_RELATIVE"))
end_second_relative = int(os.getenv("END_SECOND_RELATIVE"))
class ReqTimeZone:
required_tz = os.getenv("REQUIRED_TZ")
try:
start_date = (datetime.now(pytz.utc) - timedelta(days=int(DateRange.start_relative_days),
hours=int(DateRange.start_hour_relative))). \
astimezone(pytz.timezone(ReqTimeZone.required_tz)).replace(hour=int(DateRange.day_starting_hour),
minute=int(DateRange.start_minute_relative),
second=int(DateRange.start_second_relative),
microsecond=0)
end_date = (datetime.now(pytz.utc) - timedelta(days=int(DateRange.end_relative_days),
hours=int(DateRange.end_hour_relative))). \
astimezone(pytz.timezone(ReqTimeZone.required_tz)).replace(minute=int(DateRange.end_minute_relative),
second=int(DateRange.end_second_relative),
microsecond=0)
start_timestamp = int(start_date.timestamp()) * 1000
end_timestamp = int(end_date.timestamp()) * 1000
final_dict = {
"start_timestamp": start_timestamp,
"end_timestamp": end_timestamp}
print(final_dict)
return final_dict
except Exception as e:
logger.exception(f'Exception - {e}')
deployment:
environmentVar:
- name: START_RELATIVE
value: 0
- name: START_HOUR_RELATIVE
value: 1
- name: DAY_STARTING_HOUR
value: 5
- name: START_MINUTE_RELATIVE
value: 0
- name: START_SECOND_RELATIVE
value: 0
- name: END_RELATIVE
value: 0
- name: END_HOUR_RELATIVE
value: 1
- name: END_MINUTE_RELATIVE
value: 59
- name: END_SECOND_RELATIVE
value: 59
- name: REQUIRED_TZ
value: "Asia/Bangkok"
pythonVersion: "3.9"
......@@ -21,11 +21,23 @@ def forecast_pipeline(pipeline_param: dict, plant_info: dict):
"input_components/get_final_predicted_tags/component.yml")
get_inv_and_level_efficiency_tags = kfp.components.load_component_from_file(
"input_components/get_inv_and_level_efficiency_tags/component.yml")
# get_start_end_date = kfp.components.load_component_from_file(
# "input_components/get_start_and_end_date/component.yml")
get_inv_and_mppt_level_efficiency = kfp.components.load_component_from_file(
"transform_components/inv_and_mppt_level_efficiency/component.yml")
# Calling the component
get_tags_function_task = get_tags_function_component(pipeline_param).set_memory_request('600M').set_memory_limit('1200M').\
set_cpu_request('700m').set_cpu_limit('1400m')
get_final_predicted_tags_task = get_final_predicted_tags(get_tags_function_task.output)
get_inv_and_level_efficiency_tags_task = get_inv_and_level_efficiency_tags(get_tags_function_task.output)
# get_start_end_date_task = get_start_end_date().set_memory_request('600M').set_memory_limit('1200M').\
# set_cpu_request('700m').set_cpu_limit('1400m')
get_inv_and_mppt_level_efficiency_task = get_inv_and_mppt_level_efficiency(
get_tags_function_task.output,
get_final_predicted_tags_task.output,
get_inv_and_level_efficiency_tags_task.output)
# Disabling cacheing for all the components
get_tags_function_task.execution_options.caching_strategy.max_cache_staleness = "P0D"
......
This source diff could not be displayed because it is too large. You can view the blob instead.
# Inv and mppt level efficiency
## Overview
- **Component Name** : Inv and mppt level efficiency
- **Component Description** :
- **Component Type** : Transform type
## Component Param
Variable Name |Datatype |Required/Optional |Default Value |Type |Description |Example
--- |--- |--- |--- |--- |--- |--- |
get_tags_component_output |String |Required |None |inputPath | |
get_final_predicted_tags |String |Required |None |inputPath | |
get_inv_level_efficiency_tags |String |Required |None |inputPath | |
KAIROS_URI |String |Required |None |env | |
METRIC_NAME |String |Required |None |env | |
AGGREGATOR |String |Required |None |env | |
AGGREGATOR_VALUE |String |Required |None |env | |
AGGREGATOR_UNIT |String |Required |None |env | |
REQUIRED_TZ |String |Required |None |env | |
KAFKA_HOST |String |Required |None |env | |
KAFKA_PORT |String |Required |None |env | |
KAFKA_TOPIC |String |Required |None |env | |
START_RELATIVE |String |Required |None |env | |
START_HOUR_RELATIVE |String |Required |None |env | |
DAY_STARTING_HOUR |String |Required |None |env | |
START_MINUTE_RELATIVE |String |Required |None |env | |
START_SECOND_RELATIVE |String |Required |None |env | |
END_RELATIVE |String |Required |None |env | |
END_HOUR_RELATIVE |String |Required |None |env | |
END_MINUTE_RELATIVE |String |Required |None |env | |
END_SECOND_RELATIVE |String |Required |None |env | |
REQUIRED_TZ |String |Required |None |env | |
output |String |Required |None |outputPath | |
> Note 1 : Available Component types are: Input, Transform, Output.
> Note 2 : Available Environment types are: env, InputValues, InputPath, OutputValues, OutputPath, PipelineParm
\ No newline at end of file
import kfp
from loguru import logger
from src import program
import yaml
import inspect
import os
function = \
[func[1] for func in inspect.getmembers(program, inspect.isfunction) if inspect.getmodule(func[1]) == program][0]
def read_data_from_yaml(path):
"""
It opens the file at the given path, reads the contents, and then parses the contents as YAML
:param path: The path to the YAML file
:return: A dictionary
"""
with open(path, "r") as stream:
return yaml.load(stream, Loader=yaml.FullLoader)
def get_component_yml():
"""
:param file_name:
:return:
"""
try:
requirements = list()
with open('requirements.txt', 'r') as file:
for line in file:
if "=" in line and "#" not in line:
requirements.append(line.strip())
elif "#" in line:
...
else:
logger.exception(f"Mentioned package does not have version {line.strip()}")
date_function_yml = kfp.components.func_to_component_text(
function, packages_to_install=requirements)
variables_path = "variables.yml"
if os.path.exists(variables_path):
yaml_data: dict = read_data_from_yaml(variables_path)
if yaml_data:
envs: dict = yaml_data.get("deployment", {}).get("environmentVar", [])
python_version: str = yaml_data.get("deployment", {}).get("pythonVersion", None)
if python_version is not None and python_version in ["3.7", "3.8", "3.9", "3.10"]:
date_function_yml = kfp.components.func_to_component_text(
function, packages_to_install=requirements, base_image=f"python:{python_version}")
date_function = date_function_yml + f" env:\n"
for env_var in envs:
date_function += f" {env_var['name']}: '{env_var['value']}'\n"
with open('component.yml', 'w') as file:
file.write(date_function)
else:
with open('component.yml', 'w') as file:
file.write(date_function_yml)
else:
with open('component.yml', 'w') as file:
file.write(date_function_yml)
except Exception as e:
logger.exception(f"Unable to get the component yml {e}")
def create_table(data, key):
"""
:return:
"""
try:
rows_list = list()
for each_input in data.get(key, []):
rows_dict = dict()
rows_dict['name'] = each_input.get("name", '')
rows_dict['data_type'] = each_input.get('type', 'String')
if each_input.get('optional'):
req_opt = "Optional"
default_value = each_input.get('default', '')
else:
req_opt = "Required"
default_value = "None"
rows_dict['req_opt'] = req_opt
rows_dict['default_value'] = default_value
for each_arg in data.get('implementation', {}).get('container', {}).get('args', []):
if type(each_arg) == dict and rows_dict['name'] in each_arg.values():
rows_dict['Type'] = list(each_arg.keys())[0]
rows_dict['Description'] = each_input.get('description', '')
rows_dict['Example'] = ''
rows_list.append(list(rows_dict.values()))
if key == "inputs" and os.path.exists("variables.yml"):
yaml_data: dict = read_data_from_yaml("variables.yml")
if yaml_data:
env_var = yaml_data.get("deployment", {}).get("environmentVar", [])
for each in env_var:
env_dict = dict()
env_dict['name'] = each.get("name")
env_dict['data_type'] = "String"
env_dict['req_opt'] = "Required"
env_dict['default_value'] = "None"
env_dict['Type'] = "env"
env_dict['description'] = ""
env_dict['example'] = ""
rows_list.append(list(env_dict.values()))
return rows_list
except Exception as e:
logger.exception(f"Unable to create the table for README.MD file {e}")
def create_readme():
"""
Function is to create the readme file for the given components details
:return: Create the README.MD file in the given path
"""
try:
note_1 = "Note 1 : Available Component types are: Input, Transform, Output."
note_2 = "Note 2 : Available Environment types are: env, InputValues, InputPath, OutputValues," \
" OutputPath, PipelineParm"
column_list = ["Variable Name", "Datatype", "Required/Optional", "Default Value", "Type", "Description",
"Example"]
with open("component.yml", "r") as file:
data = yaml.safe_load(file)
if "inputs" in list(data.keys()) and "outputs" in list(data.keys()):
component_type = "Transform type"
elif "inputs" not in data:
component_type = "Input type"
else:
component_type = "Output type"
component_overview_json = dict()
component_overview_json['Component Name'] = data.get("name", " ")
component_overview_json['Component Description'] = data.get("description", " ")
component_overview_json['Component Type'] = component_type
rows_list_input = create_table(data, "inputs")
rows_list_output = create_table(data, "outputs")
rows_list = rows_list_input + rows_list_output
header = component_overview_json.get("Component Name")
table_header = " |".join(column_list) + "\n"
table_line = "--- |" * len(column_list) + "\n"
table_body = "\n".join(map(lambda x: " |".join(x), rows_list))
table = table_header + table_line + table_body
readme = f"""
# {header}
## {"Overview"}
- **Component Name** : {component_overview_json.get("Component Name")}
- **Component Description** : {component_overview_json.get("Component Description")}
- **Component Type** : {component_overview_json.get("Component Type")}
## Component Param
{table}
> {note_1}
> {note_2}
"""
with open('README.md', 'w') as f:
f.write(readme)
except Exception as e:
logger.exception(f"Unable to create the README.MD file {e}")
if __name__ == "__main__":
get_component_yml()
create_readme()
pytz==2021.3
loguru==0.5.3
scipy==1.7.1
numpy==1.21.0
mlflow==1.20.2
simplejson==3.17.5
requests==2.27.1
pydantic==1.8.2
python-dotenv==0.19.2
kafka-python==1.4.7
SQLAlchemy==1.3.20
sqlparse==0.4.2
protobuf==3.20.*
pandas==1.5.3
PyYAML==5.4
azure-storage-blob==12.14.1
azure-core==1.27.0
scikit-learn==1.0.2
\ No newline at end of file
deployment:
environmentVar:
- name: KAIROS_URI
value: "https://iLens:iLensCLD$456@cloud.ilens.io/kairos/"
- name: METRIC_NAME
value: "project_264__ilens.live_data.raw"
- name: AGGREGATOR
value: "max"
- name: AGGREGATOR_VALUE
value: "15"
- name: AGGREGATOR_UNIT
value: "minutes"
- name: REQUIRED_TZ
value: "Asia/Bangkok"
- name: KAFKA_HOST
value: "192.168.0.220"
- name: KAFKA_PORT
value: 9092
- name: KAFKA_TOPIC
value: "ilens_dev"
- name: START_RELATIVE
value: 0
- name: START_HOUR_RELATIVE
value: 1
- name: DAY_STARTING_HOUR
value: 5
- name: START_MINUTE_RELATIVE
value: 0
- name: START_SECOND_RELATIVE
value: 0
- name: END_RELATIVE
value: 0
- name: END_HOUR_RELATIVE
value: 1
- name: END_MINUTE_RELATIVE
value: 59
- name: END_SECOND_RELATIVE
value: 59
- name: REQUIRED_TZ
value: "Asia/Bangkok"
pythonVersion: "3.9"
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment