Skip to content

B
bgrimm-string-inference
Commit 95c1d2b6 authored by dasharatha.vamshi's avatar dasharatha.vamshi
Browse files
Options

added get tags

parent b34dc33e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 185 additions and 151 deletions
input_components/get_tags_component/component.yml
View file @ 95c1d2b6
...@@ -30,14 +30,28 @@ implementation: ...@@ -30,14 +30,28 @@ implementation:
city = os.getenv("CITY") city = os.getenv("CITY")
db_ = os.getenv("MONGO_DB") db_ = os.getenv("MONGO_DB")
print(pipeline_param) print(pipeline_param)
print("--",pipeline_param["MONGO_URI"]) print("--", pipeline_param["MONGO_URI"])
# collections # collections
collection_ = os.getenv("MONGO_COLLECTION") collection_ = os.getenv("MONGO_COLLECTION")
mongo_uri_ = pipeline_param['MONGO_URI'] mongo_uri_ = pipeline_param['MONGO_URI']
print("mongo_uri",mongo_uri_) print("mongo_uri", mongo_uri_)
project_id_ = pipeline_param['PROJECT_ID'] project_id_ = pipeline_param['PROJECT_ID']
query_filter_ = pipeline_param['QUERY_FILTER'] query_filter_ = pipeline_param['QUERY_FILTER']
try: try:
class CommonConstants:
bgrimm_string_level_tags = 'bgrimm_string_level_tags'
panel_id = 'panel_id'
sub_id = 'sub_id'
inv_id_mppt_id = 'inv_id_mppt_id'
tags_property_raw = 'raw'
tags_property_predicted = 'predicted'
tags_property_efficiency = 'efficiency'
bgrim_tags_property_efficiency = 'Efficiency'
tags_property_efficiency_inv = 'efficiency'
tags_property_efficiency_plant = 'efficiency_plant'
mppt_coefficients = 'mppt_coefficients'
class MongoConstants: class MongoConstants:
# DB # DB
db = db_ db = db_
...@@ -92,49 +106,47 @@ implementation: ...@@ -92,49 +106,47 @@ implementation:
except Exception as e: except Exception as e:
logger.exception(f'Exception - {e}') logger.exception(f'Exception - {e}')
tracemalloc.clear_traces()
mongo_conn = MongoConnect(uri=Mongo.mongo_uri, database=MongoConstants.db, mongo_conn = MongoConnect(uri=Mongo.mongo_uri, database=MongoConstants.db,
collection=MongoConstants.collection) collection=MongoConstants.collection)
if mongo_conn is None: if mongo_conn is None:
logger.info(f'mongodb is not connected, please check') logger.info(f'mongodb is not connected, please check')
else: else:
logger.info(f'mongodb is connected, mongo conn - {mongo_conn}') tracemalloc.clear_traces()
logger.info(f'mongo conn - {mongo_conn}')
df_raw_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [
{"id": CommonConstants.bgrimm_string_level_tags}, {"city": city},
{"tags_property": CommonConstants.tags_property_raw}]})['input_data'], orient='index')
df_raw_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [{"id": "bgrimm_string_level_tags"}, df_predicted_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [
{"city": city}, {"id": CommonConstants.bgrimm_string_level_tags}, {"city": city},
{"tags_property": "raw"}]}) {"tags_property": CommonConstants.tags_property_predicted}]})['input_data'], orient='index')
['input_data'], orient='index')
df_predicted_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [{"id": "bgrimm_string_level_tags"}, df_efficiency_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [
{"city": city}, {"id": CommonConstants.bgrimm_string_level_tags}, {"city": city},
{"tags_property": "predicted"}]}) {"tags_property": CommonConstants.tags_property_efficiency}]})['input_data'], orient='index')
['input_data'], orient='index')
df_raw_tags.reset_index(inplace=True) df_raw_tags.reset_index(inplace=True)
df_raw_tags.rename(columns={'index': 'tag_name'}, inplace=True) df_raw_tags.rename(columns={'index': 'tag_name'}, inplace=True)
df_predicted_tags.reset_index(inplace=True) df_predicted_tags.reset_index(inplace=True)
df_predicted_tags.rename(columns={'index': 'tag_name'}, inplace=True) df_predicted_tags.rename(columns={'index': 'tag_name'}, inplace=True)
df_efficiency_tags.reset_index(inplace=True)
df_efficiency_tags.rename(columns={'index': 'tag_name'}, inplace=True)
try: # df_coefficients = pd.DataFrame.from_dict(mongo_conn.find_one(
# df_coefficients = pd.DataFrame.from_dict( # {"$and": [{"id": CommonConstants.bgrimm_string_level_tags}, {"city": city},
# mongo_conn.find_one({"$and": [{"id": "bgrimm_string_level_tags"}, # {"tags_property": CommonConstants.mppt_coefficients}]})['input_data'], orient='index')
# {"city": city},
# {"tags_property":
# "mppt_coefficients"}]})
# ['input_data'], orient='index')
df_coefficients = pd.DataFrame()
except Exception as er:
logger.exception(f"Coefficient dataframe unavailable with message: {er}")
df_coefficients = pd.DataFrame()
del mongo_conn # df_coefficients.reset_index(inplace=True)
# df_coefficients.rename(columns={'index': CommonConstants.inv_id_mppt_id}, inplace=True)
df_coefficients.reset_index(inplace=True)
df_coefficients.rename(columns={'index': 'inv_id_mppt_id'}, inplace=True)
df_coefficients = pd.DataFrame()
tracemalloc.clear_traces() tracemalloc.clear_traces()
tracemalloc.get_traced_memory() del mongo_conn
final_dict = {"raw": df_raw_tags.to_dict('records'), "predicted": df_predicted_tags.to_dict('records'), final_dict = {"raw": df_raw_tags.to_dict('records'), "predicted": df_predicted_tags.to_dict('records'),
"coefficients": df_coefficients.to_dict('records')} "coefficients": df_coefficients.to_dict('records'),
"efficiency": df_efficiency_tags.to_dict('records')}
print(final_dict) print(final_dict)
return final_dict return final_dict
except Exception as e: except Exception as e:
......
input_components/get_tags_component/src/program.py
View file @ 95c1d2b6
def get_tags_function(pipeline_param: dict) -> dict: def get_tags_function(pipeline_param: dict) -> dict:
import pandas as pd import pandas as pd
from loguru import logger from loguru import logger
...@@ -9,14 +8,28 @@ def get_tags_function(pipeline_param: dict) -> dict: ...@@ -9,14 +8,28 @@ def get_tags_function(pipeline_param: dict) -> dict:
city = os.getenv("CITY") city = os.getenv("CITY")
db_ = os.getenv("MONGO_DB") db_ = os.getenv("MONGO_DB")
print(pipeline_param) print(pipeline_param)
print("--",pipeline_param["MONGO_URI"]) print("--", pipeline_param["MONGO_URI"])
# collections # collections
collection_ = os.getenv("MONGO_COLLECTION") collection_ = os.getenv("MONGO_COLLECTION")
mongo_uri_ = pipeline_param['MONGO_URI'] mongo_uri_ = pipeline_param['MONGO_URI']
print("mongo_uri",mongo_uri_) print("mongo_uri", mongo_uri_)
project_id_ = pipeline_param['PROJECT_ID'] project_id_ = pipeline_param['PROJECT_ID']
query_filter_ = pipeline_param['QUERY_FILTER'] query_filter_ = pipeline_param['QUERY_FILTER']
try: try:
class CommonConstants:
bgrimm_string_level_tags = 'bgrimm_string_level_tags'
panel_id = 'panel_id'
sub_id = 'sub_id'
inv_id_mppt_id = 'inv_id_mppt_id'
tags_property_raw = 'raw'
tags_property_predicted = 'predicted'
tags_property_efficiency = 'efficiency'
bgrim_tags_property_efficiency = 'Efficiency'
tags_property_efficiency_inv = 'efficiency'
tags_property_efficiency_plant = 'efficiency_plant'
mppt_coefficients = 'mppt_coefficients'
class MongoConstants: class MongoConstants:
# DB # DB
db = db_ db = db_
...@@ -71,49 +84,47 @@ def get_tags_function(pipeline_param: dict) -> dict: ...@@ -71,49 +84,47 @@ def get_tags_function(pipeline_param: dict) -> dict:
except Exception as e: except Exception as e:
logger.exception(f'Exception - {e}') logger.exception(f'Exception - {e}')
tracemalloc.clear_traces()
mongo_conn = MongoConnect(uri=Mongo.mongo_uri, database=MongoConstants.db, mongo_conn = MongoConnect(uri=Mongo.mongo_uri, database=MongoConstants.db,
collection=MongoConstants.collection) collection=MongoConstants.collection)
if mongo_conn is None: if mongo_conn is None:
logger.info(f'mongodb is not connected, please check') logger.info(f'mongodb is not connected, please check')
else: else:
logger.info(f'mongodb is connected, mongo conn - {mongo_conn}') tracemalloc.clear_traces()
logger.info(f'mongo conn - {mongo_conn}')
df_raw_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [{"id": "bgrimm_string_level_tags"}, df_raw_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [
{"city": city}, {"id": CommonConstants.bgrimm_string_level_tags}, {"city": city},
{"tags_property": "raw"}]}) {"tags_property": CommonConstants.tags_property_raw}]})['input_data'], orient='index')
['input_data'], orient='index')
df_predicted_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [{"id": "bgrimm_string_level_tags"}, df_predicted_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [
{"city": city}, {"id": CommonConstants.bgrimm_string_level_tags}, {"city": city},
{"tags_property": "predicted"}]}) {"tags_property": CommonConstants.tags_property_predicted}]})['input_data'], orient='index')
['input_data'], orient='index')
df_efficiency_tags = pd.DataFrame.from_dict(mongo_conn.find_one({"$and": [
{"id": CommonConstants.bgrimm_string_level_tags}, {"city": city},
{"tags_property": CommonConstants.tags_property_efficiency}]})['input_data'], orient='index')
df_raw_tags.reset_index(inplace=True) df_raw_tags.reset_index(inplace=True)
df_raw_tags.rename(columns={'index': 'tag_name'}, inplace=True) df_raw_tags.rename(columns={'index': 'tag_name'}, inplace=True)
df_predicted_tags.reset_index(inplace=True) df_predicted_tags.reset_index(inplace=True)
df_predicted_tags.rename(columns={'index': 'tag_name'}, inplace=True) df_predicted_tags.rename(columns={'index': 'tag_name'}, inplace=True)
df_efficiency_tags.reset_index(inplace=True)
df_efficiency_tags.rename(columns={'index': 'tag_name'}, inplace=True)
try: # df_coefficients = pd.DataFrame.from_dict(mongo_conn.find_one(
# df_coefficients = pd.DataFrame.from_dict( # {"$and": [{"id": CommonConstants.bgrimm_string_level_tags}, {"city": city},
# mongo_conn.find_one({"$and": [{"id": "bgrimm_string_level_tags"}, # {"tags_property": CommonConstants.mppt_coefficients}]})['input_data'], orient='index')
# {"city": city},
# {"tags_property":
# "mppt_coefficients"}]})
# ['input_data'], orient='index')
df_coefficients = pd.DataFrame()
except Exception as er:
logger.exception(f"Coefficient dataframe unavailable with message: {er}")
df_coefficients = pd.DataFrame()
del mongo_conn
df_coefficients.reset_index(inplace=True) # df_coefficients.reset_index(inplace=True)
df_coefficients.rename(columns={'index': 'inv_id_mppt_id'}, inplace=True) # df_coefficients.rename(columns={'index': CommonConstants.inv_id_mppt_id}, inplace=True)
df_coefficients = pd.DataFrame()
tracemalloc.clear_traces() tracemalloc.clear_traces()
tracemalloc.get_traced_memory() del mongo_conn
final_dict = {"raw": df_raw_tags.to_dict('records'), "predicted": df_predicted_tags.to_dict('records'), final_dict = {"raw": df_raw_tags.to_dict('records'), "predicted": df_predicted_tags.to_dict('records'),
"coefficients": df_coefficients.to_dict('records')} "coefficients": df_coefficients.to_dict('records'),
"efficiency": df_efficiency_tags.to_dict('records')}
print(final_dict) print(final_dict)
return final_dict return final_dict
except Exception as e: except Exception as e:
......
pipeline.yml
View file @ 95c1d2b6
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment