Skip to content

M
mppt-level-training
Commit 7cd06240 authored by aakash.bedi's avatar aakash.bedi
Browse files
Options

updated training module

parent 2b48d9ce
Pipeline #59979 canceled with stage
    Show whitespace changes
    Inline Side-by-side
    Showing with 97 additions and 73 deletions
    app.py
    View file @ 7cd06240
    ...@@ -5,17 +5,14 @@ import warnings ...@@ -5,17 +5,14 @@ import warnings
    from loguru import logger from loguru import logger
    import tracemalloc import tracemalloc
    import gc import gc
    from scripts.core.engine.mppt_data import GetData
    from scripts.core.engine.tags_data import get_tags_data
    from scripts.utils.start_end_date import KairosStartEndDate from scripts.utils.start_end_date import KairosStartEndDate
    from scripts.utils.preprocessing import DataPreprocessing
    from scripts.core.engine.data_training_and_inference import Training
    from scripts.core.engine.raw_predicted_tags import get_raw_predicted_tags from scripts.core.engine.raw_predicted_tags import get_raw_predicted_tags
    from scripts.core.engine.ml_modelling import AiModelling
    warnings.filterwarnings("ignore") warnings.filterwarnings("ignore")
    start_date, end_date, start_timestamp, end_timestamp = KairosStartEndDate().start_end_date() start_date, end_date, start_timestamp, end_timestamp = KairosStartEndDate().start_end_date()
    logger.info(f'start date - {start_date}')
    def orchestrator(): def orchestrator():
    try: try:
    ...@@ -30,75 +27,10 @@ def orchestrator(): ...@@ -30,75 +27,10 @@ def orchestrator():
    logger.info(f'raw tags dataframe shape - {df_raw_tags.shape}') logger.info(f'raw tags dataframe shape - {df_raw_tags.shape}')
    logger.info(f'predicted tags dataframe shape - {df_predicted_tags.shape}') logger.info(f'predicted tags dataframe shape - {df_predicted_tags.shape}')
    for inv_id in list(df_raw_tags['inv_id'].unique()): ai_modelling = AiModelling(df_raw_tags=df_raw_tags, df_coefficients=df_coefficients)
    df = df_raw_tags[df_raw_tags['inv_id'] == inv_id] ai_modelling.all_calculations()
    for mppt_id in list(df_raw_tags['mppt_id'].unique()): del ai_modelling
    print(f'1st memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'2nd memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'3rd memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    df_mppt_level = df[df['mppt_id'] == mppt_id]
    df_kairos_data = get_tags_data(df_input_tags=df_mppt_level, start_timestamp=start_timestamp,
    end_timestamp=end_timestamp, inv_id=inv_id, mppt_id=mppt_id)
    print(f'4th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'5th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'6th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    logger.info(f'Shape of final df - {df_kairos_data.shape}')
    mppt_data = GetData()
    df_mppt = mppt_data.associate_inv_mppt_id(df=df_kairos_data)
    print(f'7th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'8th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'9th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    df_coefficient_multiply = mppt_data.multiply_mppt_coefficients(df_mppt=df_mppt,
    df_coefficients=df_coefficients)
    print(f'10th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'11th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'12th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    data_preprocessing = DataPreprocessing()
    df_clean = data_preprocessing.remove_outliers(df=df_coefficient_multiply,
    param_list=['tilt_irradiance', 'voltage_mppt',
    'current_mppt'])
    print(f'13th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'14th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'15th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    Training(df=df_clean).data_training(inv_id=inv_id, mppt_id=mppt_id)
    del df_kairos_data
    del df_mppt
    del df_coefficient_multiply
    print(f'16th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'17th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'18th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    print(f'final allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    logger.info(f'data training for {inv_id}, {mppt_id} has been completed !')
    tracemalloc.stop() tracemalloc.stop()
    except Exception as e: except Exception as e:
    logger.exception(f'Exception - {e}') logger.exception(f'Exception - {e}')
    ......
    scripts/core/engine/ml_modelling.py 0 → 100644
    View file @ 7cd06240
    from loguru import logger
    import tracemalloc
    import gc
    from scripts.core.engine.tags_data import get_tags_data
    from scripts.core.engine.mppt_data import GetData
    from scripts.utils.preprocessing import DataPreprocessing
    from scripts.core.engine.data_training_and_inference import Training
    class AiModelling:
    def __init__(self, df_raw_tags, df_coefficients, start_timestamp, end_timestamp):
    self.df_raw_tags = df_raw_tags
    self.df_coefficients = df_coefficients
    self.start_timestamp = start_timestamp
    self.end_timestamp = end_timestamp
    def all_calculations(self):
    try:
    for inv_id in list(self.df_raw_tags['inv_id'].unique()):
    df = self.df_raw_tags[self.df_raw_tags['inv_id'] == inv_id]
    for mppt_id in list(self.df_raw_tags['mppt_id'].unique()):
    print(f'1st memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'2nd memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'3rd memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    df_mppt_level = df[df['mppt_id'] == mppt_id]
    df_kairos_data = get_tags_data(df_input_tags=df_mppt_level, start_timestamp=self.start_timestamp,
    end_timestamp=self.end_timestamp, inv_id=inv_id, mppt_id=mppt_id)
    print(f'4th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'5th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'6th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    logger.info(f'Shape of final df - {df_kairos_data.shape}')
    mppt_data = GetData()
    df_mppt = mppt_data.associate_inv_mppt_id(df=df_kairos_data)
    print(f'7th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'8th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'9th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    df_coefficient_multiply = mppt_data.multiply_mppt_coefficients(df_mppt=df_mppt,
    df_coefficients=self.df_coefficients)
    print(f'10th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'11th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'12th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    data_preprocessing = DataPreprocessing()
    df_clean = data_preprocessing.remove_outliers(df=df_coefficient_multiply,
    param_list=['tilt_irradiance', 'voltage_mppt',
    'current_mppt'])
    print(f'13th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'14th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'15th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    Training(df=df_clean).data_training(inv_id=inv_id, mppt_id=mppt_id)
    del df_kairos_data
    del df_mppt
    del df_coefficient_multiply
    print(f'16th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    gc.collect()
    tracemalloc.reset_peak()
    print(f'17th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    tracemalloc.clear_traces()
    print(f'18th memory allocation for {inv_id} & {mppt_id} - {tracemalloc.get_traced_memory()}')
    except Exception as e:
    logger.exception(f'Exception - {e}')
    def __del__(self):
    try:
    print('destructor called, die!')
    except Exception as e:
    logger.exception(f'Exception - {e}')
    \ No newline at end of file
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment