Skip to content

3
3cp_test
Commit f3d4565b authored by aakash.bedi's avatar aakash.bedi
Browse files
Options

modifications

parent 6bc89d6c
Hide whitespace changes
Inline Side-by-side
Showing with 686 additions and 209 deletions
app.py
View file @ f3d4565b
......@@ -9,7 +9,34 @@ from datetime import datetime
import pytz
from loguru import logger
from scripts.utils.yield_sheet_3cp_utils.all_tags_3cp import AllTags
from scripts.utils.yield_sheet_3cp_utils.report_generator_3cp import ReportGenerator
from scripts.utils.yield_sheet_3cp_utils.report_generator_3cp import ReportGenerator, get_dpr_report_format, \
get_dpr
def form_excel_multiindex(df, df_format):
# ebpr_dir = os.path.join(FilePath.report_directory, "ebpr")
# if not os.path.exists(ebpr_dir):
# os.mkdir(ebpr_dir)
#
# output_dir = os.path.join(ebpr_dir, "yield")
# if not os.path.exists(output_dir):
# os.mkdir(output_dir)
master_output_file = 'test_prod.xlsx'
master_output_file = get_dpr(master_output_file=master_output_file, df=df, df_format=df_format)
logger.info("XLSX is getting stitched")
# logger.info(f"{os.listdir(output_dir)}")
if os.path.isfile(master_output_file):
return master_output_file, ""
else:
return None, "File not created"
# __kwargs__.update(
# start_date=input_data.property.get(
# "start_date",
......@@ -37,92 +64,23 @@ all_tags = config_engine["tag_heirarcy"]
tags_cal, tags_cal_prev, tags_manual, tags_dcs = AllTags().get_tags(all_tags_dictionary=all_tags)
# start_date = datetime.strptime(start_date, '%Y-%m-%d')
# start_date = start_date.astimezone(pytz.UTC)
start_date = datetime.strptime("2022-12-25", '%Y-%m-%d').replace(hour=5, minute=0, second=0,
start_date = datetime.strptime("2023-02-05", '%Y-%m-%d').replace(hour=5, minute=0, second=0,
microsecond=0)
start_date = start_date.astimezone(pytz.timezone("Asia/Kolkata")).replace(hour=5, minute=0, second=0,
microsecond=0)
# end_date = datetime.strptime(end_date, '%Y-%m-%d')
# end_date = end_date.astimezone(pytz.UTC)
end_date = datetime.strptime("2022-12-26", '%Y-%m-%d').replace(hour=5, minute=0, second=0,
end_date = datetime.strptime("2023-03-20", '%Y-%m-%d').replace(hour=5, minute=0, second=0,
microsecond=0)
end_date = end_date.astimezone(pytz.timezone("Asia/Kolkata")).replace(hour=5, minute=0, second=0,
microsecond=0)
df, message = ReportGenerator(tags_cal=tags_cal, tags_cal_prev=tags_cal_prev, tags_manual=tags_manual,
tags_dcs=tags_dcs, start_date=start_date,
end_date=end_date).yield_report_3cp()
df_format, df, message = ReportGenerator(tags_cal=tags_cal, tags_cal_prev=tags_cal_prev, tags_manual=tags_manual,
tags_dcs=tags_dcs, start_date=start_date,
end_date=end_date).yield_report_3cp()
logger.debug(f'{df.shape}')
logger.debug(f'{message}')
master_output_file = 'test_prod.xlsx'
writer = pd.ExcelWriter(master_output_file, engine='xlsxwriter')
df.to_excel(writer, sheet_name="DPR Sheet", index=True)
workbook = writer.book
format = workbook.add_format(
{'font_name': 'Trebuchet MS', 'text_wrap': True, 'bold': 2, 'font_color': "blue"})
format.set_align('center')
format.set_align('vcenter')
format1 = workbook.add_format({'font_name': 'Trebuchet MS', 'text_wrap': True})
format1.set_align('center')
format1.set_align('vcenter')
header_footer_format = workbook.add_format({
'text_wrap': True
})
no_of_rows = df.shape[0]
worksheet = writer.sheets["DPR Sheet"]
# set the column width as per your requirement
worksheet.set_column('A:F', 15, format)
worksheet.set_column('G:L', 20, format)
worksheet.set_column('N:T', 22, format)
worksheet.set_column('U:Z', 22, format)
worksheet.set_column('AB:AD', 20, format)
worksheet.set_column('AE:AM', 22, format)
worksheet.set_column('AN:AO', 29, format)
worksheet.set_column('AQ:AU', 27, format)
worksheet.set_column('AV:BC', 33, format)
worksheet.set_column('BD:BD', 30, format)
worksheet.set_column('BE:BH', 27, format)
worksheet.set_column('AY:AY', 40, format)
worksheet.set_column('BA:BA', 35, format)
worksheet.set_column('M:M', 10, format)
worksheet.set_column('T:T', 10, format)
worksheet.set_column('AD:AD', 10, format)
worksheet.set_column('AP:AP', 10, format)
worksheet.set_column('BK:BK', 10, format)
worksheet.set_column('AA:AA', 10, format)
worksheet.set_column('AT:AT', 30, format)
worksheet.set_column('BA:BA', 25, format)
format4 = workbook.add_format({'bg_color': 'yellow'})
format5 = workbook.add_format({'text_wrap': True})
worksheet.set_row(0, 28, format5)
worksheet.conditional_format(f'A{no_of_rows + 2}:AP{no_of_rows + 2}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'AV{no_of_rows + 2}:AZ{no_of_rows + 2}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'BB{no_of_rows + 2}:BD{no_of_rows + 2}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
# worksheet.conditional_format(f'BF{no_of_rows + 2}:BF{no_of_rows + 2}',
# {'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'AQ{no_of_rows + 3}:AU{no_of_rows + 3}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'BA{no_of_rows + 3}:BA{no_of_rows + 3}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'BE{no_of_rows + 3}:BG{no_of_rows + 3}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
writer.save()
if not master_output_file.endswith(".xlsx"):
master_output_file = master_output_file + ".xlsx"
logger.info("XLSX is getting stitched")
form_excel_multiindex(df=df, df_format=df_format)
conf/application.conf
View file @ f3d4565b
[KAIROS_DB]
uri = $KAIROS_URI
[LOGGING]
level = $LOG_LEVEL
traceback = $LOG_TRACEBACK
config.env
View file @ f3d4565b
KAIROS_URI= https://iLens:iLensJUB$456@jub-kairos.ilens.io/kairos
KAIROS_URI=https://iLens:iLensJUB$456@jub-kairos.ilens.io/kairos
LOG_LEVEL=INFO
LOG_TRACEBACK=true
logs/ebpr_report_engine.log
View file @ f3d4565b
2023-03-23 15:15:23 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 15:43:36 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 16:01:48 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 17:55:59 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 17:56:03 - ERROR - [MainThread:get_excel_format():102] - Exception - Unalignable boolean Series provided as indexer (index of the boolean Series and of the indexed object do not match).
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 86, in get_excel_format
df_format = df[df_format['month'] == latest_month]
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 3449, in __getitem__
return self._getitem_bool_array(key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 3502, in _getitem_bool_array
key = check_bool_indexer(self.index, key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 2388, in check_bool_indexer
raise IndexingError(
pandas.core.indexing.IndexingError: Unalignable boolean Series provided as indexer (index of the boolean Series and of the indexed object do not match).
2023-03-23 17:56:18 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 17:56:58 - ERROR - [MainThread:get_excel_format():102] - Exception - Unalignable boolean Series provided as indexer (index of the boolean Series and of the indexed object do not match).
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 86, in get_excel_format
df_format = df[df_format['month'] == latest_month]
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 3449, in __getitem__
return self._getitem_bool_array(key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 3502, in _getitem_bool_array
key = check_bool_indexer(self.index, key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 2388, in check_bool_indexer
raise IndexingError(
pandas.core.indexing.IndexingError: Unalignable boolean Series provided as indexer (index of the boolean Series and of the indexed object do not match).
2023-03-23 17:57:48 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 17:59:31 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 18:00:00 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 18:00:23 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 22:51:59 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 22:52:19 - ERROR - [MainThread:get_dpr_report_format():146] - Exception - 0
Traceback (most recent call last):
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\base.py", line 3361, in get_loc
return self._engine.get_loc(casted_key)
File "pandas\_libs\index.pyx", line 76, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\index.pyx", line 108, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\hashtable_class_helper.pxi", line 2131, in pandas._libs.hashtable.Int64HashTable.get_item
File "pandas\_libs\hashtable_class_helper.pxi", line 2140, in pandas._libs.hashtable.Int64HashTable.get_item
KeyError: 0
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 98, in get_dpr_report_format
df_format[df_format['day'] == latest_day]['Pure_Production_Day_Nia_DPR'][0],
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\series.py", line 942, in __getitem__
return self._get_value(key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\series.py", line 1051, in _get_value
loc = self.index.get_loc(label)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\base.py", line 3363, in get_loc
raise KeyError(key) from err
KeyError: 0
2023-03-23 22:52:49 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 22:54:52 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 22:55:08 - ERROR - [MainThread:get_dpr_report_format():146] - Exception - 0
Traceback (most recent call last):
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\base.py", line 3361, in get_loc
return self._engine.get_loc(casted_key)
File "pandas\_libs\index.pyx", line 76, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\index.pyx", line 108, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\hashtable_class_helper.pxi", line 2131, in pandas._libs.hashtable.Int64HashTable.get_item
File "pandas\_libs\hashtable_class_helper.pxi", line 2140, in pandas._libs.hashtable.Int64HashTable.get_item
KeyError: 0
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 98, in get_dpr_report_format
df_format[df_format['day'] == latest_day]['Pure_Production_Day_Nia_DPR'][0],
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\series.py", line 942, in __getitem__
return self._get_value(key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\series.py", line 1051, in _get_value
loc = self.index.get_loc(label)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\base.py", line 3363, in get_loc
raise KeyError(key) from err
KeyError: 0
2023-03-23 22:56:34 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 22:57:16 - ERROR - [MainThread:get_dpr_report_format():146] - Exception - Must have equal len keys and value when setting with an ndarray
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 142, in get_dpr_report_format
df_report_format.iloc[1:4, :10] = data_1
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 723, in __setitem__
iloc._setitem_with_indexer(indexer, value, self.name)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1730, in _setitem_with_indexer
self._setitem_with_indexer_split_path(indexer, value, name)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1769, in _setitem_with_indexer_split_path
self._setitem_with_indexer_2d_value(indexer, value)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1829, in _setitem_with_indexer_2d_value
raise ValueError(
ValueError: Must have equal len keys and value when setting with an ndarray
2023-03-23 22:57:34 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:02:10 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:02:14 - ERROR - [MainThread:get_dpr_report_format():147] - Exception - Must have equal len keys and value when setting with an iterable
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 143, in get_dpr_report_format
df_report_format.iloc[index+1, :10] = data_1[index]
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 723, in __setitem__
iloc._setitem_with_indexer(indexer, value, self.name)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1730, in _setitem_with_indexer
self._setitem_with_indexer_split_path(indexer, value, name)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1808, in _setitem_with_indexer_split_path
raise ValueError(
ValueError: Must have equal len keys and value when setting with an iterable
2023-03-23 23:03:54 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:04:13 - ERROR - [MainThread:get_dpr_report_format():147] - Exception - could not broadcast input array from shape (4,10) into shape (0,10)
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 144, in get_dpr_report_format
df_report_format.iloc[4:, :10] = data_2
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 723, in __setitem__
iloc._setitem_with_indexer(indexer, value, self.name)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1732, in _setitem_with_indexer
self._setitem_single_block(indexer, value, name)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1968, in _setitem_single_block
self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\internals\managers.py", line 355, in setitem
return self.apply("setitem", indexer=indexer, value=value)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\internals\managers.py", line 327, in apply
applied = getattr(b, f)(**kwargs)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\internals\blocks.py", line 984, in setitem
values[indexer] = value
ValueError: could not broadcast input array from shape (4,10) into shape (0,10)
2023-03-23 23:05:30 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:05:30 - ERROR - [MainThread:yield_report_3cp():73] - Exception occurred - unknown opcode
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 148, in get_dpr_report_format
except Exception as e:
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 723, in __setitem__
iloc._setitem_with_indexer(indexer, value, self.name)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1732, in _setitem_with_indexer
self._setitem_single_block(indexer, value, name)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexing.py", line 1968, in _setitem_single_block
self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\internals\managers.py", line 355, in setitem
return self.apply("setitem", indexer=indexer, value=value)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\internals\managers.py", line 327, in apply
applied = getattr(b, f)(**kwargs)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\internals\blocks.py", line 984, in setitem
values[indexer] = value
ValueError: could not broadcast input array from shape (4,10) into shape (0,10)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 66, in yield_report_3cp
df_report = get_dpr_report_format(df=df)
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 149, in get_dpr_report_format
logger.exception(f'Exception - {e}')
SystemError: unknown opcode
2023-03-23 23:05:46 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:06:18 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:06:54 - ERROR - [MainThread:get_dpr_report_format():145] - Exception - 'Date'
Traceback (most recent call last):
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\base.py", line 3361, in get_loc
return self._engine.get_loc(casted_key)
File "pandas\_libs\index.pyx", line 76, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\index.pyx", line 108, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\hashtable_class_helper.pxi", line 5198, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas\_libs\hashtable_class_helper.pxi", line 5206, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 'Date'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 81, in get_dpr_report_format
df_format['Date'] = pd.to_datetime(df_format['Date'])
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 3457, in __getitem__
return self._getitem_multilevel(key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 3508, in _getitem_multilevel
loc = self.columns.get_loc(key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\multi.py", line 2922, in get_loc
loc = self._get_level_indexer(key, level=0)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\multi.py", line 3204, in _get_level_indexer
idx = self._get_loc_single_level_index(level_index, key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\multi.py", line 2855, in _get_loc_single_level_index
return level_index.get_loc(key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\base.py", line 3363, in get_loc
raise KeyError(key) from err
KeyError: 'Date'
2023-03-23 23:08:33 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:10:57 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:11:25 - ERROR - [MainThread:get_dpr_report_format():147] - Exception - cannot handle a non-unique multi-index!
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 80, in get_dpr_report_format
df_format = df_format[df_format['Date'] != 'Total']
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 3457, in __getitem__
return self._getitem_multilevel(key)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 3513, in _getitem_multilevel
result = self.reindex(columns=new_columns)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\util\_decorators.py", line 324, in wrapper
return func(*args, **kwargs)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 4772, in reindex
return super().reindex(**kwargs)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\generic.py", line 4818, in reindex
return self._reindex_axes(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 4591, in _reindex_axes
frame = frame._reindex_columns(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\frame.py", line 4633, in _reindex_columns
new_columns, indexer = self.columns.reindex(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\core\indexes\multi.py", line 2524, in reindex
raise ValueError("cannot handle a non-unique multi-index!")
ValueError: cannot handle a non-unique multi-index!
2023-03-23 23:14:32 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:41:44 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:41:44 - ERROR - [MainThread:get_dpr():240] - Exception - [Errno 13] Permission denied: 'test_prod.xlsx'
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 152, in get_dpr
writer = pd.ExcelWriter(master_output_file, engine='xlsxwriter')
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\excel\_xlsxwriter.py", line 191, in __init__
super().__init__(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\excel\_base.py", line 925, in __init__
self.handles = get_handle(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\common.py", line 711, in get_handle
handle = open(handle, ioargs.mode)
PermissionError: [Errno 13] Permission denied: 'test_prod.xlsx'
2023-03-23 23:42:12 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:45:29 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:45:45 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-23 23:45:58 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:26:46 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:27:34 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:28:18 - ERROR - [MainThread:get_dpr():245] - Exception - 'Worksheet' object has no attribute 'add_format'
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 217, in get_dpr
cell_format = worksheet_format.add_format({'align': 'center', 'valign': 'vcenter', 'border': 1})
AttributeError: 'Worksheet' object has no attribute 'add_format'
2023-03-24 00:32:13 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:32:16 - ERROR - [MainThread:get_dpr():245] - Exception - 'Worksheet' object has no attribute 'add_format'
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 221, in get_dpr
format6 = worksheet_format.add_format({'bg_color': 'yellow', 'text_wrap': True})
AttributeError: 'Worksheet' object has no attribute 'add_format'
2023-03-24 00:33:21 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:36:05 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:38:00 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:38:00 - ERROR - [MainThread:get_dpr():235] - Exception - [Errno 13] Permission denied: 'test_prod.xlsx'
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 153, in get_dpr
writer = pd.ExcelWriter(master_output_file, engine='xlsxwriter')
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\excel\_xlsxwriter.py", line 191, in __init__
super().__init__(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\excel\_base.py", line 925, in __init__
self.handles = get_handle(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\common.py", line 711, in get_handle
handle = open(handle, ioargs.mode)
PermissionError: [Errno 13] Permission denied: 'test_prod.xlsx'
2023-03-24 00:38:05 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:39:06 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:39:47 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:44:33 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:48:54 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:51:14 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 00:59:38 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:03:01 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:06:25 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:12:19 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:13:06 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:14:26 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:15:31 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:16:55 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:20:19 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:23:09 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:23:45 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:29:48 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:30:14 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:31:51 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:32:23 - INFO - [MainThread:yield_report_3cp():36] -
2023-03-24 01:40:44 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:40:44 - ERROR - [MainThread:get_dpr():254] - Exception - [Errno 13] Permission denied: 'test_prod.xlsx'
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 154, in get_dpr
writer = pd.ExcelWriter(master_output_file, engine='xlsxwriter')
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\excel\_xlsxwriter.py", line 191, in __init__
super().__init__(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\excel\_base.py", line 925, in __init__
self.handles = get_handle(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\common.py", line 711, in get_handle
handle = open(handle, ioargs.mode)
PermissionError: [Errno 13] Permission denied: 'test_prod.xlsx'
2023-03-24 01:40:53 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:41:22 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:41:22 - ERROR - [MainThread:get_dpr():254] - Exception - [Errno 13] Permission denied: 'test_prod.xlsx'
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 154, in get_dpr
writer = pd.ExcelWriter(master_output_file, engine='xlsxwriter')
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\excel\_xlsxwriter.py", line 191, in __init__
super().__init__(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\excel\_base.py", line 925, in __init__
self.handles = get_handle(
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\pandas\io\common.py", line 711, in get_handle
handle = open(handle, ioargs.mode)
PermissionError: [Errno 13] Permission denied: 'test_prod.xlsx'
2023-03-24 01:41:43 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:44:48 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:44:49 - ERROR - [MainThread:get_dpr():259] - Exception - not enough values to unpack (expected 2, got 1)
Traceback (most recent call last):
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\xlsxwriter\worksheet.py", line 139, in column_wrapper
int(args[0])
ValueError: invalid literal for int() with base 10: 'A'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 173, in get_dpr
worksheet.set_column('A', 20, _format)
File "C:\Users\aakash.bedi\Anaconda3\envs\3cp_valid_recommendations\lib\site-packages\xlsxwriter\worksheet.py", line 142, in column_wrapper
cell_1, cell_2 = [col + '1' for col in args[0].split(':')]
ValueError: not enough values to unpack (expected 2, got 1)
2023-03-24 01:45:28 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:47:14 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:47:47 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:48:27 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:50:38 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:52:59 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:55:04 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:57:47 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:58:48 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 01:59:31 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 02:00:59 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 02:03:12 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 02:03:50 - INFO - [MainThread:yield_report_3cp():37] -
2023-03-24 02:03:50 - ERROR - [MainThread:get_dpr_report_format():149] - Exception - index 0 is out of bounds for axis 0 with size 0
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 99, in get_dpr_report_format
df_format[df_format['day'] == 1]['Pure_Production_Opening_DPR'].values[0],
IndexError: index 0 is out of bounds for axis 0 with size 0
2023-03-24 02:03:50 - ERROR - [MainThread:get_dpr():259] - Exception - cannot unpack non-iterable NoneType object
Traceback (most recent call last):
File "E:\Data\GitCodes\3cp changes\3cp_test\scripts\utils\yield_sheet_3cp_utils\report_generator_3cp.py", line 156, in get_dpr
df_report_format, last_date = get_dpr_report_format(df=df_format)
TypeError: cannot unpack non-iterable NoneType object
2023-03-24 02:04:50 - INFO - [MainThread:yield_report_3cp():37] -
scripts/constants/app_configuration.py
View file @ f3d4565b
......@@ -43,6 +43,11 @@ class DB:
uri = config["KAIROS_DB"]["uri"]
class Logging:
level = config.get("LOGGING", "level", fallback="INFO")
level = level or "INFO"
tb_flag = config.getboolean("LOGGING", "traceback", fallback=True)
tb_flag = tb_flag or True
......
scripts/utils/yield_sheet_3cp_utils/data_puller_3cp.py
View file @ f3d4565b
......@@ -21,7 +21,7 @@ class Kairos_query:
"tags": {
"c3": tag
},
"name": "project_227__ilens.live_data.raw",
"name": "ilens.live_data.raw",
"group_by": [
{
"name": "tag",
......@@ -32,7 +32,7 @@ class Kairos_query:
{
"name": "last",
"sampling": {
"value": "1",
"value": "2",
"unit": "minutes"
}
}
......@@ -52,7 +52,6 @@ class Kairos_query:
output = {}
try:
response = requests.post(self.kairos_url, data=json.dumps(query))
abb = response.json()
grouped_output_data = response.json()["queries"][0]["results"]
for each_grouped_data in grouped_output_data:
value = (each_grouped_data["values"])
......@@ -63,11 +62,10 @@ class Kairos_query:
else:
for k, v in tags_dict.items():
output[v] = 0
# return output
try:
output[tags_dict[tag_id]] = round(value[0][1], 2)
except Exception as e:
logger.exception(f"Exception occurred for tag = {tag_id} and date = {date}", exc_info=True)
logger.exception(f"Exception - {e}", exc_info=True)
output[tags_dict[tag_id]] = 0
return output
except Exception as e:
......
scripts/utils/yield_sheet_3cp_utils/manual_dcs_dataframe.py
View file @ f3d4565b
......@@ -5,53 +5,85 @@ from loguru import logger
from scripts.utils.yield_sheet_3cp_utils.data_puller_3cp import Kairos_query
class ManualDcsData:
def __init__(self, start_date, end_date, tags_cal, tags_cal_prev, tags_manual, tags_dcs):
class AllTagsDataPuller:
def __init__(self, start_date, end_date):
self.start_date = start_date
self.end_date = end_date
self.tags_cal = tags_cal
self.tags_cal_prev = tags_cal_prev
self.tags_manual = tags_manual
self.tags_dcs = tags_dcs
def manual_dcs_dataframe(self):
def get_kairos_data(self, tags_lst, tags_dict):
try:
all_manual_dcs_tags_dict, all_cal_tags_dict = TagsDict().all_tags(self.tags_cal, self.tags_cal_prev,
self.tags_manual, self.tags_dcs)
all_manual_dcs_tags = list(all_manual_dcs_tags_dict.values())
# logger.info(f"No of manual and dcs tags = {len(all_manual_dcs_tags)}")
# All calculated tags combined
all_calculated_tags = list(all_cal_tags_dict.values())
# logger.info(f"No of all calculated tags = {len(all_calculated_tags)}")
all_tags = [*all_calculated_tags, *all_manual_dcs_tags]
all_tags_dict = {**all_cal_tags_dict, **all_manual_dcs_tags_dict}
# All dates for which we are going to do calculations
all_dates = [self.start_date + timedelta(days=x) for x in range((self.end_date - self.start_date).days + 1)]
# all_dates.insert(0, self.start_date)
logger.debug(f"Data required for dates : {all_dates}")
# Pulling data for all manual and dcs dates for required dates
date_output = {}
for dates in all_dates:
try:
periodic_start_time = dates
logger.info(f"Pulling Manual and DCS data for date - {periodic_start_time}")
current_date = periodic_start_time
periodic_start_time = int(periodic_start_time.timestamp()) * 1000
periodic_end_time = int(dates.timestamp()) * 1000
query_manual_dcs = Kairos_query().kairos_query(start=periodic_start_time, end=periodic_end_time,
tag=all_manual_dcs_tags)
logger.info(f"{query_manual_dcs}")
data_manual_dcs = Kairos_query().get_data_from_kairos(query=query_manual_dcs,
tags_dict=dict((v, k)
for k, v in all_manual_dcs_tags_dict.items()),
date=current_date)
date_output[current_date] = data_manual_dcs
except Exception as e:
logger.exception(f"Exception occurred", exc_info=True)
periodic_start_time = dates - timedelta(minutes=0)
logger.info(f"Pulling data for date - {periodic_start_time}")
current_date = periodic_start_time
periodic_start_time = int(periodic_start_time.timestamp()) * 1000
periodic_end_time = dates + timedelta(minutes=10)
periodic_end_time = int(periodic_end_time.timestamp()) * 1000
query_manual_dcs = Kairos_query().kairos_query(start=periodic_start_time, end=periodic_end_time,
tag=tags_lst)
logger.info(f"{query_manual_dcs}")
data_manual_dcs = Kairos_query().get_data_from_kairos(query=query_manual_dcs,
tags_dict=dict((v, k) for k, v in
tags_dict.items()),
date=current_date)
date_output[dates] = data_manual_dcs
return date_output
except Exception as e:
logger.exception(f"Exception occurred - {e}", exc_info=True)
\ No newline at end of file
logger.exception(f'Exception - {e}')
def get_dataframe(date_dict, tags_lst):
try:
df = pd.DataFrame(index=[i for i in range(len(date_dict))], columns=tags_lst)
df['Date'] = list(date_dict.keys())
col = df.pop("Date")
df.insert(0, col.name, col)
for index in range(df.shape[0]):
for params in df.columns:
if params not in ['Date']:
present_date = df.iloc[index, df.columns.get_loc('Date')]
available_tags = list(date_dict[present_date].keys())
if params not in available_tags:
df.iloc[index, df.columns.get_loc(params)] = None
else:
df.iloc[index, df.columns.get_loc(params)] = date_dict[present_date][params]
df.sort_values('Date', inplace=True)
df.reset_index(drop=True, inplace=True)
df = df.astype({'Date': str,
'7302011030_Consumptions_Closing_DPR': float, 'D1D001Readings_T_2703_A_DPR': float,
'D1D001Readings_T_2703_B_DPR': float, 'D1D001Readings_Conv_DPR': float,
'D1D001_consumptions_Day_Receipt_DPR': float, '7302011030_Consumptions_Day_Receipt_DPR': float,
'7302011061_Consumption_Day_Receipt_DPR': float, '7302011061_Consumption_Day_Cons_DPR': float,
'Pure_Production_Day_Nia_DPR': float, 'Pure_Production_Day_Drum_Filling_DPR': float,
'Pure_Production_Pure_tank_Dead_Volumes_DPR': float, 'Utility_report_Day_Power': float,
'Utility_report_Day_Steam': float, 'Utility_report_Day_Raffinate': float,
'Utility_report_Vent_Gas_Raffinate': float, 'Utility_report_Day_DM': float,
'Utility_report_Day_Treated_Water': float, 'Utility_report_Raffinate_Incinerated': float,
'Beta_Purification_Column_C_2409_Outlet_Flow_TZ': float, 'D1D001Readings_T_2101_A_DPR': float,
'D1D001Readings_T_2101_B_DPR': float, 'Pure_Production_LT_2701_A_DPR': float,
'Pure_Production_LT_2701_B_DPR': float, 'D1D001_consumptions_Opening_DPR': float,
'D1D001_consumptions_Day_Cons_DPR': float, '7302011030_Consumptions_Opening_DPR': float,
'7302011030_Consumptions_Day_Cons_DPR': float, '7302011061_Consumption_Opening_DPR': float,
'Crude_Prod_Day_Prod_DPR': float, 'Pure_Production_Opening_DPR': float,
'Pure_Production_Day_Prod_DPR': float, 'Utility_report_Power_Norms': float,
'Utility_report_Steam_Norms': float, 'Utility_report_Raffinate_Norms': float,
'Utility_report_Raffinate_Vent_Gas': float, 'Utility_report_Raw_Water_Norms': float,
'Utility_report_per_hr_burn_rate': float, 'D1D001_consumptions_Closing_DPR': float,
'Utility_report_Actual_Ammonia_Norms': float, 'Utility_report_Actual_Beta_Norms': float,
'Utility_report_Actual_Benzene_Norms': float, 'Utility_report_Day_DM_norm': float,
'D1D001_consumptions_Day_Recovery_DPR': float, 'D1D001_consumptions_Total_Receipt_DPR': float,
'7302011030_Consumptions_Total_Receipt_DPR': float,
'7302011030_Consumptions_Total_Cons_DPR': float,
'7302011061_Consumption_Closing_DPR': float, '7302011061_Consumption_Total_Receipt_DPR': float,
'7302011061_Consumption_Total_Cons_DPR': float, 'Crude_Prod_Total_Prod_DPR': float,
'Pure_Production_Closing_of_Pure_Tanks_only_DPR': float,
'Pure_Production_Total_Prod_DPR': float, 'Pure_Production_Total_Nia_DPR': float,
'Pure_Production_Total_Drum_Filling_DPR': float})
return df
except Exception as e:
logger.exception(f'Exception - {e}')
\ No newline at end of file
scripts/utils/yield_sheet_3cp_utils/reorder_renaming_ebpr.py
View file @ f3d4565b
......@@ -3,8 +3,8 @@ from loguru import logger
class ReorderRename:
def __init__(self,df_ebpr):
self.df_ebpr=df_ebpr
def __init__(self, df_ebpr):
self.df_ebpr = df_ebpr
def reorder_rename(self):
try:
df1 = self.df_ebpr[['Date',
......
scripts/utils/yield_sheet_3cp_utils/report_generator_3cp.py
View file @ f3d4565b
......@@ -2,18 +2,16 @@ import numpy as np
import warnings
warnings.filterwarnings("ignore")
import traceback
from loguru import logger
import xlsxwriter
from scripts.logging import logger
import pandas as pd
from datetime import datetime, timedelta
from scripts.utils.yield_sheet_3cp_utils.all_tags_3cp import TagsDict
enable_traceback = True
from scripts.utils.yield_sheet_3cp_utils.manual_dcs_dataframe import ManualDcsData
from scripts.utils.yield_sheet_3cp_utils.unavailable_manual_dcs_tags import UnavailableManualDcsTags
from scripts.utils.yield_sheet_3cp_utils.ebpr_manual_dataframe import EbprManualDcsDataframe
from scripts.utils.yield_sheet_3cp_utils.calculated_dataframe import CalculatedDataframe
from scripts.utils.yield_sheet_3cp_utils.concat_manual_calculated_dataframe import ConcatenateManualCalculatedDf
from scripts.utils.yield_sheet_3cp_utils.manual_dcs_dataframe import AllTagsDataPuller, get_dataframe
from scripts.utils.yield_sheet_3cp_utils.reorder_renaming_ebpr import ReorderRename
class ReportGenerator:
def __init__(self, tags_cal, tags_cal_prev, tags_manual, tags_dcs, start_date, end_date):
self.tags_cal = tags_cal
......@@ -25,89 +23,237 @@ class ReportGenerator:
def yield_report_3cp(self):
try:
date_output = ManualDcsData(start_date=self.start_date, end_date=self.end_date, tags_cal=self.tags_cal,
tags_cal_prev=self.tags_cal_prev, tags_manual=self.tags_manual,
tags_dcs=self.tags_dcs).manual_dcs_dataframe()
all_manual_dcs_tags_dict, all_cal_tags_dict = TagsDict().all_tags(self.tags_cal, self.tags_cal_prev,
self.tags_manual, self.tags_dcs)
all_manual_dcs_tags = list(all_manual_dcs_tags_dict.values())
all_calculated_tags = list(all_cal_tags_dict.values())
all_tags = [*all_calculated_tags, *all_manual_dcs_tags]
all_tags_dict = {**all_cal_tags_dict, **all_manual_dcs_tags_dict}
unavailable_tags, unavailable_data_date = UnavailableManualDcsTags(date_output=date_output,
all_manual_dcs_tags=all_manual_dcs_tags,
all_manual_dcs_tags_dict=all_manual_dcs_tags_dict,
first_date=self.start_date).unavailable_manual_dcs_tags()
if len(unavailable_data_date)>0:
logger.info(f"No data is available for - {unavailable_data_date}")
return None, f"No data is available for - {unavailable_data_date}"
else:
logger.info(f"Given tags are missing for dates - {unavailable_tags}")
unavailable_tags.pop(self.start_date, None)
if len(unavailable_tags)>=1:
logger.info(f"Given tags are missing for Date - {unavailable_tags}")
return None, f"Given tags are missing for Date - {unavailable_tags}"
else:
df_manual_dcs, df_manual_dcs_ebpr, date_output_filtered = EbprManualDcsDataframe(date_output=date_output,
all_manual_dcs_tags=all_manual_dcs_tags).\
ebpr_manual_dcs_dataframe()
df_calculated_ebpr, df_calculated, unavailable_tags_cal = CalculatedDataframe(date_output_filtered=
date_output_filtered,
all_calculated_tags=all_calculated_tags,
all_cal_tags_dict=all_cal_tags_dict,
df_manual_dcs=df_manual_dcs).calculated_df()
df_ebpr = ConcatenateManualCalculatedDf(df_manual_dcs_ebpr=df_manual_dcs_ebpr,
df_calculated_ebpr=df_calculated_ebpr,
start_date=self.start_date).concate_manual_calculated_df()
df_ebpr_copy = df_ebpr.copy()
df_ebpr.replace([np.inf, -np.inf], np.nan, inplace=True)
# a = df_ebpr['Utility_report_Power_Norms'].unique()
# col_name = df_ebpr.columns[1:]
# for col in col_name:
# df_ebpr[col] = df_ebpr[col].replace([inf], 'nan')
total_list = []
for col in [col for col in list(df_ebpr.columns) if col not in ('Date',
'Utility_report_Power_Norms',
'Utility_report_Steam_Norms',
'Utility_report_Raffinate_Norms',
'Utility_report_Raffinate_Vent_Gas',
'Utility_report_Raw_Water_Norms',
'Utility_report_Actual_Ammonia_Norms',
'Utility_report_Actual_Beta_Norms',
'Utility_report_Day_DM_norm',
'Utility_report_Actual_Benzene_Norms')]:
# total_list.append(df_ebpr[col].sum())
df_ebpr.loc['Total', col] = df_ebpr[col].sum()
# df_ebpr.loc['Total'] = [i for i in range(df_ebpr.shape[1])]
# df_ebpr.iloc[-1, 1:] = total_list
df_ebpr['Date'][-1] = 'Total'
average_list = []
for col in [col for col in list(df_ebpr.columns) if col in ('Utility_report_Power_Norms',
'Utility_report_Steam_Norms',
'Utility_report_Raffinate_Norms',
'Utility_report_Raffinate_Vent_Gas',
'Utility_report_Raw_Water_Norms',
'Utility_report_Actual_Ammonia_Norms',
'Utility_report_Actual_Beta_Norms',
'Utility_report_Day_DM_norm',
'Utility_report_Actual_Benzene_Norms')]:
df_ebpr.loc['Average', col] = df_ebpr[col].mean()
# average_list.append(df_ebpr[col].mean())
# df_ebpr.loc['Average'] = [i for i in range(df_ebpr.shape[1])]
# df_ebpr.iloc[-1, 1:] = average_list
df_ebpr['Date'][-1] = 'Average'
df_ebpr = df_ebpr.round(3)
df_concat = ReorderRename(df_ebpr=df_ebpr).reorder_rename()
df_concat.replace({'inf': 'nan'}, inplace=True)
return df_concat, f"Report is ready"
all_tags_dict = {**all_manual_dcs_tags_dict, **all_cal_tags_dict}
all_tags = list(all_tags_dict.values())
all_tags_data_puller = AllTagsDataPuller(start_date=self.start_date, end_date=self.end_date)
manual_dcs_input = all_tags_data_puller.get_kairos_data(tags_dict=all_tags_dict,
tags_lst=all_tags)
df = get_dataframe(date_dict=manual_dcs_input, tags_lst=list(all_tags_dict.keys()))
logger.info(f'')
for col in [col for col in list(df.columns) if col not in ('Date',
'Utility_report_Power_Norms',
'Utility_report_Steam_Norms',
'Utility_report_Raffinate_Norms',
'Utility_report_Raffinate_Vent_Gas',
'Utility_report_Raw_Water_Norms',
'Utility_report_Actual_Ammonia_Norms',
'Utility_report_Actual_Beta_Norms',
'Utility_report_Day_DM_norm',
'Utility_report_Actual_Benzene_Norms')]:
df.loc['Total', col] = df[col].sum()
df['Date'][-1] = 'Total'
average_list = []
for col in [col for col in list(df.columns) if col in ('Utility_report_Power_Norms',
'Utility_report_Steam_Norms',
'Utility_report_Raffinate_Norms',
'Utility_report_Raffinate_Vent_Gas',
'Utility_report_Raw_Water_Norms',
'Utility_report_Actual_Ammonia_Norms',
'Utility_report_Actual_Beta_Norms',
'Utility_report_Day_DM_norm',
'Utility_report_Actual_Benzene_Norms')]:
df.loc['Average', col] = df[col].mean()
df['Date'][-1] = 'Average'
df = df.round(3)
df_concat = ReorderRename(df_ebpr=df).reorder_rename()
df_concat.replace({'inf': 'nan'}, inplace=True)
return df, df_concat, f"Report is ready"
except Exception as e:
logger.exception(f"Exception occurred - {e}", exc_info=True)
return None, f"Error - {e}"
def get_dpr_report_format(df):
try:
df_format = df.copy()
df_format = df_format.drop(['Total', 'Average'])
df_format['Date'] = pd.to_datetime(df_format['Date'])
df_format['month'] = df_format['Date'].dt.month
df_format['day'] = df_format['Date'].dt.day
last_date = df_format.iloc[-1, df_format.columns.get_loc('Date')].date()
latest_month = max(list(df_format['month'].unique()))
df_format = df_format[df_format['month'] == latest_month]
df_format.reset_index(drop=True, inplace=True)
latest_day = max(list(df_format['day'].unique()))
data_0 = [[last_date, last_date, last_date, last_date, last_date,
last_date, last_date, last_date, last_date, last_date]]
data_1 = [['Production', 'UOM', "Month's opening", "Day Production", "To date Production",
"Day Nia Transfer", "To date Nia Transfer", "Day 3CP Drum Filling Dispatch",
"To Date 3CP Drum Filling Dispatch", "Closing Stock"],
["Pure 3CP", "MT",
df_format[df_format['day'] == 1]['Pure_Production_Opening_DPR'].values[0],
df_format[df_format['day'] == latest_day]['Pure_Production_Day_Prod_DPR'].values[0],
df_format[df_format['day'] == latest_day]['Pure_Production_Total_Prod_DPR'].values[0],
df_format[df_format['day'] == latest_day]['Pure_Production_Day_Nia_DPR'].values[0],
df_format[df_format['day'] == latest_day]['Pure_Production_Total_Nia_DPR'].values[0], None, None,
df_format[df_format['day'] == latest_day]['Pure_Production_Closing_of_Pure_Tanks_only_DPR'].values[0]],
['Crude 3CP', 'MT', None,
df_format[df_format['day'] == latest_day]['Crude_Prod_Day_Prod_DPR'].values[0],
df_format[df_format['day'] == latest_day]['Crude_Prod_Total_Prod_DPR'].values[0],
None, None, None, None, None]]
data_2 = [
['ITEM CODE', 'Raw Material', 'UOM', "Month's Opening", 'Day Opening Stock', 'Day Receipt',
'To Date Receipt', 'Day Consumption', 'To Date Consumption', 'Closing Stock'],
['F00040-BULK-0001', 'Beta Picoline', 'MT',
df_format[df_format['day'] == 1]['D1D001_consumptions_Opening_DPR'].values[0],
df_format[df_format['day'] == latest_day]['D1D001_consumptions_Opening_DPR'].values[0],
df_format[df_format['day'] == latest_day]['D1D001_consumptions_Day_Receipt_DPR'].values[0],
df_format[df_format['day'] == latest_day]['D1D001_consumptions_Total_Receipt_DPR'].values[0],
df_format[df_format['day'] == latest_day]['Crude_Prod_Day_Prod_DPR'].values[0]*1.04,
df_format[df_format['day'] == 1]['D1D001_consumptions_Opening_DPR'].values[0] +
df_format[df_format['day'] == latest_day]['D1D001_consumptions_Total_Receipt_DPR'].values[0] -
df_format[df_format['day'] == latest_day]['D1D001_consumptions_Closing_DPR'].values[0],
df_format[df_format['day'] == latest_day]['D1D001_consumptions_Closing_DPR'].values[0]
],
['7302011030', 'Ammonia', 'MT',
df_format[df_format['day'] == 1]['7302011030_Consumptions_Opening_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011030_Consumptions_Opening_DPR'].values[0],
df_format[df_format['day'] == latest_day]['D1D001_consumptions_Day_Receipt_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011030_Consumptions_Total_Receipt_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011030_Consumptions_Day_Cons_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011030_Consumptions_Total_Cons_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011030_Consumptions_Closing_DPR'].values[0]
],
['7302011061', 'Benzene', 'MT',
df_format[df_format['day'] == 1]['7302011061_Consumption_Opening_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011061_Consumption_Opening_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011061_Consumption_Day_Receipt_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011061_Consumption_Total_Receipt_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011061_Consumption_Day_Cons_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011061_Consumption_Total_Cons_DPR'].values[0],
df_format[df_format['day'] == latest_day]['7302011061_Consumption_Closing_DPR'].values[0]
]
]
final_lst = data_0 + data_1 + data_2
df_report_format = pd.DataFrame(final_lst)
return df_report_format, last_date
except Exception as e:
logger.exception(f'Exception - {e}')
def get_dpr(master_output_file, df, df_format):
try:
writer = pd.ExcelWriter(master_output_file, engine='xlsxwriter')
df.to_excel(writer, sheet_name="DPR Sheet", index=True)
df_report_format, last_date = get_dpr_report_format(df=df_format)
df_report_format.to_excel(writer, sheet_name="Report Format", index=False)
workbook = writer.book
_format = workbook.add_format(
{'font_name': 'Trebuchet MS', 'text_wrap': True, 'bold': 2, 'font_color': "blue"})
_format.set_align('center')
_format.set_align('vcenter')
format1 = workbook.add_format({'font_name': 'Trebuchet MS', 'text_wrap': True})
format1.set_align('center')
format1.set_align('vcenter')
header_footer_format = workbook.add_format({
'text_wrap': True
})
no_of_rows = df.shape[0]
worksheet = writer.sheets["DPR Sheet"]
# set the column width as per your requirement
worksheet.set_column('A:A', 25, _format)
worksheet.set_column('B:F', 15, _format)
worksheet.set_column('G:L', 20, _format)
worksheet.set_column('N:T', 22, _format)
worksheet.set_column('U:Z', 22, _format)
worksheet.set_column('AB:AD', 20, _format)
worksheet.set_column('AE:AM', 22, _format)
worksheet.set_column('AN:AO', 29, _format)
worksheet.set_column('AQ:AU', 27, _format)
worksheet.set_column('AV:BC', 33, _format)
worksheet.set_column('BD:BD', 30, _format)
worksheet.set_column('BE:BH', 27, _format)
worksheet.set_column('AY:AY', 40, _format)
worksheet.set_column('BA:BA', 35, _format)
worksheet.set_column('M:M', 10, _format)
worksheet.set_column('T:T', 10, _format)
worksheet.set_column('AD:AD', 10, _format)
worksheet.set_column('AP:AP', 10, _format)
worksheet.set_column('BK:BK', 10, _format)
worksheet.set_column('AA:AA', 10, _format)
worksheet.set_column('AT:AT', 30, _format)
worksheet.set_column('BA:BA', 25, _format)
format4 = workbook.add_format({'bg_color': 'yellow'})
format5 = workbook.add_format({'text_wrap': True})
worksheet.set_row(0, 28, format5)
worksheet.conditional_format(f'A{no_of_rows + 2}:AP{no_of_rows + 2}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'AV{no_of_rows + 2}:AZ{no_of_rows + 2}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'BB{no_of_rows + 2}:BD{no_of_rows + 2}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'AQ{no_of_rows + 3}:AU{no_of_rows + 3}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'BA{no_of_rows + 3}:BA{no_of_rows + 3}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
worksheet.conditional_format(f'BE{no_of_rows + 3}:BG{no_of_rows + 3}',
{'type': 'cell', 'criteria': '<=', 'value': 10000000, 'format': format4})
border_fmt = workbook.add_format({'bottom': 2, 'top': 2, 'left': 2, 'right': 2})
worksheet.conditional_format(xlsxwriter.utility.xl_range(0, 0, df_format.shape[0]+4, 58),
{'type': 'no_errors', 'format': border_fmt})
_format1 = workbook.add_format(
{'font_name': 'Trebuchet MS', 'text_wrap': True, 'bold': 2, 'font_color': "black"})
_format1.set_align('center')
_format1.set_align('vcenter')
header_footer_format = workbook.add_format({
'text_wrap': True
})
no_of_rows_format = df_report_format.shape[0]
worksheet_format = writer.sheets["Report Format"]
worksheet_format.set_column('A:J', 15, _format1)
cell_format = workbook.add_format({'align': 'center', 'valign': 'vcenter', 'border': 1, 'bold': True,
'text_wrap': True})
worksheet_format.merge_range('A1:J2', str(last_date), cell_format)
cell_format_headings = workbook.add_format({'align': 'center', 'valign': 'vcenter',
'bold': True, 'text_wrap': True, 'font_color': 'black'})
worksheet_format.set_row(2, 15, cell_format_headings)
worksheet_format.set_row(5, 15, cell_format_headings)
cell_format_content = workbook.add_format({'align': 'center', 'valign': 'vcenter',
'bold': True, 'text_wrap': True, 'font_color': 'blue'})
worksheet_format.set_row(3, 13, cell_format_content)
worksheet_format.set_row(4, 13, cell_format_content)
worksheet_format.set_row(6, 13, cell_format_content)
worksheet_format.set_row(7, 13, cell_format_content)
worksheet_format.set_row(8, 13, cell_format_content)
border_fmt = workbook.add_format({'bottom': 2, 'top': 2, 'left': 2, 'right': 2})
worksheet_format.conditional_format(xlsxwriter.utility.xl_range(0, 0, 8, 9),
{'type': 'no_errors', 'format': border_fmt})
writer.save()
writer.save()
if not master_output_file.endswith('.xlsx'):
master_output_file = master_output_file + '.xlsx'
return master_output_file
except Exception as e:
logger.exception(f'Exception - {e}')
test_prod.xlsx
View file @ f3d4565b
No preview for this file type
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment