Skip to content

O
oee-services
Commit f8692e94 authored by harshavardhan.c's avatar harshavardhan.c
Browse files
Options

oee config services commit

parent 15ccf0a8
Hide whitespace changes
Inline Side-by-side
Showing with 774 additions and 376 deletions
app.py 0 → 100644
View file @ f8692e94
if __name__ == '__main__':
from dotenv import load_dotenv
load_dotenv()
from scripts.logging import logger
import argparse
import gc
import uvicorn
from scripts.config import Service
gc.collect()
ap = argparse.ArgumentParser()
if __name__ == "__main__":
ap.add_argument(
"--port",
"-p",
required=False,
default=Service.PORT,
help="Port to start the application.",
)
ap.add_argument(
"--bind",
"-b",
required=False,
default=Service.HOST,
help="IP to start the application.",
)
arguments = vars(ap.parse_args())
logger.info(f"App Starting at {arguments['bind']}:{arguments['port']}")
uvicorn.run("main:app", host=arguments["bind"], port=int(arguments["port"]))
conf/application.conf
View file @ f8692e94
# To specify a variable to be read from environment, use $<env_name> # To specify a variable to be read from environment, use $<env_name>
[service] [SERVICE]
host=0.0.0.0 host=0.0.0.0
port=6869 port=6869
workers=1 workers=1
...@@ -14,16 +14,17 @@ allow_cross_origin = true ...@@ -14,16 +14,17 @@ allow_cross_origin = true
level=DEBUG level=DEBUG
[postgres] [POSTGRES]
uri=$OEE_POSTGRES_URI uri=$OEE_POSTGRES_URI
downtime_uri=$DOWNTIME_URI
[mongo]
[MONGO]
uri=$MONGO_URI uri=$MONGO_URI
[PATH_TO_SERVICES] [PATH_TO_SERVICES]
event_explorer=$EVENT_EXPLORER event_explorer=$EVENT_EXPLORER
metadata_proxy=$METADATA_PROXY metadata_proxy=$METADATA_PROXY
downtime_proxy=$DOWNTIME_PROXY
[DIRECTORY] [DIRECTORY]
base_path = $BASE_PATH base_path = $BASE_PATH
...@@ -34,4 +35,20 @@ keys_path = data/keys ...@@ -34,4 +35,20 @@ keys_path = data/keys
uri=$REDIS_URI uri=$REDIS_URI
login_db = 9 login_db = 9
project_tags_db = 18 project_tags_db = 18
downtime_db=30 downtime_db=30
\ No newline at end of file oee_production_db=31
live_tags_db=12
[MQTT]
host = $MQTT_URL
port = $MQTT_PORT
publish_base_topic = ilens/notifications
[KAFKA]
host=$KAFKA_HOST
port=$KAFKA_PORT
topic=$KAFKA_TOPIC
enable_sites_partition=$ENABLE_KAFKA_PARTITION
split_key=$KAFKA_PARTITION_KEY
round_robin_enable=$ROUND_ROBIN_PARTITION
partition_db=13
\ No newline at end of file
data/keys/private 0 → 100644
View file @ f8692e94
-----BEGIN RSA PRIVATE KEY-----
MIICWwIBAAKBgQClilTaeHq6Zc+kWHCNl1O0btGRm7ct3O5zqWx1mwwLUWH14eft
Hi5wIbOYh79JQ9BO2OA4UjPq31uwmJ96Okl0OULfENhwd/D7P3mnoRlktPT2t+tt
RRrKvx3wNpOy/3nBsXnNt8EKxyA7k9vbqLbv9pGw2hcqOYe/NGTkmm1PswIDAQAB
AoGAZPARR1l5NBkKYGKQ1rU0E+wSmx+AtVVmjF39RUSyNmB8Q+poebwSgsr58IKt
T6Yq6Tjyl0UAZTGmferCK0xJJrqyP0hMn4nNNut+acWMKyt+9YrA2FO+r5Jb9JuT
SK35xXnM4aZLGppgWJxRzctpIz+qkf6oLRSZme0AuiqcwYECQQDY+QDL3wbWplRW
bze0DsZRMkDAkNY5OCydvjte4SR/mmAzsrpNrS5NztWbaaQrefoPbsdYBPbd8rS7
C/s/0L1zAkEAw1EC5zt2STuhkcKLa/tL+bk8WHHHtf19aC9kBj1TvWBFh+JojWCo
86iK5fLcHzhyQx5Qi3E9LG2HvOWhS1iUwQJAKbEHHyWW2c4SLJ2oVXf1UYrXeGkc
UNhjclgobl3StpZCYAy60cwyNo9E6l0NR7FjhG2j7lzd1t4ZLkvqFmQU0wJATLPe
yQIwBLh3Te+xoxlQD+Tvzuf3/v9qpWSfClhBL4jEJYYDeynvj6iry3whd91J+hPI
m8o/tNfay5L+UcGawQJAAtbqQc7qidFq+KQYLnv5gPRYlX/vNM+sWstUAqvWdMze
JYUoTHKgiXnSZ4mizI6/ovsBOMJTb6o1OJCKQtYylw==
-----END RSA PRIVATE KEY-----
data/keys/public 0 → 100644
View file @ f8692e94
-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQClilTaeHq6Zc+kWHCNl1O0btGR
m7ct3O5zqWx1mwwLUWH14eftHi5wIbOYh79JQ9BO2OA4UjPq31uwmJ96Okl0OULf
ENhwd/D7P3mnoRlktPT2t+ttRRrKvx3wNpOy/3nBsXnNt8EKxyA7k9vbqLbv9pGw
2hcqOYe/NGTkmm1PswIDAQAB
-----END PUBLIC KEY-----
main.py
View file @ f8692e94
import json
import os import os
from dataclasses import dataclass, field
from typing import Optional
from fastapi import FastAPI, Request, Response from fastapi import FastAPI, Depends
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from scripts.services.calculate_oee import calc_oee_router from scripts.db.psql.create_default_tables import create_default_psql_dependencies
from scripts.services.form_services import form_router from scripts.utils.security_utils.decorators import CookieAuthentication
from scripts.services.meta_services import meta_service_router
from scripts.services.ui_services import ui_service_router auth = CookieAuthentication()
app = FastAPI(
title="iLens OEE V3", @dataclass
version="5.5.0", class FastAPIConfig:
description="OEE App", title: str = "OEE SERVICES"
docs_url=os.environ.get("SW_DOCS_URL"), description: str = "A comprehensive tool for calculating oee"
openapi_url=os.environ.get("SW_OPENAPI_URL") docs_url: str = os.environ.get("SW_DOCS_URL")
) redoc_url: str = field(default=None)
if os.environ.get("CORS_URLS").split(","): # root_path: str = os.environ.get("MODULE_PROXY", default="/d_oee")
openapi_url: Optional[str] = os.environ.get("SW_OPENAPI_URL")
SECURE_ACCESS = True if os.environ.get("SECURE_ACCESS") in {'true', 'True', True} else False
if SECURE_ACCESS:
app_config_dict = FastAPIConfig().__dict__
app_config_dict.update({"dependencies": [Depends(auth)]})
app = FastAPI(**app_config_dict)
else:
app = FastAPI(**FastAPIConfig().__dict__)
if os.environ.get("ENABLE_CORS") in (True, "true", "True") and os.environ.get(
"CORS_URLS"
):
app.add_middleware( app.add_middleware(
CORSMiddleware, CORSMiddleware,
allow_origins=os.environ.get("CORS_URLS").split(","), allow_origins=os.environ.get("CORS_URLS").split(","),
allow_credentials=True, allow_credentials=True,
allow_methods=["GET", "POST", "DELETE", "PUT"], allow_methods=["GET", "POST", "DELETE", "PUT"],
allow_headers=["*"] allow_headers=["*"],
) )
print(os.environ.get("CORS_URLS").split(","))
@app.middleware("http") @app.on_event("startup")
async def add_process_time_header(request: Request, call_next): async def startup_event():
cookies = request.cookies """
response: Response = await call_next(request) This function is to create default database and tables,
response.headers.append("Cookie", json.dumps(cookies)) """
return response create_default_psql_dependencies()
app.include_router(calc_oee_router)
app.include_router(ui_service_router)
app.include_router(meta_service_router)
app.include_router(form_router)
production_monitoring.py
View file @ f8692e94
import json
import time import time
from datetime import datetime from datetime import datetime
from typing import List
import pytz import pytz
from scripts.config import read_settings, DBConf from scripts.constants import OEETagMappingKeys
from scripts.core.engine.automation_engine import AutomationEngine from scripts.core.engine.automation_engine import AutomationEngine
from scripts.db.mongo.dbs.siam_oee import SiamOEE from scripts.db.redis_connections import live_tags_db
from scripts.db.redis_conn import live_tags_db_object from scripts.logging import logger
from scripts.utils.common_utils import CommonUtils
from scripts.utils.db_name_util import get_project_specific_key
class ProductionMonitor: class ProductionMonitor:
def __init__(self): def __init__(self, project_id=None):
self.automation_engine = AutomationEngine() self.automation_engine = AutomationEngine()
self.settings = read_settings() self.common_util = CommonUtils()
self.oee_mongo = SiamOEE()
self.machine_mode_tags = self.settings["automation"]["setup_time_logic"]["tags"]
self.manual_mode_tag = self.settings["automation"]["setup_time_logic"]["manual_mode_tag"]
self.auto_mode_tag = self.settings["automation"]["setup_time_logic"]["auto_mode_tag"]
self.production_count_tags = self.settings["automation"]["production_end_logic"]["tags"]
self.good_count_tag = self.settings["automation"]["production_end_logic"]["good_count_tag"]
self.inspection_count_tag = self.settings["automation"]["production_end_logic"]["inspection_count_tag"]
self.ng_count_tag = self.settings["automation"]["production_end_logic"]["ng_count_tag"]
@staticmethod @staticmethod
def get_redis_data(tag_list): def get_redis_data(tags_list: List, project_id: str):
tag_data = {} tag_data = {}
redis_response = live_tags_db_object.mget(tag_list) redis_project_prefix = get_project_specific_key(project_id=project_id)
for index, each_tag in enumerate(tag_list): updated_tag_list = [f'{redis_project_prefix}{_tag}' for _tag in tags_list]
_val = redis_response.__getitem__(index) redis_response = live_tags_db.mget(updated_tag_list)
if not _val: for _value in redis_response:
continue tag_data.update(zip(updated_tag_list, _value))
_redis_resp = json.loads(_val)
hierarchy = each_tag.removeprefix(DBConf.REDIS_PREFIX)
if hierarchy not in tag_data:
tag_data[hierarchy] = dict()
tag_data[hierarchy] = _redis_resp.get("value")
return tag_data return tag_data
def calculate_oee_params(self, data, downtime): @staticmethod
start_time = datetime.fromtimestamp( def calculate_setup_time(production_start_time: datetime, tz, data):
data.get("start_time") // 1000, tz=pytz.timezone("Asia/Bangkok"))
end_time = datetime.now(tz=pytz.timezone("Asia/Bangkok"))
available_time = (end_time - start_time).total_seconds() / 60
if downtime > available_time:
downtime = 0
operating_time = available_time - downtime
availability = operating_time / available_time
good_count, units_produced = self.get_current_produced_count()
if not good_count:
good_count = 0
if not units_produced:
units_produced = 0
productive_time = units_produced * (1 / data.get("cycle_time"))
performance = productive_time / operating_time
if units_produced:
quality = good_count / units_produced
else:
quality = 0
oee = availability * performance * quality
return oee * 100, availability * 100, performance * 100, quality * 100
def calculate_setup_time(self, production_start_time: datetime, tz):
tag_list = [f"{DBConf.REDIS_PREFIX}{i}" for i in self.machine_mode_tags]
while True: while True:
try: try:
if (datetime.now(tz=pytz.timezone(tz)) - production_start_time).total_seconds() > 600: if (datetime.now(tz=pytz.timezone(tz)) - production_start_time).total_seconds() > 600:
return round((datetime.now(tz=pytz.timezone(tz)) - production_start_time).total_seconds() / 60) return round((datetime.now(tz=pytz.timezone(tz)) - production_start_time).total_seconds() / 60)
tag_data = self.get_redis_data(tag_list) if data.get(OEETagMappingKeys.OEE_MANUAL_MODE_TAG) == 0 and data.get(
OEETagMappingKeys.OEE_AUTO_MODE_TAG) == 1:
if tag_data.get(self.manual_mode_tag) == 0 and tag_data.get(self.auto_mode_tag) == 1:
print("production started!!!") print("production started!!!")
return round((datetime.now(tz=pytz.timezone(tz)) - production_start_time).total_seconds() / 60) return round((datetime.now(tz=pytz.timezone(tz)) - production_start_time).total_seconds() / 60)
print(tag_data)
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
def check_mongo_for_finish(self, job_id, machine_type) -> bool: def calculate_oee_params(self, data, tz):
data = self.oee_mongo.find_record(job_id, machine_type) try:
if data.get("prod_status", "") == "completed": start_time = datetime.fromtimestamp(
return True data.get("prod_start_time") // 1000, tz=pytz.timezone(tz))
else: end_time = datetime.now(tz=pytz.timezone(tz))
return False
available_time = (end_time - start_time).total_seconds() / 60
def check_production_end(self, data): # if downtime > available_time:
total_count = data.get("actual_received_qty", data.get("qty_released")) downtime = self.common_util.get_downtime_details_by_hierarchy(hierarchy=data["hierarchy"],
if total_count <= 0: project_id=data["project_id"],
total_count = data.get("qty_released") user_id=data.get("trigger_by"))
job_id = data.get("job", "") operating_time = available_time - downtime
machine_type = data.get("uf_process", "") oee_tags_list = data.get("oee_tags_list")
tag_list = [f"{DBConf.REDIS_PREFIX}{i}" for i in self.production_count_tags] project_id = data.get("project_id")
while True: if not oee_tags_list:
if self.check_mongo_for_finish(job_id, machine_type): return
break tags_data = self.get_redis_data(tags_list=oee_tags_list, project_id=project_id)
try: good_count, units_produced = self.get_current_produced_count(input_data=tags_data)
tag_data = self.get_redis_data(tag_list) if not good_count:
if tag_data.get(self.good_count_tag, 0) >= total_count \ good_count = 0
or tag_data.get(self.inspection_count_tag, 0) >= total_count \
or tag_data.get(self.inspection_count_tag, -1) == 0 \ if not units_produced:
or tag_data.get(self.good_count_tag, -1) == 0: units_produced = 0
print("production ended")
break productive_time = units_produced * (1 / data.get("cycle_time"))
time.sleep(1)
except Exception as e: performance = productive_time / operating_time
print(e) availability = operating_time / available_time
return True
if units_produced:
quality = good_count / units_produced
else:
quality = 0
oee = availability * performance * quality
return oee * 100, availability * 100, performance * 100, quality * 100
except Exception as e:
logger.exception(f"Exception occurred while updating the production batch {e.args[0]}")
def check_production_run(self): @staticmethod
tag_list = [f"{DBConf.REDIS_PREFIX}{i}" for i in self.production_count_tags] def check_production_run(input_data: dict):
while True: while True:
try: try:
tag_data = self.get_redis_data(tag_list) if input_data.get(OEETagMappingKeys.OEE_INSPECTION_TAG, 0) > 0:
if tag_data.get(self.inspection_count_tag) > 0:
return True return True
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
print(e) print(e)
def get_current_produced_count(self): @staticmethod
tag_list = [f"{DBConf.REDIS_PREFIX}{i}" for i in self.production_count_tags] def get_current_produced_count(input_data: dict):
try: try:
tag_data = self.get_redis_data(tag_list)
return ( return (
tag_data.get(self.good_count_tag), input_data.get(OEETagMappingKeys.OEE_GOOD_COUNT_TAG),
tag_data.get(self.ng_count_tag, 0) + tag_data.get(self.good_count_tag, 0) input_data.get(OEETagMappingKeys.OEE_NG_TAG, 0) + input_data.get(OEETagMappingKeys.OEE_GOOD_COUNT_TAG,
0)
) )
except Exception as e: except Exception as e:
print(e) print(e)
requirements.txt
View file @ f8692e94
cryptography~=36.0.1 cryptography~=36.0.1
crypto~=1.4.1 crypto~=1.4.1
fastapi~=0.74.1 fastapi==0.75.2
httpx~=0.22.0 httpx~=0.19.0
ilens-kafka-publisher==0.4.2 ilens-kafka-publisher==0.4.2
kafka-python~=2.0.2 kafka-python~=2.0.2
pandas~=1.4.1 pandas==1.4.2
psycopg2-binary~=2.9.3 psycopg2-binary~=2.9.3
pydantic~=1.9.0 pydantic==1.9.0
pyjwt~=2.3.0 PyJWT==2.3.0
pymongo~=4.0.1 pytz==2022.1
pymongo~=3.7.2
pymssql~=2.2.4 pymssql~=2.2.4
pytest==3.2.4 pytest==3.2.4
python-dotenv~=0.19.2 python-dotenv~=0.19.2
pytz~=2021.3 pyyaml~=5.1
pyyaml~=6.0 redis==4.2.2
redis~=4.1.4 shortuuid==1.0.8
requests==2.26.0 SQLAlchemy==1.4.35
sqlalchemy-utils~=0.38.2 SQLAlchemy-Utils==0.38.2
sqlalchemy~=1.3.24 uvicorn~=0.17.5
uvicorn~=0.17.5 argparse~=1.4.0
\ No newline at end of file DateTime~=4.3
paho-mqtt~=1.5.0
pendulum~=2.1.2
requests~=2.27.1
aiohttp~=3.8.1
numpy==1.22.3
\ No newline at end of file
scripts/config/__init__.py
View file @ f8692e94
...@@ -37,21 +37,18 @@ except Exception as e: ...@@ -37,21 +37,18 @@ except Exception as e:
class Service: class Service:
HOST = config.get("service", "host") HOST = config.get("SERVICE", "host")
PORT = config.getint("service", "port") PORT = config.getint("SERVICE", "port")
WORKERS = config.getint("service", "workers") WORKERS = config.getint("SERVICE", "workers")
secure_cookie = config.get("SERVICE", "secure_cookie")
class DBConf: class DBConf:
POSTGRES_URI = config.get('postgres', 'uri') POSTGRES_URI = config.get('POSTGRES', 'uri')
if not POSTGRES_URI: if not POSTGRES_URI:
print("Error, environment variable OEE_POSTGRES_URI not set") print("Error, environment variable OEE_POSTGRES_URI not set")
sys.exit(1) sys.exit(1)
DOWNTIME_URI = config.get('postgres', 'downtime_uri') MONGO_URI = config.get('MONGO', 'uri')
if not DOWNTIME_URI:
print("Error, environment variable DOWNTIME_URI not set")
sys.exit(1)
MONGO_URI = config.get('mongo', 'uri')
if not MONGO_URI: if not MONGO_URI:
print("Error, environment variable MONGO_URI not set") print("Error, environment variable MONGO_URI not set")
sys.exit(1) sys.exit(1)
...@@ -73,10 +70,12 @@ class PathToStorage(object): ...@@ -73,10 +70,12 @@ class PathToStorage(object):
class RedisConfig(object): class RedisConfig(object):
uri = config.get("REDIS", "uri") REDIS_URI = config.get("REDIS", "uri")
login_db = config["REDIS"]["login_db"] LOGIN_DB = config["REDIS"]["login_db"]
project_tags_db = config.getint("REDIS", "project_tags_db") PROJECT_TAGS_DB = config.getint("REDIS", "project_tags_db")
downtime_db = config.getint("REDIS", "downtime_db") DOWNTIME_DB = config.getint("REDIS", "downtime_db")
OEE_PRODUCTION_DB = config.getint("REDIS", "oee_production_db")
LIVE_TAGS_DB = config.getint("REDIS", "live_tags_db")
class KeyPath(object): class KeyPath(object):
...@@ -110,3 +109,21 @@ class MQTTConf: ...@@ -110,3 +109,21 @@ class MQTTConf:
host = config["MQTT"]["host"] host = config["MQTT"]["host"]
port = int(config["MQTT"]["port"]) port = int(config["MQTT"]["port"])
publish_base_topic = config["MQTT"]["publish_base_topic"] publish_base_topic = config["MQTT"]["publish_base_topic"]
class KafkaConf:
host = config.get('KAFKA', 'host')
port = config.get('KAFKA', 'port')
topic = config.get('KAFKA', 'topic')
if not any([topic, host, port]):
print("KAFKA env variables missing, continuing without Kafka/Kairos support")
enable_sites_partition = config.getboolean("KAFKA", "ENABLE_KAFKA_PARTITION", fallback=True)
split_key = config["KAFKA"].get('KAFKA_PARTITION_KEY', 'site_id')
round_robin_enable = config.getboolean("KAFKA", "ROUND_ROBIN_PARTITION", fallback=True)
redis_db = config.getint("KAFKA", "partition_db")
class PathToServices:
event_explorer = config["PATH_TO_SERVICES"]["event_explorer"]
metadata_proxy = config["PATH_TO_SERVICES"]["metadata_proxy"]
downtime_proxy = config["PATH_TO_SERVICES"]["downtime_proxy"]
scripts/constants/__init__.py
View file @ f8692e94
import os
from collections import namedtuple from collections import namedtuple
...@@ -59,3 +60,38 @@ class DBConstants: ...@@ -59,3 +60,38 @@ class DBConstants:
db_ilens_assistant = "ilens_assistant" db_ilens_assistant = "ilens_assistant"
collection_constants = "constants" collection_constants = "constants"
lookup_table = "lookup_table"
class CommonConstants:
ui = 'ui_datetime_format'
utc = 'utc_datetime_format'
nsc = 'no_special_chars_datetime_format'
umtf = 'user_meta_time_format'
__temporary_format__ = '%Y-%m-%dT%H:%M:%S+0530'
__iso_format__ = '%Y-%m-%dT%H:%M:%S%z'
__form_iso_format__ = '%Y-%m-%dT%H:%M:%S.%f%z'
__utc_datetime_format__ = '%Y-%m-%dT%H:%M:%S'
__ui_datetime_format__ = '%Y-%m-%d %H:%M:%S'
__no_special_chars_datetime_format__ = '%Y%m%d%H%M%S'
__user_meta_time_format__ = "%d-%m-%Y %H:%M:%S"
__user_meta_time_format_ws__ = "%d-%m-%Y %H:%M"
__ui_display_datetime_format__ = "%d %b %Y %H:%M"
__ui_display_date_format__ = "%d %b %Y"
__time_format__ = "%H:%M:%S"
class OEETagMappingKeys:
OEE_GOOD_COUNT_TAG = os.environ.get("OEE_GOOD_TAG", default="good_count_tag")
OEE_INSPECTION_TAG = os.environ.get("OEE_INSPECTION_TAG", default="oee_inspection_tag")
OEE_MANUAL_MODE_TAG = os.environ.get("OEE_INSPECTION_TAG", default="oee_manual_mode_tag")
OEE_AUTO_MODE_TAG = os.environ.get("OEE_INSPECTION_TAG", default="oee_auto_mode_tag")
OEE_NG_TAG = os.environ.get("OEE_NG_TAG", default="oee_ng_tag")
class CommonStatusCode:
SUCCESS_CODES = (
200,
201,
204,
)
scripts/core/engine/automation_engine.py 0 → 100644
View file @ f8692e94
from scripts.db.redis_connections import oee_production_db
class AutomationEngine:
def __init__(self, project_id=None):
...
@staticmethod
def get_activate_batches_from_redis():
final_dict = {}
keys = list(oee_production_db.keys())
values = oee_production_db.mget(keys)
for _value in values:
final_dict.update(zip(keys, _value))
return final_dict
scripts/core/engine/oee_aggregator.py
View file @ f8692e94
...@@ -6,8 +6,8 @@ from scripts.schemas.batch_oee import ChartResponse, ChartDBResponse ...@@ -6,8 +6,8 @@ from scripts.schemas.batch_oee import ChartResponse, ChartDBResponse
def processor(data): def processor(data):
db_response = ChartDBResponse(**data) db_response = ChartDBResponse(**data)
db_response.total_time = ( db_response.total_time = (
db_response.batch_end_time - db_response.batch_start_time db_response.batch_end_time - db_response.batch_start_time
) / 60000 ) / 60000
db_response.actual_cycle = round( db_response.actual_cycle = round(
db_response.total_units / db_response.total_time, 2 db_response.total_units / db_response.total_time, 2
) )
...@@ -20,7 +20,7 @@ def processor(data): ...@@ -20,7 +20,7 @@ def processor(data):
return chart_response.dict() return chart_response.dict()
def aggregator(data, activity_length): def aggregator(data, activity_length=1):
df = pd.DataFrame(data) df = pd.DataFrame(data)
df["total_time"] = (df["batch_end_time"] - df["batch_start_time"]) / 60000 df["total_time"] = (df["batch_end_time"] - df["batch_start_time"]) / 60000
df["actual_cycle"] = df["total_units"] / df["total_time"] df["actual_cycle"] = df["total_units"] / df["total_time"]
......
scripts/core/engine/oee_calculator.py
View file @ f8692e94
import time from datetime import datetime
from scripts.errors import ErrorCodes
from scripts.constants import UOM from scripts.constants import UOM
from scripts.errors import ErrorCodes
from scripts.logging import logger from scripts.logging import logger
from scripts.schemas.batch_oee import BatchOEEDataSaveRequest, BatchOEEData from scripts.schemas.batch_oee import OEEDataSaveRequest, BatchOEEData
from scripts.utils.common_utils import CommonUtils
class OEECalculator: class OEECalculator:
def __init__(self):
self.common_utils = CommonUtils()
@staticmethod @staticmethod
async def calculate_availability(operating_time, planned_prod_time): def calculate_availability(operating_time, planned_prod_time):
if operating_time > planned_prod_time: if operating_time > planned_prod_time:
logger.error(ErrorCodes.ERR001) logger.error(ErrorCodes.ERR001)
raise ValueError(ErrorCodes.ERR001) raise ValueError(ErrorCodes.ERR001)
...@@ -20,7 +23,7 @@ class OEECalculator: ...@@ -20,7 +23,7 @@ class OEECalculator:
raise raise
@staticmethod @staticmethod
async def calculate_performance(units_produced, cycle_time, operating_time): def calculate_performance(units_produced, cycle_time, operating_time):
try: try:
if cycle_time == 0 or operating_time == 0: if cycle_time == 0 or operating_time == 0:
logger.error(ErrorCodes.ERR002) logger.error(ErrorCodes.ERR002)
...@@ -35,7 +38,7 @@ class OEECalculator: ...@@ -35,7 +38,7 @@ class OEECalculator:
raise raise
@staticmethod @staticmethod
async def calculate_productive_time(units_produced, cycle_time): def calculate_productive_time(units_produced, cycle_time):
try: try:
if cycle_time == 0: if cycle_time == 0:
logger.error(ErrorCodes.ERR002) logger.error(ErrorCodes.ERR002)
...@@ -47,7 +50,7 @@ class OEECalculator: ...@@ -47,7 +50,7 @@ class OEECalculator:
raise raise
@staticmethod @staticmethod
async def calculate_quality(rejected_units, total_units): def calculate_quality(rejected_units, total_units):
if rejected_units > total_units: if rejected_units > total_units:
logger.error(ErrorCodes.ERR004) logger.error(ErrorCodes.ERR004)
raise ValueError(ErrorCodes.ERR004) raise ValueError(ErrorCodes.ERR004)
...@@ -60,7 +63,7 @@ class OEECalculator: ...@@ -60,7 +63,7 @@ class OEECalculator:
raise raise
@staticmethod @staticmethod
async def calculate_oee(availability, performance, quality): def calculate_oee(availability, performance, quality):
try: try:
return availability * performance * quality return availability * performance * quality
except Exception as e: except Exception as e:
...@@ -70,16 +73,16 @@ class OEECalculator: ...@@ -70,16 +73,16 @@ class OEECalculator:
class OEELossesCalculator: class OEELossesCalculator:
@staticmethod @staticmethod
async def calculate_availability_loss(downtime, available_time): def calculate_availability_loss(downtime, available_time):
return (downtime / available_time) * 100 return (downtime / available_time) * 100
@staticmethod @staticmethod
async def calculate_quality_loss(reject_units, cycle_time, available_time): def calculate_quality_loss(reject_units, cycle_time, available_time):
return ((reject_units * (1 / cycle_time)) / available_time) * 100 return ((reject_units * (1 / cycle_time)) / available_time) * 100
@staticmethod @staticmethod
async def calculate_performance_loss( def calculate_performance_loss(
oee_percentage, availability_loss, quality_loss oee_percentage, availability_loss, quality_loss
): ):
return 100 - availability_loss - quality_loss - oee_percentage return 100 - availability_loss - quality_loss - oee_percentage
...@@ -88,71 +91,71 @@ class OEEEngine: ...@@ -88,71 +91,71 @@ class OEEEngine:
def __init__(self): def __init__(self):
self.oee_calc = OEECalculator() self.oee_calc = OEECalculator()
self.oee_loss_calc = OEELossesCalculator() self.oee_loss_calc = OEELossesCalculator()
self.common_util = CommonUtils()
async def start_batch_oee_calc( def start_batch_oee_calc(
self, self,
product_info: BatchOEEDataSaveRequest request_data: OEEDataSaveRequest
) -> BatchOEEData: ) -> BatchOEEData:
try: try:
logger.debug(f"Calculating OEE for {product_info.batch_id}") logger.debug(f"Calculating OEE for {request_data.reference_id}")
# Start and End time should be in milliseconds since epoch. # Start and End time should be in milliseconds since epoch.
if product_info.uom == UOM.minutes: if request_data.uom == UOM.minutes:
divisor = UOM.time_divs.minutes divisor = UOM.time_divs.minutes
elif product_info.uom == UOM.seconds: elif request_data.uom == UOM.seconds:
divisor = UOM.time_divs.seconds divisor = UOM.time_divs.seconds
elif product_info.uom == UOM.hours: elif request_data.uom == UOM.hours:
divisor = UOM.time_divs.hours divisor = UOM.time_divs.hours
elif product_info.uom == UOM.millis: elif request_data.uom == UOM.millis:
divisor = UOM.time_divs.millis divisor = UOM.time_divs.millis
else: else:
divisor = UOM.time_divs.minutes divisor = UOM.time_divs.minutes
planned_production_time = ( planned_production_time = self.common_util.get_duration(tz=request_data.tz, meta=request_data.dict(),
product_info.batch_end_time - product_info.batch_start_time difference=True) / divisor
) / divisor
operating_time = planned_production_time - product_info.downtime operating_time = planned_production_time - request_data.downtime
availability = await self.oee_calc.calculate_availability( availability = self.oee_calc.calculate_availability(
operating_time=operating_time, operating_time=operating_time,
planned_prod_time=planned_production_time, planned_prod_time=planned_production_time,
) )
performance = await self.oee_calc.calculate_performance( performance = self.oee_calc.calculate_performance(
units_produced=product_info.total_units, units_produced=request_data.total_units,
operating_time=operating_time, operating_time=operating_time,
cycle_time=product_info.cycle_time, cycle_time=request_data.cycle_time,
) )
quality = await self.oee_calc.calculate_quality( quality = self.oee_calc.calculate_quality(
total_units=product_info.total_units, total_units=request_data.total_units,
rejected_units=product_info.reject_units, rejected_units=request_data.reject_units,
) )
oee = await self.oee_calc.calculate_oee( oee = self.oee_calc.calculate_oee(
availability=availability, availability=availability,
performance=performance, performance=performance,
quality=quality, quality=quality,
) )
productive_time = await self.oee_calc.calculate_productive_time( productive_time = self.oee_calc.calculate_productive_time(
cycle_time=product_info.cycle_time, cycle_time=request_data.cycle_time,
units_produced=product_info.total_units, units_produced=request_data.total_units,
) )
availability_loss = await self.oee_loss_calc.calculate_availability_loss( availability_loss = self.oee_loss_calc.calculate_availability_loss(
downtime=product_info.downtime, downtime=request_data.downtime,
available_time=planned_production_time, available_time=planned_production_time,
) )
quality_loss = await self.oee_loss_calc.calculate_quality_loss( quality_loss = self.oee_loss_calc.calculate_quality_loss(
reject_units=product_info.reject_units, reject_units=request_data.reject_units,
available_time=planned_production_time, available_time=planned_production_time,
cycle_time=product_info.cycle_time, cycle_time=request_data.cycle_time,
) )
performance_loss = await self.oee_loss_calc.calculate_performance_loss( performance_loss = self.oee_loss_calc.calculate_performance_loss(
oee_percentage=oee * 100, oee_percentage=oee * 100,
availability_loss=availability_loss, availability_loss=availability_loss,
quality_loss=quality_loss, quality_loss=quality_loss,
...@@ -170,12 +173,12 @@ class OEEEngine: ...@@ -170,12 +173,12 @@ class OEEEngine:
"performance_loss": performance_loss, "performance_loss": performance_loss,
} }
logger.debug(f"OEE: {product_info.batch_id}: {oee_dict}") logger.debug(f"OEE: {request_data.reference_id}: {oee_dict}")
logger.debug(f"OEE Loss: {product_info.batch_id}: {oee_loss}") logger.debug(f"OEE Loss: {request_data.reference_id}: {oee_loss}")
batch_oee = BatchOEEData( batch_oee = BatchOEEData(
**product_info.dict(), **request_data.dict(),
calculated_on=int(time.time() * 1000), calculated_on=datetime.now().astimezone(tz=request_data.tz).isoformat(),
productive_time=productive_time, productive_time=productive_time,
availability=availability * 100, availability=availability * 100,
performance=performance * 100, performance=performance * 100,
......
scripts/core/handlers/api_handler.py
View file @ f8692e94
...@@ -3,9 +3,9 @@ import time ...@@ -3,9 +3,9 @@ import time
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from scripts.core.engine.chart_creators import ChartMaker from scripts.core.engine.chart_creators import ChartMaker
from scripts.core.engine.oee_aggregator import processor, aggregator from scripts.core.engine.oee_aggregator import processor, aggregator
from scripts.db_layer import batch_oee_table from scripts.db.psql.oee_discrete import DiscreteOEE
from scripts.logging.logging import logger from scripts.logging import logger
from scripts.schemas.batch_oee import ( from scripts.schemas.batch_oee import (
GetOEERequest, GetOEERequest,
GetOEERequestOneBatch, GetOEERequestOneBatch,
...@@ -14,63 +14,57 @@ from scripts.schemas.batch_oee import ( ...@@ -14,63 +14,57 @@ from scripts.schemas.batch_oee import (
ChartRequest, ChartRequest,
) )
from scripts.schemas.meta import LabelValue from scripts.schemas.meta import LabelValue
from scripts.utils.mongo_util import MongoConnect
class APIHandler: class APIHandler:
@staticmethod @staticmethod
async def get_oee_all(db: Session, get_oee_request: GetOEERequest): def get_oee_all(db: Session, get_oee_request: GetOEERequest):
table_obj = DiscreteOEE(db=db)
try: try:
table_obj = batch_oee_table.BatchOEETable(db=db)
data = table_obj.get_oee_data_all( data = table_obj.get_oee_data_all(
start_time=get_oee_request.start_time, prod_start_time=get_oee_request.prod_start_time,
end_time=get_oee_request.end_time, prod_end_time=get_oee_request.prod_end_time,
hierarchy=get_oee_request.hierarchy, hierarchy=get_oee_request.hierarchy,
) )
return data return data
except Exception: except Exception:
raise raise
finally:
del table_obj
@staticmethod @staticmethod
async def get_oee_batch(db: Session, get_oee_request: GetOEERequestOneBatch): def get_oee_batch(db: Session, get_oee_request: GetOEERequestOneBatch):
table_obj = DiscreteOEE(db=db)
try: try:
table_obj = batch_oee_table.BatchOEETable(db=db) data = table_obj.get_oee_data_by_reference_id(
data = table_obj.get_oee_data_batch_id( hierarchy=get_oee_request.hierarchy, reference_id=get_oee_request.reference_id,
hierarchy=get_oee_request.hierarchy, batch_id=get_oee_request.batch_id project_id=get_oee_request.project_id
) )
return data return data
except Exception: except Exception:
raise raise
finally:
del table_obj
@staticmethod @staticmethod
async def get_batches(db: Session, request_data: GetBatches): def get_batches(db: Session, request_data: GetBatches):
table_obj = DiscreteOEE(db=db)
try: try:
table_obj = batch_oee_table.BatchOEETable(db=db) if not request_data.prod_end_time:
if not request_data.end_time: request_data.prod_end_time = int(time.time() * 1000)
request_data.end_time = int(time.time() * 1000)
data = table_obj.get_batches( data = table_obj.get_batches(
hierarchy=request_data.hierarchy, hierarchy=request_data.hierarchy,
start_time=request_data.start_time, prod_start_time=request_data.prod_start_time,
end_time=request_data.end_time, prod_end_time=request_data.prod_end_time,
) )
return data return data
except Exception: except Exception:
raise raise
finally:
del table_obj
@staticmethod @staticmethod
async def get_products(db: Session, request_data: GetProducts): async def get_products(db: Session, request_data: GetProducts):
table_obj = DiscreteOEE(db=db)
try: try:
table_obj = batch_oee_table.BatchOEETable(db=db)
data = table_obj.get_products( data = table_obj.get_products(
hierarchy=request_data.hierarchy, hierarchy=request_data.hierarchy,
start_time=request_data.queryDate[0], prod_start_time=request_data.queryDate[0],
end_time=request_data.queryDate[1], prod_end_time=request_data.queryDate[1],
) )
return [ return [
...@@ -86,21 +80,19 @@ class APIHandler: ...@@ -86,21 +80,19 @@ class APIHandler:
except Exception as e: except Exception as e:
logger.exception(e, exc_info=True) logger.exception(e, exc_info=True)
raise raise
finally:
del table_obj
@staticmethod @staticmethod
async def get_chart_data(db: Session, request_data: ChartRequest): async def get_chart_data(db: Session, request_data: ChartRequest):
table_obj = DiscreteOEE(db=db)
try: try:
table_obj = batch_oee_table.BatchOEETable(db=db)
if not request_data.hierarchy: if not request_data.hierarchy:
return dict() return dict()
chart_maker = ChartMaker() chart_maker = ChartMaker()
data = table_obj.get_chart_data( data = table_obj.get_chart_data(
hierarchy=request_data.hierarchy, hierarchy=request_data.hierarchy,
start_time=request_data.queryDate[0], prod_start_time=request_data.queryDate[0],
end_time=request_data.queryDate[1], prod_end_time=request_data.queryDate[1],
product_id=request_data.productId, reference_id=request_data.productId,
aggregation=request_data.aggregation, aggregation=request_data.aggregation,
) )
if not request_data.aggregation or len(data) == 1: if not request_data.aggregation or len(data) == 1:
......
scripts/core/handlers/batch_oee_calc_handler.py
View file @ f8692e94
import json
from datetime import datetime
import pytz
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from scripts.constants import ResponseCodes from scripts.constants import ResponseCodes, CommonConstants
from scripts.core.engine.oee_calculator import OEEEngine from scripts.core.engine.oee_calculator import OEEEngine
from scripts.db.psql.oee_discrete import DiscreteOEE from scripts.db.psql.oee_discrete import DiscreteOEE
from scripts.errors import ILensError, ErrorCodes from scripts.db.redis_connections import oee_production_db
from scripts.logging import logger from scripts.logging import logger
from scripts.schemas.batch_oee import BatchOEEDataRequest, BatchOEEData, GetOEERequestOneBatch, BatchOEEDataSaveRequest from scripts.schemas.batch_oee import OEEDataInsertRequest, BatchOEEData, OEEDataSaveRequest
from scripts.schemas.response_models import DefaultResponse from scripts.schemas.response_models import DefaultResponse
oee_engine = OEEEngine() oee_engine = OEEEngine()
...@@ -13,25 +17,37 @@ oee_engine = OEEEngine() ...@@ -13,25 +17,37 @@ oee_engine = OEEEngine()
class CalculateBatchOEEHandler: class CalculateBatchOEEHandler:
async def calculate_oee(self, db, product_info: BatchOEEDataRequest): def calculate_oee(self, db, request_data: OEEDataInsertRequest):
table_obj = DiscreteOEE(db=db) table_obj = DiscreteOEE(db=db)
try: try:
record_presence = table_obj.get_oee_data_batch_id(batch_id=product_info.batch_id, record_presence = table_obj.get_oee_data_by_reference_id(reference_id=request_data.reference_id,
hierarchy=product_info.hierarchy) hierarchy=request_data.hierarchy,
project_id=request_data.project_id)
redis_key = f"{request_data.project_id}${request_data.reference_id}"
if not record_presence: if not record_presence:
if not product_info.batch_start_time: if not request_data.prod_start_time:
raise ILensError(ErrorCodes.ERR005) request_data.prod_start_time = datetime.now().astimezone(
product_info = BatchOEEDataSaveRequest(**product_info.dict(exclude_none=True)) tz=pytz.timezone(request_data.tz)).isoformat()
oee_calculation = await oee_engine.start_batch_oee_calc(product_info=product_info) request_data = OEEDataSaveRequest(**request_data.dict(exclude_none=True))
await self.save_oee_data(oee_calculation, db) oee_calculation = oee_engine.start_batch_oee_calc(request_data=request_data)
self.save_oee_data(oee_calculation, db)
oee_production_db.set(name=redis_key, value=json.dumps(
{"created_on": datetime.now().strftime(CommonConstants.__ui_datetime_format__)}))
response = DefaultResponse( response = DefaultResponse(
status=ResponseCodes.SUCCESS, status=ResponseCodes.SUCCESS,
data=oee_calculation, data=oee_calculation,
message="OEE saved Successfully", message="OEE saved Successfully",
) )
return response return response
status = await self.update_oee_data(product_info.dict(exclude_none=True), record_presence, db) status = self.update_oee_data(oee_data=BatchOEEData(**request_data.dict(exclude_none=True)),
old_record=record_presence, db=db)
if status:
if request_data.prod_end_time:
oee_production_db.delete(redis_key)
else:
oee_production_db.set(name=redis_key,
value=json.dumps(BatchOEEData(**request_data.dict(exclude_none=True))))
response = DefaultResponse( response = DefaultResponse(
status=ResponseCodes.SUCCESS, status=ResponseCodes.SUCCESS,
data=status, data=status,
...@@ -41,32 +57,35 @@ class CalculateBatchOEEHandler: ...@@ -41,32 +57,35 @@ class CalculateBatchOEEHandler:
except Exception as e: except Exception as e:
logger.exception(f"Exception while saving oee record: {e}") logger.exception(f"Exception while saving oee record: {e}")
raise e raise e
finally:
del table_obj
@staticmethod @staticmethod
async def save_oee_data(oee_data: BatchOEEData, db: Session): def save_oee_data(oee_data: BatchOEEData, db: Session):
table_obj = DiscreteOEE(db=db)
try: try:
oee_table = BatchOEETable(**oee_data.dict()) table_obj.insert_one(table=table_obj.table, insert_json=oee_data.dict())
table_obj = batch_oee_table.BatchOEETable(db=db)
table_obj.add_data(oee_table)
return True return True
except Exception as e: except Exception as e:
raise e raise e
finally:
del table_obj
@staticmethod @staticmethod
async def update_oee_data(product_info: dict, old_record: dict, db: Session): def update_oee_data(oee_data: BatchOEEData, old_record: dict, db: Session):
table_obj = DiscreteOEE(db=db)
try: try:
table_obj = batch_oee_table.BatchOEETable(db=db) old_record.update(**oee_data.dict(exclude_none=True))
old_record.update(**product_info) oee_calculation = oee_engine.start_batch_oee_calc(
oee_calculation = await oee_engine.start_batch_oee_calc( request_data=OEEDataSaveRequest(**old_record))
product_info=BatchOEEDataSaveRequest(**old_record)) table_obj.update(update_json=oee_calculation.dict(), table=table_obj.table, filters=[{'expression': 'eq',
filters = GetOEERequestOneBatch(batch_id=product_info["batch_id"], hierarchy=product_info["hierarchy"]) 'column': table_obj.table.project_id,
table_obj.update_record(filters=filters, update_obj=oee_calculation.dict()) 'value': oee_data.project_id
},
{'expression': 'eq',
'column': table_obj.table.reference_id,
'value': oee_data.reference_id
},
{'expression': 'eq',
'column': table_obj.table.hierarchy,
'value': oee_data.hierarchy
}], update_one=True)
return True return True
except Exception as e: except Exception as e:
raise e raise e
finally:
del table_obj
scripts/core/handlers/form_handler.py
View file @ f8692e94
...@@ -7,7 +7,7 @@ from sqlalchemy import create_engine ...@@ -7,7 +7,7 @@ from sqlalchemy import create_engine
from scripts.config import DBConf from scripts.config import DBConf
from scripts.config import Metadata from scripts.config import Metadata
from scripts.core.engine.automation_engine import AutomationEngine from scripts.core.engine.automation_engine import AutomationEngine
from scripts.core.handlers.batch_oee_calc_handler import CalculateBatchOEEHandler, BatchOEEDataRequest from scripts.core.handlers.batch_oee_calc_handler import CalculateBatchOEEHandler, OEEDataInsertRequest
from scripts.db.mongo.dbs.siam_oee import SiamOEE from scripts.db.mongo.dbs.siam_oee import SiamOEE
from scripts.db_layer.job_table import JobTable from scripts.db_layer.job_table import JobTable
from scripts.logging.logging import logger as logging from scripts.logging.logging import logger as logging
...@@ -83,15 +83,15 @@ class FormHandler: ...@@ -83,15 +83,15 @@ class FormHandler:
table_data = JobTable(db_session) table_data = JobTable(db_session)
job_model, db_data = await self.get_job_data(request_data) job_model, db_data = await self.get_job_data(request_data)
table_data.update_record(job_model.job, job_model.uf_process, db_data.dict(exclude_none=True)) table_data.update_record(job_model.job, job_model.uf_process, db_data.dict(exclude_none=True))
calculate_oee_payload = CalculateOEE(batch_start_time=job_model.start_time, calculate_oee_payload = CalculateOEE(batch_start_time=job_model.prod_start_time,
batch_end_time=job_model.end_time, batch_end_time=job_model.prod_end_time,
batch_id=job_model.job, batch_id=job_model.job,
setup_time=job_model.setup_time, setup_time=job_model.setup_time,
cycle_time=job_model.cycle_time, cycle_time=job_model.cycle_time,
total_units=job_model.qty_released) total_units=job_model.qty_released)
calculate_oee_payload.downtime = await self.get_oee_downtime(request_data.submitted_data["data"], calculate_oee_payload.downtime = await self.get_oee_downtime(request_data.submitted_data["data"],
job_model.end_time, job_model.tz) job_model.prod_end_time, job_model.tz)
_ = await CalculateBatchOEEHandler().calculate_oee(db_session, BatchOEEDataRequest( _ = await CalculateBatchOEEHandler().calculate_oee(db_session, OEEDataInsertRequest(
**calculate_oee_payload.dict())) **calculate_oee_payload.dict()))
form_response = await self.save_to_form(request_data, request_cookies, job_model) form_response = await self.save_to_form(request_data, request_cookies, job_model)
logging.info(f"FORM SAVE RESPONSE, {form_response}") logging.info(f"FORM SAVE RESPONSE, {form_response}")
...@@ -118,10 +118,10 @@ class FormHandler: ...@@ -118,10 +118,10 @@ class FormHandler:
async def save_to_form(self, request_data: EndProductionRequest, async def save_to_form(self, request_data: EndProductionRequest,
request_cookies, request_cookies,
job_model: EndProdJobModel): job_model: EndProdJobModel):
end_date_time = datetime.fromtimestamp(job_model.end_time // 1000, tz=pytz.timezone(request_data.tz)) end_date_time = datetime.fromtimestamp(job_model.prod_end_time // 1000, tz=pytz.timezone(request_data.tz))
end_str = end_date_time.strftime("%Y-%m-%d %H:%M") end_str = end_date_time.strftime("%Y-%m-%d %H:%M")
start_date_time = datetime.fromtimestamp(job_model.start_time // 1000, tz=pytz.timezone(request_data.tz)) start_date_time = datetime.fromtimestamp(job_model.prod_start_time // 1000, tz=pytz.timezone(request_data.tz))
start_str = start_date_time.strftime("%Y-%m-%d %H:%M") start_str = start_date_time.strftime("%Y-%m-%d %H:%M")
tag_data = self.automation_engine.get_all_tags(end_date_time) tag_data = self.automation_engine.get_all_tags(end_date_time)
......
scripts/db/db_models/__init__.py
View file @ f8692e94
# Copyright (c) NOIR from sqlalchemy import Column, Integer, String, Float, TIMESTAMP
from sqlalchemy import Column, Integer, String, BigInteger, Float
from scripts.db.psql.databases import Base from scripts.db.psql.databases import Base
...@@ -9,13 +8,15 @@ class OEEDiscreteTable(Base): ...@@ -9,13 +8,15 @@ class OEEDiscreteTable(Base):
id = Column(Integer, autoincrement=True, primary_key=True) id = Column(Integer, autoincrement=True, primary_key=True)
hierarchy = Column(String) hierarchy = Column(String)
process = Column(String, nullable=True) process = Column(String, nullable=True)
reference = Column(String) prod_status = Column(String, default="running")
prod_start_time = Column(BigInteger) reference_id = Column(String, nullable=True, unique=True)
prod_end_time = Column(BigInteger) prod_start_time = Column(TIMESTAMP(timezone=True), nullable=False)
prod_end_time = Column(TIMESTAMP(timezone=True), nullable=True)
total_downtime = Column(Float, default=0) total_downtime = Column(Float, default=0)
cycle_time = Column(Float) planned_units = Column(Float, nullable=False)
total_units = Column(Float) total_units = Column(Float)
reject_units = Column(Float, default=0) reject_units = Column(Float, default=0)
cycle_time = Column(Float)
productive_time = Column(Float, default=0) productive_time = Column(Float, default=0)
availability = Column(Float) availability = Column(Float)
performance = Column(Float) performance = Column(Float)
...@@ -24,6 +25,8 @@ class OEEDiscreteTable(Base): ...@@ -24,6 +25,8 @@ class OEEDiscreteTable(Base):
quality_loss = Column(Float) quality_loss = Column(Float)
performance_loss = Column(Float) performance_loss = Column(Float)
oee = Column(Float) oee = Column(Float)
calculated_on = Column(BigInteger) calculated_on = Column(TIMESTAMP(timezone=True), nullable=False)
uom = Column(String, default="mins") uom = Column(String, default="mins")
setup_time = Column(Float, default=0) setup_time = Column(Float, default=0)
trigger_by = Column(String, nullable=False)
project_id = Column(String, nullable=False)
scripts/db/mongo/ilens_configuration/collections/lookup_table.py 0 → 100644
View file @ f8692e94
from typing import Any, Optional, List
from scripts.constants import CommonKeys
from scripts.constants import DBConstants
from scripts.db.mongo.schema import MongoBaseSchema
from scripts.utils.mongo_util import MongoCollectionBaseClass
class LookupTableSchema(MongoBaseSchema):
"""
This is the Schema for the Mongo DB Collection.
All datastore and general responses will be following the schema.
"""
lookup_id: Optional[str]
lookup_name: Optional[str]
lookup_data: Optional[List]
project_id: Optional[Any]
description: Optional[str]
class LookupTable(MongoCollectionBaseClass):
def __init__(self, mongo_client, project_id=None):
super().__init__(mongo_client, database=DBConstants.db_metadata,
collection=DBConstants.lookup_table)
self.project_id = project_id
@property
def key_type(self):
return CommonKeys.KEY_TYPE
def find_constant_by_dict(self, _type):
"""
The following function will give one record for a given set of
search parameters as keyword arguments
:param _type:
:return:
"""
record = self.find_one(query={self.key_type: _type})
if not record:
return dict(record)
return dict(record)
def find_lookup_dict(self, lookup_name, project_id, filter_dict=None):
query = {self.key_type: lookup_name, "project_id": project_id}
record = self.find_one(query=query, filter_dict=filter_dict)
if not record:
return dict()
return dict(record)
def map_lookup_keys(self, lookup_name, project_id):
query = {self.key_type: lookup_name, "project_id": project_id}
_record = self.find_one(query=query)
if not _record:
return dict()
return {record["lookupdata_id"]: record["lookup_value"] for record in _record["lookup_data"]}
def insert_one_constant(self, data):
"""
The following function will insert one tag in the
tags collections
:param self:
:param data:
:return:
"""
return self.insert_one(data)
def find_constant_by_content(self, content_type):
"""
The following function will give one record for a given set of
search parameters as keyword arguments
"""
query = {"content_type": content_type}
search_option = {"data": 1}
record = self.find_one(query=query, filter_dict=search_option)
if not record:
return dict()
return record
scripts/db/mongo/schema/__init__.py 0 → 100644
View file @ f8692e94
from typing import Optional,List
from pydantic import BaseModel
class MongoBaseSchema(BaseModel):
pass
class BaseRequestSchema(BaseModel):
"""
This is base schema for input requests to the Collection Class
"""
scripts/db/psql/create_default_tables.py
View file @ f8692e94
...@@ -4,7 +4,7 @@ from sqlalchemy import create_engine ...@@ -4,7 +4,7 @@ from sqlalchemy import create_engine
from sqlalchemy_utils import database_exists, create_database from sqlalchemy_utils import database_exists, create_database
from scripts.config import DBConf from scripts.config import DBConf
from scripts.db.db_models import DownTimeCategory, DownTimeLog, DownTimeMasterError from scripts.db.db_models import OEEDiscreteTable
from scripts.logging import logger from scripts.logging import logger
engine = create_engine(DBConf.POSTGRES_URI) engine = create_engine(DBConf.POSTGRES_URI)
...@@ -14,9 +14,7 @@ def create_default_psql_dependencies(): ...@@ -14,9 +14,7 @@ def create_default_psql_dependencies():
try: try:
if not database_exists(engine.url): if not database_exists(engine.url):
create_database(engine.url) create_database(engine.url)
DownTimeCategory.__table__.create(bind=engine, checkfirst=True) OEEDiscreteTable.__table__.create(bind=engine, checkfirst=True)
DownTimeMasterError.__table__.create(bind=engine, checkfirst=True)
DownTimeLog.__table__.create(bind=engine, checkfirst=True)
except Exception as e: except Exception as e:
logger.error(f"Error occurred while creating: {e}", exc_info=True) logger.error(f"Error occurred while creating: {e}", exc_info=True)
......
scripts/db/psql/oee_discrete.py
View file @ f8692e94
...@@ -12,7 +12,7 @@ class DiscreteOEE(SQLDBUtils): ...@@ -12,7 +12,7 @@ class DiscreteOEE(SQLDBUtils):
super().__init__(db) super().__init__(db)
self.table = OEEDiscreteTable self.table = OEEDiscreteTable
def get_oee_data_all(self, start_time, end_time, hierarchy): def get_oee_data_all(self, prod_start_time, prod_end_time, hierarchy):
try: try:
data = ( data = (
...@@ -20,8 +20,8 @@ class DiscreteOEE(SQLDBUtils): ...@@ -20,8 +20,8 @@ class DiscreteOEE(SQLDBUtils):
.order_by(self.table.calculated_on) .order_by(self.table.calculated_on)
.filter( .filter(
self.table.hierarchy == hierarchy, self.table.hierarchy == hierarchy,
self.table.calculated_on >= start_time, self.table.calculated_on >= prod_start_time,
self.table.calculated_on <= end_time, self.table.calculated_on <= prod_end_time,
) )
) )
if data: if data:
...@@ -32,13 +32,14 @@ class DiscreteOEE(SQLDBUtils): ...@@ -32,13 +32,14 @@ class DiscreteOEE(SQLDBUtils):
logger.exception(e) logger.exception(e)
raise raise
def get_oee_data_batch_id(self, batch_id, hierarchy): def get_oee_data_by_reference_id(self, reference_id, hierarchy, project_id):
try: try:
data = ( data = (
self.session.query(self.table) self.session.query(self.table)
.order_by(self.table.calculated_on) .order_by(self.table.calculated_on)
.filter( .filter(
self.table.hierarchy == hierarchy, self.table.batch_id == batch_id self.table.hierarchy == hierarchy, self.table.reference_id == reference_id,
self.table.project_id == project_id
) )
.first() .first()
) )
...@@ -50,26 +51,26 @@ class DiscreteOEE(SQLDBUtils): ...@@ -50,26 +51,26 @@ class DiscreteOEE(SQLDBUtils):
logger.exception(e) logger.exception(e)
raise raise
def get_batches(self, hierarchy, start_time, end_time): def get_batches(self, hierarchy, prod_start_time, prod_end_time):
try: try:
data = ( data = (
self.session.query(self.table.batch_id) self.session.query(self.table.batch_id)
.order_by(self.table.calculated_on) .order_by(self.table.calculated_on)
.filter( .filter(
self.table.hierarchy == hierarchy, self.table.hierarchy == hierarchy,
self.table.calculated_on >= start_time, self.table.calculated_on >= prod_start_time,
self.table.calculated_on <= end_time, self.table.calculated_on <= prod_end_time,
) )
) )
if data: if data:
return [getattr(i, self.column_batch_id) for i in data] return [getattr(i, self.table.reference_id) for i in data]
else: else:
return list() return list()
except Exception as e: except Exception as e:
logger.exception(e) logger.exception(e)
raise raise
def get_products(self, hierarchy, start_time, end_time): def get_products(self, hierarchy, prod_start_time, prod_end_time):
try: try:
data = ( data = (
self.session.query( self.session.query(
...@@ -80,8 +81,8 @@ class DiscreteOEE(SQLDBUtils): ...@@ -80,8 +81,8 @@ class DiscreteOEE(SQLDBUtils):
.order_by(self.table.calculated_on) .order_by(self.table.calculated_on)
.filter( .filter(
self.table.hierarchy == hierarchy, self.table.hierarchy == hierarchy,
self.table.batch_start_time >= start_time, self.table.batch_start_time >= prod_start_time,
self.table.batch_end_time <= end_time, self.table.batch_end_time <= prod_end_time,
) )
) )
if data: if data:
...@@ -93,7 +94,7 @@ class DiscreteOEE(SQLDBUtils): ...@@ -93,7 +94,7 @@ class DiscreteOEE(SQLDBUtils):
raise raise
def get_chart_data( def get_chart_data(
self, start_time, end_time, hierarchy, product_id, aggregation=False self, prod_start_time, prod_end_time, hierarchy, reference_id, aggregation=False
): ):
try: try:
if not aggregation: if not aggregation:
...@@ -101,9 +102,9 @@ class DiscreteOEE(SQLDBUtils): ...@@ -101,9 +102,9 @@ class DiscreteOEE(SQLDBUtils):
self.session.query(self.table) self.session.query(self.table)
.filter( .filter(
self.table.hierarchy == hierarchy, self.table.hierarchy == hierarchy,
self.table.batch_id == product_id, self.table.reference_id == reference_id,
self.table.batch_start_time >= start_time, self.table.prod_start_time >= prod_start_time,
self.table.batch_end_time <= end_time, self.table.prod_end_time <= prod_end_time,
) )
.first() .first()
) )
...@@ -114,8 +115,8 @@ class DiscreteOEE(SQLDBUtils): ...@@ -114,8 +115,8 @@ class DiscreteOEE(SQLDBUtils):
self.session.query(self.table) self.session.query(self.table)
.filter( .filter(
self.table.hierarchy == hierarchy, self.table.hierarchy == hierarchy,
self.table.batch_start_time >= start_time, self.table.prod_start_time >= prod_start_time,
self.table.batch_end_time <= end_time, self.table.prod_end_time <= prod_end_time,
) )
.options( .options(
defer(self.table.hierarchy), defer(self.table.hierarchy),
......
scripts/db/redis_connections.py
View file @ f8692e94
import redis import redis
from scripts.config import RedisConfig from scripts.config import RedisConfig, KafkaConf
login_db = redis.from_url(RedisConfig.uri, db=int(RedisConfig.login_db), decode_responses=True) login_db = redis.from_url(RedisConfig.REDIS_URI, db=int(RedisConfig.LOGIN_DB), decode_responses=True)
project_details_db = redis.from_url(RedisConfig.uri, db=int(RedisConfig.project_tags_db), decode_responses=True) project_details_db = redis.from_url(RedisConfig.REDIS_URI, db=int(RedisConfig.PROJECT_TAGS_DB), decode_responses=True)
downtime_db = redis.from_url(RedisConfig.uri, db=int(RedisConfig.downtime_db), decode_responses=True) downtime_db = redis.from_url(RedisConfig.REDIS_URI, db=int(RedisConfig.DOWNTIME_DB), decode_responses=True)
oee_production_db = redis.from_url(RedisConfig.REDIS_URI, db=int(RedisConfig.OEE_PRODUCTION_DB), decode_responses=True)
live_tags_db = redis.from_url(RedisConfig.REDIS_URI, db=int(RedisConfig.LIVE_TAGS_DB), decode_responses=True)
partition_db = redis.from_url(RedisConfig.REDIS_URI, db=int(KafkaConf.redis_db), decode_responses=True)
\ No newline at end of file
scripts/schemas/batch_oee.py
View file @ f8692e94
from typing import Optional, Union, List from typing import Optional, Union, List
import time
from pydantic import BaseModel, validator from pydantic import BaseModel, validator
# from scripts.utils.common_utils import CommonUtils
#
# common_utils = CommonUtils()
class GetProducts(BaseModel): class GetProducts(BaseModel):
queryDate: List[int] queryDate: List[int]
...@@ -90,34 +94,95 @@ class ProductInfo(BaseModel): ...@@ -90,34 +94,95 @@ class ProductInfo(BaseModel):
reject_units: Optional[Union[float, int]] = 0 reject_units: Optional[Union[float, int]] = 0
class BatchOEEDataRequest(BaseModel): class OEEDataInsertRequest(BaseModel):
batch_start_time: Optional[int] prod_start_time: Optional[str]
batch_end_time: Optional[int] prod_end_time: Optional[str]
prod_status: Optional[str] = "running"
downtime: Optional[Union[float, int]] downtime: Optional[Union[float, int]]
hierarchy: Optional[str] hierarchy: Optional[str]
batch_id: Optional[str] reference_id: Optional[str]
setup_time: Optional[Union[float, int]] setup_time: Optional[Union[float, int]]
cycle_time: Optional[Union[float, int]] cycle_time: Optional[Union[float, int]]
total_units: Optional[Union[float, int]] total_units: Optional[Union[float, int]]
reject_units: Optional[Union[float, int]] reject_units: Optional[Union[float, int]]
tz: Optional[str] = 'Asia/Kolkata'
project_id: str
uom: Optional[str] uom: Optional[str]
class Config:
class BatchOEEDataSaveRequest(BaseModel): schema_extra = {
batch_start_time: Optional[int] "example": {
batch_end_time: Optional[int] = int(time.time() * 1000) "hierarchy": "site_100$dept_100$line_100$equipment_100",
"prod_start_time": "2022-04-22 19:49:00",
"prod_end_time": "2022-04-22 19:49:00",
"tz": "Asia/Kolkata",
"project_id": "project_099",
"reference_id": "reference_id",
"downtime": "",
"setup_time": "",
"cycle_time": "",
"total_units": "",
"reject_units": "",
"uom": "'"
}
}
# @validator('prod_start_time')
# def date_format_validator_start_date(cls, v):
# common_utils.check_date_format(v, "%Y-%m-%d %H:%M:%S")
# return v
#
# @validator('prod_end_time')
# def date_format_validator_end_date(cls, v):
# common_utils.check_date_format(v, "%Y-%m-%d %H:%M:%S")
# return v
class OEEDataSaveRequest(BaseModel):
prod_start_time: Optional[str]
prod_end_time: Optional[str]
downtime: Optional[Union[float, int]] = 0 downtime: Optional[Union[float, int]] = 0
hierarchy: Optional[str] hierarchy: Optional[str]
batch_id: Optional[str] reference_id: Optional[str]
setup_time: Optional[Union[float, int]] = 0 setup_time: Optional[Union[float, int]] = 0
cycle_time: Union[float, int] cycle_time: Union[float, int] = 3
total_units: Optional[Union[float, int]] = 0 total_units: Optional[Union[float, int]] = 0
reject_units: Optional[Union[float, int]] = 0 reject_units: Optional[Union[float, int]] = 0
uom: Optional[str] = "mins" uom: Optional[str] = "mins"
tz: Optional[str] = 'Asia/Kolkata'
project_id: str
class Config:
class BatchOEEData(BatchOEEDataRequest): schema_extra = {
calculated_on: int "example": {
"hierarchy": "site_100$dept_100$line_100$equipment_100",
"prod_start_time": "2022-04-22 19:49:00",
"prod_end_time": "2022-04-22 19:49:00",
"tz": "Asia/Kolkata",
"project_id": "project_099",
"reference_id": "reference_id",
"downtime": "",
"setup_time": "",
"cycle_time": "",
"total_units": "",
"reject_units": "",
"uom": "'"
}
}
# @validator('prod_start_time')
# def date_format_validator_start_date(cls, v):
# common_utils.check_date_format(v, "%Y-%m-%d %H:%M:%S")
# return v
#
# @validator('prod_end_time')
# def date_format_validator_end_date(cls, v):
# common_utils.check_date_format(v, "%Y-%m-%d %H:%M:%S")
# return v
class BatchOEEData(OEEDataInsertRequest):
calculated_on: str
productive_time: float productive_time: float
availability: float availability: float
performance: float performance: float
...@@ -132,14 +197,15 @@ class BatchOEEData(BatchOEEDataRequest): ...@@ -132,14 +197,15 @@ class BatchOEEData(BatchOEEDataRequest):
class GetOEERequest(BaseModel): class GetOEERequest(BaseModel):
start_time: int prod_start_time: int
end_time: int prod_end_time: int
hierarchy: str hierarchy: str
class GetOEERequestOneBatch(BaseModel): class GetOEERequestOneBatch(BaseModel):
hierarchy: str hierarchy: str
batch_id: str reference_id: str
project_id: str
class GetBatches(GetOEERequest): class GetBatches(GetOEERequest):
......
scripts/services/calculate_oee.py
View file @ f8692e94
import json
import traceback import traceback
from fastapi import APIRouter, Depends from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from pydantic import ValidationError from pydantic import ValidationError
from sqlalchemy.orm import Session
from scripts.constants import Endpoints from scripts.constants import Endpoints
from scripts.core.handlers.batch_oee_calc_handler import CalculateBatchOEEHandler from scripts.core.handlers.batch_oee_calc_handler import CalculateBatchOEEHandler
from scripts.db.psql.databases import get_db from scripts.db.psql.databases import get_db
from scripts.errors import ILensError
from scripts.logging import logger from scripts.logging import logger
from scripts.schemas.batch_oee import OEEDataInsertRequest
from scripts.schemas.batch_oee import BatchOEEDataRequest
from scripts.schemas.response_models import DefaultFailureResponse from scripts.schemas.response_models import DefaultFailureResponse
from scripts.errors import ILensError
import json
batch_oee_handler = CalculateBatchOEEHandler() batch_oee_handler = CalculateBatchOEEHandler()
calc_oee_router = APIRouter(prefix=Endpoints.calc_oee_base, tags=["OEE Calculator"]) calc_oee_router = APIRouter(prefix=Endpoints.calc_oee_base, tags=["OEE Calculator"])
@calc_oee_router.post(Endpoints.calculate_batch_oee) @calc_oee_router.post(Endpoints.calculate_batch_oee)
async def calculate_oee_for_batch( def calculate_oee_for_batch(
product_info: BatchOEEDataRequest, db: Session = Depends(get_db) product_info: OEEDataInsertRequest, db: Session = Depends(get_db)
): ):
try: try:
return await batch_oee_handler.calculate_oee(product_info=product_info, db=db) return batch_oee_handler.calculate_oee(request_data=product_info, db=db)
except ILensError as error_code: except ILensError as error_code:
return DefaultFailureResponse(error=error_code.args[0]) return DefaultFailureResponse(error=error_code.args[0])
except ValidationError as e: except ValidationError as e:
...@@ -36,10 +36,10 @@ async def calculate_oee_for_batch( ...@@ -36,10 +36,10 @@ async def calculate_oee_for_batch(
@calc_oee_router.post(Endpoints.update_batch_oee) @calc_oee_router.post(Endpoints.update_batch_oee)
def calculate_oee_for_batch( def calculate_oee_for_batch(
product_info: BatchOEEDataRequest, db: Session = Depends(get_db) product_info: OEEDataInsertRequest, db: Session = Depends(get_db)
): ):
try: try:
return batch_oee_handler.calculate_oee(product_info=product_info, db=db) return batch_oee_handler.calculate_oee(request_data=product_info, db=db)
except ValidationError as e: except ValidationError as e:
return DefaultFailureResponse(error=json.loads(e.json())) return DefaultFailureResponse(error=json.loads(e.json()))
except Exception as e: except Exception as e:
......
scripts/services/meta_services.py deleted 100644 → 0
View file @ 15ccf0a8
import traceback
from fastapi import APIRouter, Request, Cookie
from scripts.constants import Endpoints, ResponseCodes
from scripts.core.handlers.meta_handler import MetaHandler
from scripts.logging.logging import logger
from scripts.schemas.meta import GetHierarchyRequest
from scripts.schemas.response_models import DefaultFailureResponse, DefaultResponse
meta_handler = MetaHandler()
meta_service_router = APIRouter(prefix=Endpoints.api_hierarchy, tags=["Meta Services"])
@meta_service_router.post(Endpoints.api_get)
async def find_hierarchy(get_hierarchy_request: GetHierarchyRequest, request: Request):
try:
data = await meta_handler.find_hierarchy(
get_hierarchy_request=get_hierarchy_request, request=request
)
return DefaultResponse(
data=data,
status=ResponseCodes.SUCCESS,
message="Hierarchies listed successfully",
)
except Exception as e:
tb = traceback.format_exc()
logger.exception(e)
logger.exception(tb)
return DefaultFailureResponse(error=e.args)
scripts/utils/auth_util.py
View file @ f8692e94
...@@ -6,7 +6,7 @@ import httpx ...@@ -6,7 +6,7 @@ import httpx
import requests import requests
from aiohttp import ClientResponse from aiohttp import ClientResponse
from scripts.constants.app_constants import CommonStatusCode, Secrets from scripts.constants import CommonStatusCode, Secrets
from scripts.logging import logger from scripts.logging import logger
from scripts.utils.security_utils.apply_encrytion_util import create_token from scripts.utils.security_utils.apply_encrytion_util import create_token
......
scripts/utils/common_utils.py 0 → 100644
View file @ f8692e94
from datetime import datetime
import pendulum
from scripts.config import PathToServices
from scripts.constants import Secrets
from scripts.logging import logger
from scripts.utils.auth_util import ILensRequest, AuthenticationError
from scripts.utils.security_utils.apply_encrytion_util import create_token
class CommonUtils:
@staticmethod
def check_date_format(date_time_str, time_format):
try:
date_time = datetime.strptime(date_time_str, time_format)
except ValueError:
raise ValueError("Invalid Date Format")
if date_time_str != date_time.strftime(time_format):
raise ValueError("Invalid Date Format")
@staticmethod
def get_duration(tz, meta: dict, difference=False):
try:
from_time = meta.get("prod_start_time")
if not from_time:
return ""
if not meta.get("prod_end_time"):
to_time = pendulum.now(tz=tz)
else:
to_time = meta.get("prod_end_time")
to_time = pendulum.parse(to_time, tz=tz)
from_time = pendulum.parse(from_time, tz=tz)
diff = to_time - from_time
if difference:
return diff
return f"{int(diff.in_hours())} hours {int(diff.minutes)} minutes"
except Exception as e:
logger.exception(f"Exception in getting data: {e}")
raise
def get_downtime_details_by_hierarchy(self, hierarchy, project_id, user_id=None):
connection_obj = ILensRequest(url=PathToServices.downtime_proxy,
project_id=project_id)
try:
cookies = {'login-token': self.create_token(user_id=user_id), "user_id": user_id}
downtime_response = connection_obj.post(path="/downtime_log/get/overall_downtime",
json={"project_id": project_id,
"hierarchy": hierarchy})
response = downtime_response.json()
return response.get("data", 0)
except AuthenticationError:
logger.exception(f"Authentication Error when trying to connect with downtime module")
return 0
except Exception as e:
logger.exception(f'{e.args}')
return 0
@staticmethod
def create_token(host: str = '127.0.0.1', user_id=None, internal_token=Secrets.token, age=2):
"""
This method is to create a cookie
"""
try:
if user_id is None:
user_id = "user_099"
new_token = create_token(
user_id=user_id,
ip=host,
token=internal_token,
age=age
)
return new_token
except Exception as e:
logger.exception(str(e))
raise
scripts/utils/db_name_util.py
View file @ f8692e94
import json import json
from functools import lru_cache from functools import lru_cache
from scripts.db.redis_connections import project_details_db
@lru_cache() @lru_cache()
def get_db_name(redis_client, project_id: str, database: str, delimiter="__"): def get_db_name(redis_client, project_id: str, database: str, delimiter="__"):
...@@ -23,3 +25,16 @@ def get_db_name(redis_client, project_id: str, database: str, delimiter="__"): ...@@ -23,3 +25,16 @@ def get_db_name(redis_client, project_id: str, database: str, delimiter="__"):
prefix_name = val.get("source_meta", {}).get("prefix") or project_id prefix_name = val.get("source_meta", {}).get("prefix") or project_id
return f"{prefix_name}{delimiter}{database}" return f"{prefix_name}{delimiter}{database}"
return database return database
@lru_cache()
def get_project_specific_key(project_id=None, delimiter="__"):
if not project_id:
return ""
specific_key = ""
prefix_detail = project_details_db.get(project_id)
if prefix_detail is not None:
prefix_detail = json.loads(prefix_detail)
if prefix_detail.get('source_meta', {}).get('add_prefix_to_database', False):
specific_key = prefix_detail.get('source_meta').get('prefix') + delimiter
return specific_key
scripts/utils/kafka_util.py 0 → 100644
View file @ f8692e94
from ilens_kafka_publisher.v2 import KafkaPublisher
from scripts.config import KafkaConf
from scripts.logging import logger
class DataPush:
def __init__(self):
try:
self.obj = KafkaPublisher(kafka_host=KafkaConf.host,
kafka_port=int(KafkaConf.port),
kafka_topic=KafkaConf.topic,
redis_client=partition_db,
enable_sites_partition=KafkaConf.enable_sites_partition,
split_key=KafkaConf.split_key,
round_robin_enable=KafkaConf.round_robin_enable)
except Exception as e:
logger.error(f"Could not connect to Kafka: {e}")
def publish_message(self, msg):
try:
self.obj.perform_task(msg)
except Exception as e:
logger.debug(f"Failed to publish message - {e}")
logger.debug(f"Trying reconnect")
scripts/utils/security_utils/decorators.py
View file @ f8692e94
...@@ -7,6 +7,7 @@ from fastapi.security.api_key import APIKeyBase ...@@ -7,6 +7,7 @@ from fastapi.security.api_key import APIKeyBase
from scripts.config import Service from scripts.config import Service
from scripts.constants import Secrets from scripts.constants import Secrets
from scripts.db.redis_connections import login_db from scripts.db.redis_connections import login_db
from scripts.utils.security_utils.apply_encrytion_util import create_token from scripts.utils.security_utils.apply_encrytion_util import create_token
from scripts.utils.security_utils.jwt_util import JWT from scripts.utils.security_utils.jwt_util import JWT
...@@ -25,7 +26,6 @@ class CookieAuthentication(APIKeyBase): ...@@ -25,7 +26,6 @@ class CookieAuthentication(APIKeyBase):
def __init__( def __init__(
self, self,
cookie_name: str = "login-token", cookie_name: str = "login-token",
): ):
super().__init__() super().__init__()
self.model: APIKey = APIKey(**{"in": APIKeyIn.cookie}, name=cookie_name) self.model: APIKey = APIKey(**{"in": APIKeyIn.cookie}, name=cookie_name)
...@@ -57,11 +57,18 @@ class CookieAuthentication(APIKeyBase): ...@@ -57,11 +57,18 @@ class CookieAuthentication(APIKeyBase):
user_id = decoded_token.get("user_id") user_id = decoded_token.get("user_id")
_token = decoded_token.get("token") _token = decoded_token.get("token")
_token_age = int(decoded_token.get("age", Secrets.LOCK_OUT_TIME_MINS))
cookie_user_id = request.cookies.get(
"user_id", request.cookies.get(
"userId", request.headers.get("userId")
)
)
if not compare_digest(Secrets.token, _token): if not compare_digest(Secrets.token, _token):
raise HTTPException(status_code=401) raise HTTPException(status_code=401)
if login_token != decoded_token.get("uid"): if login_token != decoded_token.get("uid"):
raise HTTPException(status_code=401) raise HTTPException(status_code=401)
if not compare_digest(user_id, cookie_user_id):
raise HTTPException(status_code=401)
try: try:
new_token = create_token( new_token = create_token(
...@@ -69,17 +76,18 @@ class CookieAuthentication(APIKeyBase): ...@@ -69,17 +76,18 @@ class CookieAuthentication(APIKeyBase):
ip=request.client.host, ip=request.client.host,
token=Secrets.token, token=Secrets.token,
login_token=login_token, login_token=login_token,
age=_token_age
) )
except Exception as e: except Exception as e:
raise HTTPException(status_code=401, detail=e.args) raise HTTPException(status_code=401, detail=e.args)
response.set_cookie( response.set_cookie(
'login-token', "login-token",
new_token, new_token,
samesite='strict', samesite="strict",
secure=Service.secure_cookie, secure=Service.secure_cookie,
httponly=True, httponly=True,
max_age=Secrets.LOCK_OUT_TIME_MINS * 60, max_age=Secrets.LOCK_OUT_TIME_MINS * 60,
) )
response.headers['login-token'] = new_token response.headers["login-token"] = new_token
return user_id return user_id
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment