initial commit for aarti

.idea/.gitignore

+# Default ignored files
+/shelf/
+/workspace.xml

.idea/aart_ppe_detection.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>
\ No newline at end of file

.idea/inspectionProfiles/profiles_settings.xml

+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
\ No newline at end of file

.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (aart_ppe_detection)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file

.idea/modules.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/aart_ppe_detection.iml" filepath="$PROJECT_DIR$/.idea/aart_ppe_detection.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

.idea/vcs.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

Dockerfile

@@ -5,15 +5,16 @@ RUN apt-get install tzdata vim -y
 RUN apt-get update && apt-get install tzdata ffmpeg libsm6 libxext6  -y
 
 RUN pip3 install --upgrade pip
-# RUN pip3 install matplotlib>=3.2.2 tensorboard>=2.4.1 numpy>=1.18.5 opencv-python>=4.1.2 Pillow>=7.1.2 PyYAML>=5.3.1 requests>=2.23.0 scipy>=1.4.1 torch>=1.7.0 torchvision>=0.8.1 tqdm>=4.41.0 pandas seaborn expiringdict minio cachetools
-# RUN pip3 install pymongo Cython paho-mqtt==1.5.0 scikit-learn==0.22.2
+RUN pip3 install matplotlib>=3.2.2 tensorboard>=2.4.1 numpy>=1.18.5 opencv-python>=4.1.2 Pillow>=7.1.2 PyYAML>=5.3.1 requests>=2.23.0 scipy>=1.4.1 torch>=1.7.0 torchvision>=0.8.1 tqdm>=4.41.0 pandas seaborn expiringdict minio cachetools
+RUN pip3 install pymongo Cython paho-mqtt==1.5.0 scikit-learn==0.22.2
 
 RUN pip3 install absl-py==1.3.0 asttokens==2.2.1 backcall==0.2.0 cachetools==5.2.0 certifi==2022.12.7 charset-normalizer==2.1.1 colorama==0.4.6 contourpy==1.0.6
 RUN pip3 install cycler==0.11.0 Cython==0.29.32 decorator==5.1.1 dnspython==2.2.1 executing==1.2.0 expiringdict==1.2.2 fonttools==4.38.0
 RUN pip3 install google-auth==2.15.0 google-auth-oauthlib==0.4.6
 RUN pip3 install grpcio==1.51.1 idna==3.4 importlib-metadata==5.1.0 ipython jedi==0.18.2 kiwisolver==1.4.4 Markdown==3.4.1 MarkupSafe==2.1.1 matplotlib matplotlib-inline==0.1.6 minio==7.1.12 numpy>=1.18.5
 RUN pip3 install oauthlib==3.2.2 opencv-python==4.6.0.66 packaging==22.0 paho-mqtt==1.6.1 pandas parso==0.8.3 pickleshare==0.7.5 Pillow==9.3.0 prompt-toolkit==3.0.36 protobuf==3.20.3 pure-eval==0.2.2 pyasn1==0.4.8 pyasn1-modules==0.2.8 Pygments==2.13.0 pymongo==4.3.3 pyparsing==3.0.9 python-dateutil==2.8.2 pytz==2022.6 PyYAML==6.0 requests==2.28.1 requests-oauthlib==1.3.1 rsa==4.9 scipy torch==1.9.0 torchvision==0.10.0 tqdm==4.64.1 traitlets==5.7.0 typing_extensions==4.4.0 urllib3==1.26.13 wcwidth==0.2.5 Werkzeug==2.2.2 zipp==3.11.0
-RUN pip3 install seaborn
+RUN pip3 install seaborn psycopg2
+
 
 ADD . /app
 ADD Arial.ttf /root/.config/Ultralytics/

app.py

 import os
 # os.environ["config"]="{\"TZ\": \"Asia/Kolkata\", \"MONGO_URI\": \"mongodb://svc-ilens:svc2345@192.168.3.220:21017\", \"MONGO_DATABASE\": \"ilens_wps\", \"MONGO_COLLECTION\": \"janusDeployment\", \"MONGO_KEY\": \"deploymentId\", \"MONGO_VALUE\": \"rahul_12345\", \"MONGO_COLL\": \"serviceConfiguration\", \"MONGO_DB\": \"ilens_wps\"}"
 # os.environ["config"]="{\"TZ\": \"Asia/Kolkata\", \"MONGO_URI\": \"mongodb://admin:iLens$HPCLv605@10.5.2.91:2717\", \"MONGO_DATABASE\": \"ilens_ai\", \"MONGO_COLLECTION\": \"janusDeployment\", \"MONGO_KEY\": \"deploymentId\", \"MONGO_VALUE\": \"hpcl_cctv_ppe\", \"MONGO_COLL\": \"serviceConfiguration\", \"MONGO_DB\": \"ilens_ai\"}"
-# os.environ["config"]="{\"TZ\": \"Asia/Kolkata\", \"MONGO_URI\": \"mongodb://admin:iLens$1234@192.168.3.181:2717/admin\", \"MONGO_DATABASE\": \"ilens_ai\", \"MONGO_COLLECTION\": \"janusDeployment\", \"MONGO_KEY\": \"deploymentId\", \"MONGO_VALUE\": \"aarti_ppe\", \"MONGO_COLL\": \"serviceConfiguration\", \"MONGO_DB\": \"ilens_ai\"}"
+os.environ["config"]="{\"TZ\": \"Asia/Kolkata\", \"MONGO_URI\": \"mongodb://admin:iLens$1234@192.168.3.181:2717/admin\", \"MONGO_DATABASE\": \"ilens_ai\", \"MONGO_COLLECTION\": \"janusDeployment\", \"MONGO_KEY\": \"deploymentId\", \"MONGO_VALUE\": \"aarti_ppe\", \"MONGO_COLL\": \"serviceConfiguration\", \"MONGO_DB\": \"ilens_ai\"}"
 from edge_engine.edge_processor import ExecutePipeline
 from edge_engine.edge_processor import Pubs
 from scripts import Ppe

edge_engine/edge_processor.py

@@ -77,6 +77,7 @@ class ExecutePipeline:
         self.model = model
 
     def run_model(self):
+
         if EDGE_CONFIG["inputConf"]["sourceType"].lower() in ["rtsp", "usbcam"]:
             logger.info("Selected input stream as Direct cv input")
             self.threadedVideoStream = ThreadedVideoStream(stream_config=EDGE_CONFIG["inputConf"])
@@ -84,6 +85,7 @@ class ExecutePipeline:
             self.frameProcessor = FrameProcessor(stream=self.threadedVideoStream,
                                                  model=self.model)
         elif EDGE_CONFIG["inputConf"]["sourceType"].lower() == "videofile":
+
             self.fileVideoStream = FileVideoStream(stream_config=EDGE_CONFIG["inputConf"])
             self.fileVideoStream.start()
             self.frameProcessor = FrameProcessor(stream=self.fileVideoStream, model=self.model)

edge_engine/streamio/frameProcessor.py

@@ -6,17 +6,45 @@ class FrameProcessor:
     def __init__(self, stream, model):
         self.model = model
         self.stream = stream
+        self.old_video_capture_count = 0
         logger.info("Setting up frame processor !!")
+        self.count = 0
+        self.skip_frame_every = 30  # 1 does not skip any frame (n-1 frames get skipped)
+
 
     def run_model(self):
-        while self.stream.stream.isOpened():
+        count = 0
+        while self.stream.running():
             try:
+
+                # video_capture_count = self.stream.get_frame_count()
+                # if(video_capture_count != self.old_video_capture_count):
+                #
+                #     self.old_video_capture_count = video_capture_count
+                #     count = 0
+                # if(video_capture_count == 1):
+                #     count = 0
+
+                count += 1
                 logger.debug("Getting frame mask_model")
-                frame = self.stream.read()
-                logger.debug("Running mask_model")
-                data = {"frame": frame, "frameId": "{}".format(uuid4()), "deviceId": "{}".format(DEVICE_ID)}
+                frame, count_1 = self.stream.read()
+
+                if frame is not None and self.count % self.skip_frame_every == 0:
+                    fid = uuid4()
+                    data = {
+                        "frame": frame,
+                        "frameId": self.count,
+                        "deviceId": "{}".format(DEVICE_ID),
+                    }
+
                     self.model.predict(data)
-                logger.debug("publishing mask_model output")
+                self.count += 1
+                # logger.debug("Running mask_model")
+                # data = {"frame": frame, "frameId": "{}".format(uuid4()), "deviceId": "{}".format(DEVICE_ID), "frame_id": count}
+                # self.model.predict(data)
+                # logger.debug("publishing mask_model output")
+                if(self.count == 3227):
+                    self.count = 0
             except Exception as e:
                 logger.error(e)
                 logger.error(traceback.format_exc())
\ No newline at end of file

edge_engine/streamio/videostream/filevideostream.py

@@ -14,6 +14,7 @@ else:
 
 class FileVideoStream:
     def __init__(self,stream_config, transform=None):
+        self.count = 0
         # initialize the file video stream along with the boolean
         # used to indicate if the thread should be stopped or not
         self.transform = transform
@@ -21,6 +22,8 @@ class FileVideoStream:
         # initialize the queue used to store frames read from
         # the video file
         self.build_pipeline()
+        self.count_dict = {"count": 0}
+
 
 
     def start(self):
@@ -29,6 +32,7 @@ class FileVideoStream:
         return self
 
     def build_cv_obj(self):
+        # self.count += 1
         self.stream = cv2.VideoCapture(self.stream_config["uri"])
         self.stopped = False
 
@@ -43,6 +47,8 @@ class FileVideoStream:
         self.thread.daemon = True
 
 
+
+
     def is_opened(self):
         return self.stream.isOpened()
 
@@ -58,12 +64,17 @@ class FileVideoStream:
             if not self.Q.full():
                 # read the next frame from the file
                 (grabbed, frame) = self.stream.read()
+                self.count += 1
+                self.count_dict["count"] = self.count
 
                 # if the `grabbed` boolean is `False`, then we have
                 # reached the end of the video file
                 if grabbed is False or frame is None:
                     #self.stopped = True
+                    self.count = 0
+                    self.count_dict["count"] = self.count
                     self.build_cv_obj()
+
                     continue
 
                 # if there are transforms to be done, might as well
@@ -88,9 +99,14 @@ class FileVideoStream:
 
         self.stream.release()
 
+
     def read(self):
         # return next frame in the queue
-        return self.Q.get()
+        return self.Q.get(), self.count_dict
+    def get_frame_count(self):
+
+        # return next frame in the queue
+        return self.count
 
     # Insufficient to have consumer use while(more()) which does
     # not take into account if the producer has reached end of

scripts/ppe.py

@@ -14,8 +14,8 @@ import numpy as np
 
 from collections import deque
 from expiringdict import ExpiringDict
-# from sklearn.utils.linear_assignment_ import linear_assignment
-from scipy.optimize import linear_sum_assignment as linear_assignment
+from sklearn.utils.linear_assignment_ import linear_assignment
+# from scipy.optimize import linear_sum_assignment as linear_assignment
 
 from edge_engine.common.logsetup import logger
 from edge_engine.ai.model.modelwraper import ModelWrapper
@@ -30,6 +30,9 @@ from scripts.common.constants import JanusDeploymentConstants
 # from yolov5processor.infer import ExecuteInference
 from yolov5.detect import run
 from yolov5.detect import load_model
+import random
+import psycopg2
+import time
 
 
 class Ppe(ModelWrapper):
@@ -44,7 +47,7 @@ class Ppe(ModelWrapper):
         self.rtp = pubs.rtp_write
         self.mongo_logger = MongoLogger()
         self.frame_skip = self.config.get('frame_skip', False)
-        model = "data/aarti_v2.pt"
+        model = "data/aarti_v3.pt"
         self.yolo_model = load_model(model)
         # self.yp = ExecuteInference(weight=model,
         #                            gpu=model_config.get("gpu", False),
@@ -71,6 +74,7 @@ class Ppe(ModelWrapper):
         self.uncounted_objects = ExpiringDict(max_len=model_config.get("uncounted_obj_length", 50),
                                               max_age_seconds=model_config.get("uncounted_obj_age", 60))
         self.janus_metadata = ExpiringDict(max_age_seconds=120, max_len=1)
+        self.safety_equip = ExpiringDict(max_age_seconds=30, max_len=10)
         # self.polygon = np.array([[[7, 753],[774, 574], [1688, 672], [1473, 977], [39, 962]]])
         # self.polygon = np.array([[[300, 753], [774, 674], [1900, 672], [1900, 1000], [39, 1000]]])
         # self.polygon = np.array([[[300, 753], [774, 674], [1900, 672], [1900, 900], [100, 900]]])
@@ -80,6 +84,18 @@ class Ppe(ModelWrapper):
         self.final_ppe_result = {}
         self.skip_frame_bool = False
         self.skip_frame_count = 0
+        self.frame_id = 0
+        self.reported_violation_ids = {}
+        self.violation_count = {"Air Breathing Mask": [] , "Safety helmet": [], "Hand gloves": [], "coverall suit": []}
+
+        self.uri = "postgres://postgres:postgres@192.168.3.181:5432/postgres?sslmode=disable"
+        self.payload_classes = {"Air Breathing Mask": "air_breathing_mask_violation", "Safety helmet": "helmet_violation", "Hand gloves": "glove_violation", "coverall suit": "coverall_suit_violation"}
+
+    # connect to the PostgreSQL server
+    # print('Connecting to the PostgreSQL database...')
+        self.conn = psycopg2.connect(self.uri)
+
+
     def _pre_process(self, x):
         """
         Do preprocessing here, if any
@@ -98,7 +114,7 @@ class Ppe(ModelWrapper):
         return x
 
     def send_payload(self, frame, label='PPEFrame', bg_color="#474520", font_color="#FFFF00", alert_sound=None,
-                     message="PPE Frame Output"):
+                     message=[], event = "", frame_id = None):
         """
         Insert event to Mongox
         :param message:
@@ -113,7 +129,7 @@ class Ppe(ModelWrapper):
         payload = {"deviceId": self.device_id, "message": message,
                    "frame": 'data:image/jpeg;base64,' + base64.b64encode(
                        cv2.imencode('.jpg', frame)[1].tostring()).decode("utf-8"), "activity": label,
-                   "bg_color": bg_color, "font_color": font_color, "alert_sound": alert_sound}
+                   "bg_color": bg_color, "font_color": font_color, "alert_sound": alert_sound, "frame_id": frame_id, "event_type": event}
 
         self.mongo_logger.insert_attendance_event_to_mongo(payload)
 
@@ -221,6 +237,7 @@ class Ppe(ModelWrapper):
                 unmatched_trackers.append(t)
 
         for d, det in enumerate(detections):
+
             if d not in matched_idx[:, 1]:
                 unmatched_detections.append(d)
 
@@ -240,6 +257,21 @@ class Ppe(ModelWrapper):
 
         return matches, np.array(unmatched_detections), np.array(unmatched_trackers)
 
+    def get_line_coordinates(self):
+        """
+        Get the line coordinates from the deployment JSON
+        """
+        if not self.janus_metadata.get('metadata'):
+            self.janus_metadata['metadata'] = get_extra_fields(self.device_id)
+
+        _coordinates = [self.janus_metadata['metadata'].get(coordinate_key) for coordinate_key in
+                        JanusDeploymentConstants.LINE_COORDINATES]
+        _alignment = self.janus_metadata['metadata'].get(JanusDeploymentConstants.ALIGNMENT_KEY) \
+            # _coordinates = [550, 200, 555, 1100]
+        #
+        # _alignment = "vertical"
+        return _alignment, _coordinates
+
 
 
     def get_line_coordinates(self):
@@ -305,32 +337,222 @@ class Ppe(ModelWrapper):
             else:
                 return False
 
-    def ppe_detection(self, frame, bbox, class_name, other_class_name, other_centroid):
+    def ppe_detection(self, frame, bbox, detection_objects, class_name, other_class_name, other_centroid):
         """
         Maintains the bag counts
         :param frame: image
         :param detection_objects: detection object having object id and centroids
         """
         detected_objects = []
-        needed_objects = ["Helmet", "Vest"]
+        needed_objects_with_mask = {"Air Breathing Mask", "Hand gloves", "coverall suit"}
+        needed_objects_with_mask_helmet = {"Air Breathing Mask", "Hand gloves", "coverall suit", "Safety helmet"}
+        needed_objects_with_helmet = {"Safety helmet", "Hand gloves", "coverall suit"}
         person_with_safety_kit = 0
         person_without_safety_kit = 0
         total_person_safety_kits = {"Air Breathing Mask"}
+
         #logger.info("Detections: " + str(detections))
-        for person_bb in bbox:
+        for (object_id, class_detected, person_bb) in zip(
+                detection_objects, class_name, bbox
+        ):
+            centroid = object_id[1]
+            # print(object_id)
+            object_id = object_id[0]
+            # logger.debug(detections)
+            # print(object_id)
+            # print("person bb")
+            # print(person_bb)
+            # cv2.putText(frame, str(object_id), (centroid[1], centroid[0]), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0),
+            #                     2, cv2.LINE_AA)
+            # cv2.putText(frame, class_detected, (centroid[1], centroid[0]), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0),
+            #                     2, cv2.LINE_AA)
+
+            # frame = draw_circles_on_frame(
+            #     frame, centroid, radius=10, color=(0, 0, 255), thickness=-1
+            # )
+            cv2.circle(frame, (centroid[1], centroid[0]), 2, (0, 255, 0), thickness=1, lineType=8, shift=0)
+
+        # for person_bb in bbox:
             person_safety_status = set()
+            # print("other classes are")
+            # print(other_class_name)
+            # print(other_centroid)
+            temp_list = set()
             for (safety_object, object_bb) in zip(other_class_name, other_centroid):
+                # print("inside check loop")
+                violations = set()
                 if (person_bb[0] < object_bb[0] and person_bb[2] > object_bb[0]):
                     if(person_bb[1] < object_bb[1] and person_bb[3] > object_bb[1]):
                         person_safety_status.add(safety_object)
-            if("Air Breathing Mask" in person_safety_status):
+                        print("SAFETY OBJECTSSSS")
+                        print(safety_object)
+
+                        # print(safety_object, object_bb)
+                        if("object_id" in self.safety_equip):
+                            temp_list = set()
+                            temp_list = self.safety_equip["object_id"]
+                            temp_list.add(safety_object)
+                            # temp_list = self.safety_equip["object_id"]
+                            self.safety_equip["object_id"] = temp_list
+                            # print("object id present")
+                        else:
+                            temp_list = set()
+                            temp_list.add(safety_object)
+                            # print("object id not present")
+                            self.safety_equip["object_id"] = temp_list
+            # print("first li")
+            # print(self.safety_equip["object_id"])
+            # print(needed_objects)
+            # print(bbox)
+
+            if("Air Breathing Mask" in self.safety_equip["object_id"] and "Safety helmet" in self.safety_equip["object_id"]):
+                # print("with both helmet and air breathing mask if")
+                if (self.safety_equip["object_id"] == needed_objects_with_mask_helmet):
+                    # print("SAFE--------------------------------------------")
+                    person_with_safety_kit += 1
+                    cv2.rectangle(frame, (person_bb[0], person_bb[1]), (person_bb[2], person_bb[3]), (0, 255, 0), 2)
+                    # cv2.waitKey(0)
+
+
+                else:
+                    violations = needed_objects_with_mask_helmet.difference(self.safety_equip["object_id"])
+                    # print("violations")
+                    # print(violations)
+                    temp_violation_list = []
+                    for v in violations:
+                        temp_violation_list.append(self.payload_classes[v])
+
+                    # violated_items = ', '.join(list(map(str, temp_violation_list)))
+                    temp_violation_list.sort()
+                    violated_items_2 = ', '.join(list(map(str, violations)))
+                    print("violated items")
+                    print(violated_items)
+                    # violated_items = violated_items
+                    # print(violated_items)
+                    for elem in violations:
+                        self.violation_count[elem].append(elem)
+
+                    if (object_id not in self.reported_violation_ids):
+                        print("sending to mongo")
+                        self.send_payload(frame=resize_to_64_64(frame), message=temp_violation_list, event = violated_items,frame_id=self.frame_id)
+                        self.reported_violation_ids[object_id] = time.time()
+                        sql = 'INSERT INTO "aarti_violation_event_table" ("camera_id", "frame_id", "timestamp", "violation_type") VALUES (%s, %s, %s, %s);'
+                        # for violated_items in violations:
+                            # present_event_id = 'SELECT "event_id" FROM aarti_violation_event_table'
+                        cur = self.conn.cursor()
+                        frame_count = self.frame_id
+                        cur.execute(sql, [4, frame_count, datetime.datetime.now(), violated_items])
+                        self.conn.commit()
+                        cur.close()
+                    else:
+                        time_diff = time.time() - self.reported_violation_ids[object_id]
+                        if(time_diff > 30):
+                            del self.reported_violation_ids[object_id]
+
+
+
+                    person_without_safety_kit += 1
+                    cv2.rectangle(frame, (person_bb[0], person_bb[1]), (person_bb[2], person_bb[3]), (0, 0, 255), 2)
+
+            elif ("Air Breathing Mask" in self.safety_equip["object_id"]):
+                # print("air breathing mask if")
+                print(self.safety_equip["object_id"])
+                if (self.safety_equip["object_id"] == needed_objects_with_mask):
+                    # print("SAFE--------------------------------------------")
+                    person_with_safety_kit += 1
+                    cv2.rectangle(frame, (person_bb[0], person_bb[1]), (person_bb[2], person_bb[3]), (0, 255, 0), 2)
+                    # cv2.waitKey(0)
+
+
+                else:
+                    violations = needed_objects_with_mask.difference(self.safety_equip["object_id"])
+                    # print("violations")
+                    # print(violations)
+                    for elem in violations:
+                        self.violation_count[elem].append(elem)
+
+                    temp_violation_list = []
+                    for v in violations:
+                        temp_violation_list.append(self.payload_classes[v])
+                    temp_violation_list.sort()
+                    violated_items = ', '.join(list(map(str, temp_violation_list)))
+                    violated_items_2 = ', '.join(list(map(str, violations)))
+                    print("violated items")
+                    print(violated_items)
+                    # print(msg)
+                    # print("VIOLATION LIST")
+                    print(violated_items)
+                    if (object_id not in self.reported_violation_ids):
+                        print("sending to mongo")
+                        self.send_payload(frame=resize_to_64_64(frame), message=temp_violation_list , event = violated_items, frame_id=self.frame_id)
+                        self.reported_violation_ids[object_id] = time.time()
+                        sql = 'INSERT INTO "aarti_violation_event_table" ("camera_id", "frame_id", "timestamp", "violation_type") VALUES (%s, %s, %s, %s);'
+                        # for violated_items in violations:
+                            # present_event_id = 'SELECT "event_id" FROM aarti_violation_event_table'
+                        cur = self.conn.cursor()
+                        frame_count = self.frame_id
+                        cur.execute(sql, [4, frame_count, datetime.datetime.now(), violated_items])
+                        self.conn.commit()
+                        cur.close()
+
+                    else:
+                        time_diff = time.time() - self.reported_violation_ids[object_id]
+                        if(time_diff > 30):
+                            del self.reported_violation_ids[object_id]
+
+                    person_without_safety_kit += 1
+                    cv2.rectangle(frame, (person_bb[0], person_bb[1]), (person_bb[2], person_bb[3]), (0, 0, 255), 2)
+                    # cv2.waitKey(0)
+
+            else:
+                print("without air breathing mask if")
+                if (self.safety_equip["object_id"] == needed_objects_with_helmet):
                     # print("SAFE--------------------------------------------")
-                person_with_safety_kit +=1
+                    person_with_safety_kit += 1
                     cv2.rectangle(frame, (person_bb[0], person_bb[1]), (person_bb[2], person_bb[3]), (0, 255, 0), 2)
+                    # cv2.waitKey(0)
 
 
                 else:
-                person_without_safety_kit +=1
+                    violations = needed_objects_with_helmet.difference(self.safety_equip["object_id"])
+                    # print("violations")
+                    # print(violations)
+                    temp_violation_list = []
+                    for v in violations:
+                        temp_violation_list.append(self.payload_classes[v])
+
+                    # temp_violation_list.append("air_breathing_mask_violation")
+                    temp_violation_list.sort()
+                    violated_items = ', '.join(list(map(str, temp_violation_list)))
+                    violated_items_2 = ', '.join(list(map(str, violations)))
+                    print("violated items")
+                    print(violated_items)
+                    violated_items = violated_items
+                    # violated_items_2 = violated_items_2 + ", Air Breathing Mask"
+                    # print(violated_items)
+                    for elem in violations:
+                        self.violation_count[elem].append(elem)
+
+                    if (object_id not in self.reported_violation_ids):
+                        print("sending to mongo")
+                        self.send_payload(frame=resize_to_64_64(frame), message=temp_violation_list, event = violated_items, frame_id=self.frame_id)
+                        self.reported_violation_ids[object_id] = time.time()
+                        sql = 'INSERT INTO "aarti_violation_event_table" ("camera_id", "frame_id", "timestamp", "violation_type") VALUES (%s, %s, %s, %s);'
+                        # for violated_items in violations:
+                            # present_event_id = 'SELECT "event_id" FROM aarti_violation_event_table'
+                        cur = self.conn.cursor()
+                        frame_count = self.frame_id
+                        cur.execute(sql, [4, frame_count, datetime.datetime.now(), violated_items])
+                        self.conn.commit()
+                        cur.close()
+
+
+                    else:
+                        time_diff = time.time() - self.reported_violation_ids[object_id]
+                        if(time_diff > 30):
+                            del self.reported_violation_ids[object_id]
+
+                    person_without_safety_kit += 1
                     cv2.rectangle(frame, (person_bb[0], person_bb[1]), (person_bb[2], person_bb[3]), (0, 0, 255), 2)
                     # cv2.waitKey(0)
         # for (object_id, det) in zip(other_class_name, other_centroid):
@@ -382,21 +604,57 @@ class Ppe(ModelWrapper):
         #                                             }
         #         self.send_payload(frame=resize_to_64_64(frame), message = str(self.final_ppe_result))
         #         self.final_ppe_result = {}
-        print("person with safety kit", str(person_with_safety_kit))
-        print("person without safety kit", str(person_without_safety_kit))
+        # print("person with safety kit", str(person_with_safety_kit))
+        # print("person without safety kit", str(person_without_safety_kit))
+
         person_without_kit_text = "PERSON WITHOUT AIR BREATHING MASK : " + str(person_without_safety_kit)
         person_with_kit_text = "PERSON WITH AIR BREATHING MASK : " + str(person_with_safety_kit)
-        cv2.rectangle(frame, (870, 0), (1400, 50), (0, 255, 0), -1)
-        cv2.rectangle(frame, (870, 50), (1400, 120), (50, 50, 50), -1)
 
+        cv2.rectangle(frame, (870, 0), (1400, 50), (0, 0, 255), -1)
+        cv2.rectangle(frame, (870, 50), (1400, 120), (50, 50, 50), -1)
+        cv2.putText(frame, "Air Breathing Mask Violation", (880, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 4, 2)
+        cv2.rectangle(frame, (1400, 300), (1900, 350), (0, 255, 0), -1)
+        cv2.rectangle(frame, (1400, 350), (1900, 420), (50, 50, 50), -1)
+        cv2.putText(frame, "People following compliance", (1410, 330), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 4, 2)
+        cv2.rectangle(frame, (870, 300), (1400, 350), (0, 0, 255), -1)
+        cv2.rectangle(frame, (870, 350), (1400, 420), (50, 50, 50), -1)
+        cv2.putText(frame, "People not following compliance", (880, 330), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 4, 2)
         cv2.rectangle(frame, (1400, 0), (1900, 50), (0, 0, 255), -1)
         cv2.rectangle(frame, (1400, 50), (1900, 120), (50, 50, 50), -1)
-
-
-        cv2.putText(frame, "WITH AIR BREATHING MASK", (880, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 4, 2)
-        cv2.putText(frame, "WITHOUT AIR BREATHING MASK", (1400, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 4, 2)
-        cv2.putText(frame, str(person_with_safety_kit), (1100, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 4, 2)
-        cv2.putText(frame, str(person_without_safety_kit), (1600, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 4, 2)
+        cv2.putText(frame, "Coverall Suit Violation", (1410, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 4, 2)
+        cv2.rectangle(frame, (870, 150), (1400, 200), (0, 0, 255), -1)
+        cv2.rectangle(frame, (870, 200), (1400, 270), (50, 50, 50), -1)
+        cv2.putText(frame, "Helmet Violation", (880, 180), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 4, 2)
+        cv2.rectangle(frame, (1400, 150), (1900, 200), (0, 0, 255), -1)
+        cv2.rectangle(frame, (1400, 200), (1900, 270), (50, 50, 50), -1)
+        cv2.putText(frame, "Glove Violation", (1410, 180), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 4, 2)
+        # print("dict")
+        # print(self.violation_count)
+        if (person_without_safety_kit > 0):
+            cv2.putText(frame, str(person_without_safety_kit), (1100, 400), cv2.FONT_HERSHEY_SIMPLEX, 1,
+                        (255, 255, 255), 4, 2)
+
+        if (len(self.violation_count["Air Breathing Mask"]) != 0):
+            cv2.putText(frame, str(len(self.violation_count[""])), (1100, 100), cv2.FONT_HERSHEY_SIMPLEX, 1,
+                        (255, 255, 255), 4, 2)
+
+        if (person_with_safety_kit > 0):
+            cv2.putText(frame, str(person_with_safety_kit), (1600, 400), cv2.FONT_HERSHEY_SIMPLEX, 1,
+                        (255, 255, 255), 4, 2)
+
+        if (len(self.violation_count["coverall suit"]) != 0):
+            cv2.putText(frame, str(len(self.violation_count["coverall suit"])), (1600, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 4, 2)
+
+        if (len(self.violation_count["Safety helmet"]) != 0):
+            cv2.putText(frame, str(len(self.violation_count["Safety helmet"])), (1100, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 4, 2)
+
+        if (len(self.violation_count["Hand gloves"]) != 0):
+            cv2.putText(frame, str(len(self.violation_count["Hand gloves"])), (1600, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 4, 2)
+        self.violation_count = {"Air Breathing Mask": [], "Safety helmet": [], "Hand gloves": [], "coverall suit": []}
+        # print("expiring dict")
+        # print(self.safety_equip)
+        if(len(self.safety_equip) == 0):
+            cv2.waitKey(0)
         return frame
 
     def draw_line_over_image(self, frame, color=(255, 255, 255)):
@@ -452,14 +710,14 @@ class Ppe(ModelWrapper):
         bboxs = []
         other_centroid = []
         other_class_name = list()
-        #logger.info(dets)
+        logger.info(dets)
         print("PREDICTIONS ")
         print(dets)
         if dets:
             for i in dets:
 
                 if(i["class"] in classes):
-                    print("detections are there")
+                    # print("detections are there")
 
                     # c = (int(i["points"][0] + (i["points"][2] - i["points"][0])/2), int(i["points"][1] + (i["points"][3] - i["points"][1])/2))
 
@@ -468,6 +726,11 @@ class Ppe(ModelWrapper):
                     class_name.append(i["class"])
                     frame = cv2.rectangle(frame, (i["points"][0], i["points"][1]), (i["points"][2], i["points"][3]), (255, 255, 0), 2)
                     bboxs.append([i["points"][0], i["points"][1], i["points"][2], i["points"][3]])
+                    # bboxs.append([i["points"][1], i["points"][0], i["points"][3], i["points"][2]])
+                    # bboxs.append(
+                    #     [i["points"][1], i["points"][0], i["points"][3], i["points"][2]]
+                    # )
+
                     #logger.info("BBOX inside polygon: " + str(bboxs))
                 else:
                     c = tuple(i["centroid"])
@@ -479,9 +742,15 @@ class Ppe(ModelWrapper):
         return bboxs,frame, class_name, other_class_name, other_centroid
 
     def _predict(self, obj):
+        self.count+= 1
+
         class_list = ["person"]
         try:
             frame = obj['frame']
+            self.frame_id = obj["frameId"]
+            if(self.frame_id == 0):
+                self.reported_violation_ids = {}
+                self.safety_equip = ExpiringDict(max_age_seconds=30, max_len=10)
 
             if self.frame_skip:
                 if not self.frame_skipping["skip_current_frame"]:
@@ -496,18 +765,18 @@ class Ppe(ModelWrapper):
                 #                       True, (255, 255, 255), 2)
                 bbox, frame, class_name, other_class_name, other_centroid = self.inference(frame, class_list)
                 # cv2.imshow("out", cv2.resize(frame, (900, 600)))
-                # cv2.waitKey(0)
+                # cv2.waitKey(1)
             # frame = self.draw_line_over_image(frame)
             # if [True for e in dets if e['class'] == 'cement_bag']:
 
 
-            #if dets:
-            # frame, objects, boxs = self.kalman_tracker(dets, frame)
+                # if bbox:
+            frame, objects, boxs = self.kalman_tracker(bbox, frame)
         #logger.info("PRINTING KALMAN OUTPUT")
         #logger.info(objects)
         #logger.info(boxs)
 
-            frame = self.ppe_detection(frame=frame, bbox=bbox, class_name = class_name, other_class_name = other_class_name, other_centroid = other_centroid)
+            frame = self.ppe_detection(frame=frame, bbox=bbox, detection_objects = objects, class_name = class_name, other_class_name = other_class_name, other_centroid = other_centroid)
             #logger.info("Final PPE tracking people result: " + str(self.final_ppe_result))
 
             # call send payload to update final_ppe_result, empty final ppe result dict
@@ -516,8 +785,12 @@ class Ppe(ModelWrapper):
             obj['frame'] = cv2.resize(frame, (self.config.get('FRAME_WIDTH'), self.config.
                                               get('FRAME_HEIGHT')))
 
-            # cv2.imshow("output is ", cv2.resize(frame, (900, 600)))
-            # cv2.waitKey(1)
+            cv2.putText(frame, str(self.frame_id), (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0),
+                        1, cv2.LINE_AA)
+
+
+            cv2.imshow("output is ", cv2.resize(frame, (1000, 800)))
+            cv2.waitKey(1)
         except Exception as e:
             logger.exception(f"Error: {e}", exc_info=True)
             obj['frame'] = cv2.resize(obj['frame'], (self.config.get('FRAME_WIDTH'), self.config.get('FRAME_HEIGHT')))

scripts/utils/infocenter.py

@@ -40,16 +40,17 @@ class MongoLogger:
         try:
             input_data = {
                 "eventId": str(uuid1()).split('-')[0],
-                "cameraId": data['deviceId'],
+                "cameraId": "4",
                 "cameraName": self.camera_mapping_json.get(data['deviceId'], "Thermal Camera"),
                 "timestamp": datetime.now(),
                 "frame": data['frame'],
-                "eventtype": "Intrusion Detection",
+                "eventtype": data["message"],
                 "bg_color": data["bg_color"],
                 "font_color": data["font_color"],
-                "intrusion_message": data["message"],
+                "intrusion_message": data["event_type"],
                 "alert_sound": data["alert_sound"],
-                "logged_activity": data["activity"]}
+                "logged_activity": data["activity"],
+                "frame_id": data["frame_id"]}
             if os.environ.get('app') is not None:
                 input_data['app'] = os.environ.get('app')
 

yolov5/detect.py

@@ -36,7 +36,7 @@ def run(model1, # model path or triton URL
         data=ROOT / 'data/coco128.yaml',  # dataset.yaml path
         imgsz=(640, 640),  # inference size (height, width)
         conf_thres=0.2,  # confidence threshold
-        iou_thres=0.45,  # NMS IOU threshold
+        iou_thres=0.15,  # NMS IOU threshold
         max_det=1000,  # maximum detections per image
         device='',  # cuda device, i.e. 0 or 0,1,2,3 or cpu
         view_img=False,  # show results
@@ -59,7 +59,7 @@ def run(model1, # model path or triton URL
         dnn=False,  # use OpenCV DNN for ONNX inference
         vid_stride=1,  # video frame-rate stride
 ):
-    class_list = {0:"person", 1:"Air Breathing Mask"}
+    class_list = {0:"person", 1:"Safety helmet", 2:"Hand gloves", 3:"coverall suit", 4:"Air Breathing Mask"}
     # source = str(source)
     # save_img = not nosave and not source.endswith('.txt')  # save inference images
     # is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS)