-- Added build-engine

build-engine/conf/application.conf

+[LOG]
+base_path = logs/
+log_level = DEBUG
+max_bytes = 100000000
+back_up_count = 5
+name = build-engine
+
+[OS]
+os = windows

build-engine/conf/git.json

+{
+  "acquisition-engine" : {
+    "git": "https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/edge/acquisition-engine.git",
+    "branch": "v5.3.3.2",
+    "directory" : {
+      "windows": "E:/iLens/build-testing/",
+      "linux": "/iLens-builds/"
+    },
+    "clone": "clone/",
+    "entrypoint": "app",
+    "build": "builds/",
+    "backup": "backup/",
+    "environment": {
+      "windows": "E:/iLens/pipeline/environments/iLens-env/Scripts/activate",
+      "linux": "/environments/acquisition-engine/bin/activate"
+    }
+  },
+  "iLens-agent": {
+    "git": "https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/edge/acquisition-engine.git",
+    "branch": "v5.3.3.2",
+    "directory" : {
+      "windows": "E:/iLens/build-testing/"
+    },
+    "clone": "clone/",
+    "entrypoint": "app",
+    "build": "builds/",
+    "backup": "backup/",
+    "environment": {
+      "windows": "E:/iLens/pipeline/environments/iLens-env/Scripts/activate",
+      "linux": "/GLens/Environments/glens_env/bin/activate"
+    }
+  },
+  "DAS": {
+    "git": "https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/edge/acquisition-engine.git",
+    "branch": "v5.3.3.1",
+    "directory" : "E:/iLens/build-testing/",
+    "clone": "clone/",
+    "entrypoint": "app",
+    "build": "builds/",
+    "backup": "backup/",
+    "environment": {
+      "windows": "E:/iLens/pipeline/environments/iLens-env/Scripts/activate",
+      "linux": "/GLens/Environments/glens_env/bin"
+    }
+  }
+}
\ No newline at end of file

build-engine/conf/server.json

+{
+  "windows" : {
+    "api": "/windows",
+    "port": 5959
+  },
+  "ubuntu": {
+
+  },
+  "pi": {
+
+  }
+}
\ No newline at end of file

build-engine/main.py

+import platform
+import uvicorn
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from scripts.core.build_engine_service import build_engine_router
+from scripts.config.config_reader import OS_CONFIG
+
+from scripts.utils.logger import logger
+
+app = FastAPI()
+origins = [
+    "http://build-engine.local.com",
+    "https://localhost.tiangolo.com",
+    "http://localhost",
+    "http://localhost:8000",
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+app.include_router(build_engine_router)
+
+if __name__ == '__main__':
+    logger.info("********** Starting build-engine **********")
+    logger.info(f"Platform : {OS_CONFIG['os']}")
+
+    uvicorn.run(app)

build-engine/requirements.txt

+fastapi[all]~=0.63.0
+uvicorn~=0.13.4
\ No newline at end of file

build-engine/scripts/config/config_reader.py

+from configparser import ConfigParser
+
+config_file_path = r"conf/application.conf"
+parser = ConfigParser()
+parser.read(config_file_path)
+
+
+def read_config():
+    config_json = dict()
+    config_json["file_path"] = parser.get("LOG", "base_path")
+    config_json["level"] = parser.get("LOG", "log_level")
+    config_json["name"] = parser.get("LOG", "name")
+    config_json["max_bytes"] = int(parser.get("LOG", "max_bytes"))
+    config_json["back_up_count"] = int(parser.get("LOG", "back_up_count"))
+
+    return config_json
+
+
+def read_os():
+    config_json = dict()
+    config_json["os"] = parser.get("OS", "os")
+    return config_json
+
+
+LOGGING_JSON = read_config()
+OS_CONFIG = read_os()

build-engine/scripts/config/json_reader.py

+import json
+from scripts.utils.logger import logger
+
+git_json = None
+server_json = None
+
+
+def read_git_json():
+    global git_json
+    try:
+        json_file = open('conf/git.json')
+        git_json = json.loads(json_file.read())
+    except Exception as es:
+        logger.exception(f'Exception while reading git.json : {es}')
+    return git_json
+
+
+def read_server_json():
+    global server_json
+    try:
+        json_file = open('conf/server.json')
+        server_json = json.loads(json_file.read())
+    except Exception as es:
+        logger.exception(f'Exception while reading server.json : {es}')
+    return server_json
+
+
+git_config = read_git_json()
+server_config = read_server_json()

build-engine/scripts/constants/app_constants.py

+from scripts.config.json_reader import server_config
+from scripts.config.config_reader import OS_CONFIG
+
+base_url = server_config[OS_CONFIG['os']]['api']
+
+
+class Build_engine_api:
+    engine_endpoint = base_url + '/build-engine'
+    download = base_url + '/download'
+    get_installed_dates = base_url + '/get_installed_dates'

build-engine/scripts/core/build.py

+import os
+from scripts.utils.logger import logger
+from scripts.utils.command import WindowsCommands, LinuxCommands
+
+
+class WindowsBuilder:
+    """
+    Windows Builder class
+    """
+
+    def __init__(self, channel_config):
+        """
+        Initialise commands
+        """
+
+        self.repository = channel_config['repository']
+        self.timestamp = channel_config['timestamp']
+        self.branch = channel_config['branch']
+
+        # initialising windows commands
+        commands = WindowsCommands(self.repository, self.branch, self.timestamp)
+
+        self.git_command = commands.git_command
+        self.change_dir = commands.change_dir
+        self.environment = commands.environment
+        self.pip = commands.pip
+        self.installer = commands.installer
+        self.robocopy = commands.robocopy
+        self.config_copy = commands.config_copy
+        self.backup = commands.backup
+        self.remove_dir = commands.remove_dir
+        self.directory = commands.get_all_dir()
+
+    def execute_build(self):
+        """
+        Module to execute windows build commands.
+
+        :return:
+        """
+
+        # command for windows build execution
+        cmd = f"{self.remove_dir} & {self.backup} & {self.git_command} & {self.change_dir} & {self.environment}.bat &" \
+              f"{self.pip} & {self.installer} & {self.config_copy} & {self.robocopy} /E"
+
+        # print('rmdir', self.remove_dir)
+        # print('backup', self.backup)
+        # print('git', self.git_command)
+        # print('cd', self.change_dir)
+        # print('env', self.environment)
+        # print('req', self.pip)
+        # print('installer', self.installer)
+        # print('robocopy', self.robocopy)
+        # return 1
+
+        # build execution
+        build_log = os.popen(cmd).read()
+        return build_log
+
+    def check_directory(self):
+        """
+        Module to check if the directory is present, if not create the directory.
+
+        :return:
+        """
+
+        for each_directory in self.directory.values():
+            if not os.path.exists(each_directory):
+                logger.warn(f"{each_directory} not found. Creating.")
+                try:
+                    os.mkdir(each_directory)
+                except Exception as es:
+                    logger.exception(f"Exception in creating {each_directory} : {es}")
+
+
+class LinuxBuilder:
+    """
+    Linux Builder Class
+    """
+
+    def __init__(self, channel_config):
+        """
+        Initialise commands
+        """
+
+        self.repository = channel_config['repository']
+        self.timestamp = channel_config['timestamp']
+        self.branch = channel_config['branch']
+
+        # initialising linux commands
+        commands = LinuxCommands(self.repository, self.branch, self.timestamp)
+
+        self.git_command = commands.git_command
+        self.change_dir = commands.change_dir
+        self.environment = commands.environment
+        self.pip = commands.pip
+        self.installer = commands.installer
+        self.copy = commands.copy
+        self.backup = commands.backup
+        self.remove_dir = commands.remove_dir
+        self.directory = commands.get_all_dir()
+        self.config_copy = commands.config_copy
+
+    def execute_build(self):
+        """
+        Module to execute linux build commands.
+
+        :return:
+        """
+
+        # commands for linux build execution
+        cmd = f"{self.remove_dir} ; {self.backup} ; {self.git_command} ; {self.change_dir} ; {self.environment} && " \
+              f"{self.pip} && {self.installer} && {self.config_copy} && {self.copy}"
+
+        # build execution
+        build_log = os.popen(cmd).read()
+        return build_log
+
+    def check_directory(self):
+        """
+        Module to check if the directory is present, if not create the directory.
+
+        :return:
+        """
+        for each_directory in self.directory.values():
+            if not os.path.exists(each_directory):
+                logger.warn(f"{each_directory} not found. Creating.")
+                try:
+                    os.mkdir(each_directory)
+                except Exception as es:
+                    logger.exception(f"Exception in creating {each_directory} : {es}")

build-engine/scripts/core/build_engine_handler.py

+import platform
+import time
+
+from scripts.core.build import WindowsBuilder, LinuxBuilder
+from scripts.utils.common_utils import file_write, write_build_log
+from scripts.utils.logger import logger
+
+
+def build(channel_data):
+    # builder platform
+    builder = None
+    if platform.system().lower() == 'windows':
+        builder = WindowsBuilder(channel_data)
+
+    elif platform.system().lower() == 'linux':
+        builder = LinuxBuilder(channel_data)
+
+    # writing current build information
+    logger.info(f"channel data : {channel_data}")
+
+    # build execution
+    builder.check_directory()
+
+    logger.info(f"Staring build : {time.ctime()}")
+    build_info = builder.execute_build()
+    logger.info(f"Ended build : {time.ctime()}")
+
+    write_build_log(platform.system().lower(), {"channel_config": channel_data, "build_log": build_info})

build-engine/scripts/core/build_engine_service.py

+import time
+import platform
+import shutil
+
+from typing import Dict
+from fastapi import APIRouter
+from fastapi.responses import FileResponse
+
+from scripts.constants.app_constants import Build_engine_api
+from scripts.core.build_engine_handler import build
+from scripts.utils.logger import logger
+from scripts.config.json_reader import git_config
+
+build_engine_router = APIRouter()
+
+
+@build_engine_router.post(f"{Build_engine_api.engine_endpoint}")
+def build_engine(git_data: Dict):
+    logger.info('POST: build_engine')
+
+    # POST method
+    channel_data = dict()
+
+    # channel data for build
+    channel_data['timestamp'] = int(time.time()) * 1000
+    channel_data['repository'] = git_data['repository']
+    channel_data['branch'] = git_data['branch']
+
+    # Build
+    build(channel_data=channel_data)
+
+    return 'received', 200
+
+
+@build_engine_router.get(f"{Build_engine_api.download}")
+def build_engine(repository: str):
+    logger.info('GET: download')
+
+    file_path = git_config[repository]['directory'][platform.system().lower()] + repository + '/' + git_config[
+        repository]['build'] + git_config[repository]['branch']
+
+    shutil.make_archive('zips/acquisition_engine', 'zip', file_path)
+    # make_zip = f"tar.exe -a -c -f {} {}"
+
+    return FileResponse(path=f"zips/acquisition_engine.zip")

build-engine/scripts/utils/command.py

+from scripts.config.json_reader import git_config
+
+
+class CommonCommands:
+    """
+    class to get common commands that is platform independent.
+    """
+
+    def __init__(self, os, build_type, branch, timestamp):
+        """
+        initialise commands
+
+        :param os:
+        :param build_type:
+        :param branch:
+        :param timestamp:
+        """
+
+        self.os = os
+        self.build_type = build_type
+        self.branch = branch
+        self.timestamp = timestamp
+        self.directory = git_config[build_type]['directory'][os]
+        # common commands
+        self.git_command = f"git clone --branch {git_config[build_type]['branch']} {git_config[build_type]['git']} " \
+                           f"{self.directory}{build_type}/" \
+                           f"{git_config[build_type]['clone']}{timestamp}"
+        self.change_dir = f"cd {self.directory}{build_type}/" \
+                          f"{git_config[build_type]['clone']}{timestamp}"
+        self.pip = 'pip install -r requirements.txt'
+        self.installer = f"pyinstaller {git_config[build_type]['entrypoint']}.py"
+        self.build_path = f"{self.directory}{build_type}/{git_config[build_type]['build']}{branch}"
+
+    def get_all_dir(self):
+        """
+        Module returns all the directory involved in the build.
+
+        :return: dict
+        """
+
+        _dir = git_config[self.build_type]['directory'][self.os]
+        directory = {
+            "main": f"{_dir}",
+            "branch": f"{_dir}{self.build_type}",
+            "build": f"{_dir}{self.build_type}/"
+                     f"{git_config[self.build_type]['build']}",
+            "clone": f"{_dir}{self.build_type}/"
+                     f"{git_config[self.build_type]['clone']}",
+            "backup": f"{_dir}{self.build_type}/"
+                      f"{git_config[self.build_type]['backup']}",
+        }
+        return directory
+
+    @staticmethod
+    def cwd():
+        return f"cd"
+
+
+class WindowsCommands(CommonCommands):
+    """
+    class to get windows commands
+    """
+
+    def __init__(self, build_type, branch, timestamp):
+        """
+        initialise commands.
+
+        :param build_type:
+        :param branch:
+        :param timestamp:
+        """
+
+        # common commands
+        CommonCommands.__init__(self, 'windows', build_type, branch, timestamp)
+
+        # windows specific commands
+        self.directory = git_config[build_type]['directory']['windows']
+        self.backup_dir = git_config[build_type]['backup']
+        self.build_dir = git_config[build_type]['build']
+        self.environment = f"{git_config[build_type]['environment']['windows']}"
+        self.robocopy = f"robocopy {self.directory}{build_type}/" \
+                        f"{git_config[build_type]['clone']}{timestamp}/dist/{git_config[build_type]['entrypoint']} " \
+                        f"{self.directory}{build_type}/{git_config[build_type]['build']}{branch}/"
+        self.remove_dir = f'rmdir /s /q "{self.directory}{build_type}/{self.backup_dir}{branch}"'
+        self.backup = f"move {self.directory}{build_type}/{self.build_dir}{branch} {self.directory}{build_type}/backup"
+        self.config_copy = f"robocopy {self.directory}{build_type}/{git_config[build_type]['clone']}{timestamp}/conf " \
+                           f"{self.directory}{build_type}/{git_config[build_type]['clone']}{timestamp}" \
+                           f"/dist/{git_config[build_type]['entrypoint']}/conf/"
+
+
+class LinuxCommands(CommonCommands):
+    """
+    Class to get linux commands.
+    """
+
+    def __init__(self, build_type, branch, timestamp):
+        """
+        initialise commands.
+
+        :param build_type:
+        :param branch:
+        :param timestamp:
+        """
+
+        # common commands
+        CommonCommands.__init__(self, 'linux', build_type, branch, timestamp)
+
+        # linux specific commands
+        self.directory = git_config[build_type]['directory']['linux']
+        self.backup_dir = git_config[build_type]['backup']
+        self.build_dir = git_config[build_type]['build']
+        self.environment = f". {git_config[build_type]['environment']['linux']}"
+        self.copy = f"cp -r {self.directory}{build_type}/" \
+                    f"{git_config[build_type]['clone']}{timestamp}/dist/{git_config[build_type]['entrypoint']} " \
+                    f"{self.directory}{build_type}/{git_config[build_type]['build']}{branch}/"
+        self.remove_dir = f'rm -rf "{self.directory}{build_type}/{self.backup_dir}{branch}"'
+        self.backup = f"mv {self.directory}{build_type}/{self.build_dir}{branch} {self.directory}{build_type}/backup"
+        self.config_copy = f"cp -r {self.directory}{build_type}/{git_config[build_type]['clone']}{timestamp}/conf " \
+                           f"{self.directory}{build_type}/{git_config[build_type]['clone']}{timestamp}" \
+                           f"/dist/{git_config[build_type]['entrypoint']}/"

build-engine/scripts/utils/common_utils.py

+import json
+import time
+from scripts.utils.logger import logger
+from scripts.utils.command import *
+
+
+def write_build_log(os, build_info):
+    """
+    Module to write build logs.
+
+    :param build_info:
+    :param os:
+    :return:
+    """
+
+    commands = None
+
+    # getting build information
+    if os == 'windows':
+        commands = WindowsCommands(build_info['channel_config']['repository'],
+                                   build_info['channel_config']['branch'],
+                                   build_info['channel_config']['timestamp'])
+    elif os == 'linux':
+        commands = LinuxCommands(build_info['channel_config']['repository'],
+                                 build_info['channel_config']['branch'],
+                                 build_info['channel_config']['timestamp'])
+
+    # writing build info
+    file_write(f"{commands.build_path}.txt", build_info)
+    file_write(f"{commands.build_path}/version.txt", {"version": build_info['channel_config']['branch'],
+                                                      "Date": time.asctime()})
+
+
+def file_write(path, data):
+    """
+    Module to write to file.
+
+    :param path:
+    :param data:
+    :return:
+    """
+
+    try:
+        with open(path, 'w+') as file:
+            file.write(json.dumps(data))
+            file.close()
+    except Exception as es:
+        logger.exception(f"Exception while writing {path} : {es}")
+        return False
+
+    return True

build-engine/scripts/utils/logger.py

+import logging
+import os
+from logging.handlers import RotatingFileHandler
+
+from scripts.config.config_reader import LOGGING_JSON
+
+
+def get_logger():
+    """
+     Creates a rotating log
+     """
+    if not os.path.exists(LOGGING_JSON["file_path"]):
+        os.makedirs(LOGGING_JSON["file_path"])
+
+    __logger__ = logging.getLogger('')
+    __logger__.setLevel(LOGGING_JSON["level"].upper())
+    log_formatter = '%(asctime)s - %(levelname)-6s - %(message)s'
+    time_format = "%Y-%m-%d %H:%M:%S"
+    formatter = logging.Formatter(log_formatter, time_format)
+    log_file = os.path.join(LOGGING_JSON["file_path"] + LOGGING_JSON["name"] + '.log')
+
+    temp_handler = RotatingFileHandler(log_file,
+                                       maxBytes=LOGGING_JSON["max_bytes"],
+                                       backupCount=LOGGING_JSON["back_up_count"])
+    temp_handler.setFormatter(formatter)
+    __logger__.addHandler(temp_handler)
+
+    return __logger__
+
+# import logging.handlers
+# from logging.handlers import RotatingFileHandler
+#
+#
+# def get_logger():
+#     log_file = os.path.join(LOGGING_JSON["file_path"] + LOGGING_JSON["name"] + '.log')
+#     if not os.path.exists(LOGGING_JSON["file_path"]):
+#         os.makedirs(LOGGING_JSON["file_path"])
+#     __logger__ = logging.getLogger(LOGGING_JSON["name"] + '_Client')
+#     __logger__.propagate = False
+#     formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', "%Y-%m-%d %H:%M:%S")
+#     __logger__.setLevel(logging.DEBUG)
+#
+#     handler = RotatingFileHandler(log_file,
+#                                   maxBytes=int(LOGGING_JSON["max_bytes"]),
+#                                   backupCount=int(LOGGING_JSON["back_up_count"]))
+#     handler.setFormatter(formatter)
+#     if __logger__.hasHandlers():
+#         __logger__.handlers.clear()
+#
+#     __logger__.addHandler(handler)
+#     __logger__.debug('Logger Initialized')
+#
+#     return __logger__
+
+
+logger = get_logger()