Dev: Helm Chart enhancements in order to support data-processor modules.

automation_script.py

@@ -76,25 +76,22 @@ if __name__ == '__main__':
         "--git_repos",
         "-gr",
         required=False,
-        default=None,
+        default='',
         help="Git repos to be added in docker compose",
-        nargs="+"
     )
     ap.add_argument(
         "--module_names",
         "-mn",
         required=False,
-        default=None,
+        default='',
         help="Module names to be added in docker compose",
-        nargs="+"
     )
     ap.add_argument(
         "--repo_info",
         "-ri",
         required=False,
-        default=None,
+        default='',
         help="Module names to be added in docker compose",
-        nargs="+"
     )
     ap.add_argument(
         "--execution_type",
@@ -103,6 +100,13 @@ if __name__ == '__main__':
         default=None,
         help="Type of Execution will be selected.."
     )
+    ap.add_argument(
+        "--extra_modules",
+        "-em",
+        required=False,
+        default='data-processor',
+        help="Extra Modules to be included in script",
+    )
     try:
         arguments = vars(ap.parse_args())
         _ilens_version = arguments["ilens_version"]
@@ -111,6 +115,7 @@ if __name__ == '__main__':
         _reference_branch = arguments['reference_client_name']
         _git_repos = arguments["git_repos"]
         _module_names = arguments["module_names"]
+        extra_modules = arguments["extra_modules"]
         _branch_name = arguments['branch_name'] or "master"
         if repo_info := arguments["repo_info"]:
             repo_info = json.loads(repo_info)

docker_automation/docker_automate_script.py

@@ -37,12 +37,14 @@ class DockerVersionUpgrade:
             logging.error(f'Exception occurred while preparing the docker compose {e.args}')
 
     def process_docker_automation(self, docker_temp_path, output_path, general_temp_path):
+        _ilens_version = self.arguments["ilens_version"]
         _release_version = self.arguments["release_version"]
         _client_name = self.arguments['client_name']
-        _git_repos = self.arguments["git_repos"]
-        _module_names = self.arguments["module_names"]
+        _git_repos = self.arguments["git_repos"].split(',')
+        _module_names = self.arguments["module_names"].split(',')
+        extra_modules = self.arguments["extra_modules"].split(',')
         _branch_name = self.arguments['branch_name'] or "master"
-        _ilens_version = self.arguments["ilens_version"]
+        _reference_branch = self.arguments['reference_client_name']
         if not _ilens_version or not _release_version or not _client_name or not global_configmap:
             print(
                 "global_configmap, client_name, ilens_version and release_version details "

docker_automation/docker_register.py

@@ -33,14 +33,15 @@ class DockerRegistration:
         docker_temp_path = f"{int(time.time())}_docker_tmp_path"
         general_temp_path = f"{int(time.time())}_tmp"
         output_path = f"{int(time.time())}_docker-compose"
+        _ilens_version = self.arguments["ilens_version"]
+        _release_version = self.arguments["release_version"]
+        _client_name = self.arguments['client_name']
+        _git_repos = self.arguments["git_repos"].split(',')
+        _module_names = self.arguments["module_names"].split(',')
+        extra_modules = self.arguments["extra_modules"].split(',')
+        _branch_name = self.arguments['branch_name'] or "master"
+        _reference_branch = self.arguments['reference_client_name']
         try:
-            _ilens_version = self.arguments["ilens_version"]
-            _release_version = self.arguments["release_version"]
-            _client_name = self.arguments['client_name']
-            _reference_branch = self.arguments['reference_client_name']
-            _git_repos = self.arguments["git_repos"]
-            _module_names = self.arguments["module_names"]
-            _branch_name = self.arguments['branch_name'] or "master"
             if repo_info := self.arguments["repo_info"]:
                 repo_info = json.loads(repo_info)
             if not _ilens_version or not _release_version or not _client_name or not _git_repos and not _module_names or not global_configmap:

helm_automation/helm_automate_script.py

-from scripts.config import EnvironmentVariables
+from scripts.config import EnvironmentVariables, ModuleConstants
 from scripts.core.helm_handler import HelmHandler
 
 if __name__ == "__main__":
@@ -35,13 +35,14 @@ class HelmVersionUpgrade:
         helm_temp_path = f"{int(time.time())}_helm_tmp_path"
         general_temp_path = f"{int(time.time())}_tmp"
         output_path = f"{int(time.time())}_helm-charts"
+        _ilens_version = self.arguments["ilens_version"]
+        _release_version = self.arguments["release_version"]
+        _client_name = self.arguments['client_name']
+        _git_repos = self.arguments["git_repos"].split(',')
+        _module_names = self.arguments["module_names"].split(',')
+        extra_modules = self.arguments["extra_modules"].split(',')
+        _branch_name = self.arguments['branch_name'] or "master"
         try:
-            _ilens_version = self.arguments["ilens_version"]
-            _release_version = self.arguments["release_version"]
-            _client_name = self.arguments['client_name']
-            _git_repos = self.arguments["git_repos"]
-            _module_names = self.arguments["module_names"]
-            _branch_name = self.arguments['branch_name'] or "master"
             if not _ilens_version or not _release_version or not _client_name or not (
                     _git_repos or _module_names) or not global_configmap:
                 print(
@@ -63,7 +64,8 @@ class HelmVersionUpgrade:
                 files_info = os.listdir(base_helm_directory_path)
                 sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
                 _module_names = [_each.replace(".yml", "") for _each in sorted_files]
-
+                _module_names.extend(extra_modules)
+                _module_names = sorted(_module_names)
             global_config_data = self.common_util.convert_yaml_to_define_obj(
                 os.path.join(base_helm_directory_path, global_configmap))
             # global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
@@ -73,6 +75,8 @@ class HelmVersionUpgrade:
             if not os.path.exists(template_path):
                 os.makedirs(template_path)
             for _module in _module_names:
+                if _module == ModuleConstants.IOT_DATA_PROCESSOR:
+                    _module = ModuleConstants.IOT_DATA_PROCESSOR_MODULE
                 module_path = os.path.join(general_temp_path)
                 module_path = os.path.join(module_path, _module)
                 if not os.path.exists(module_path):

helm_automation/helm_register.py

@@ -40,14 +40,15 @@ class HelmRegistration:
         helm_temp_path = f"{int(time.time())}_helm_tmp_path"
         general_temp_path = f"{int(time.time())}_tmp"
         output_path = f"{int(time.time())}_helm-charts"
+        _ilens_version = self.arguments["ilens_version"]
+        _release_version = self.arguments["release_version"]
+        _client_name = self.arguments['client_name']
+        _git_repos = self.arguments["git_repos"].split(',')
+        _module_names = self.arguments["module_names"].split(',')
+        extra_modules = self.arguments["extra_modules"].split(',')
+        _branch_name = self.arguments['branch_name'] or "master"
+        _reference_branch = self.arguments['reference_client_name']
         try:
-            _ilens_version = self.arguments["ilens_version"]
-            _release_version = self.arguments["release_version"]
-            _client_name = self.arguments['client_name']
-            _reference_branch = self.arguments['reference_client_name']
-            _git_repos = self.arguments["git_repos"]
-            _module_names = self.arguments["module_names"]
-            _branch_name = self.arguments['branch_name'] or "master"
             repo_info = self.arguments["repo_info"]
             if repo_info:
                 repo_info = json.loads(repo_info)

scripts/config/__init__.py

 import os
 import sys
 from configparser import BasicInterpolation, ConfigParser
+from typing import Optional
+
+from pydantic import BaseSettings
 
 
 class EnvInterpolation(BasicInterpolation):
@@ -55,3 +58,11 @@ class EnvironmentVariables:
                                  default="https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/devops/docker-compose.git")
     global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="ilens-env-spec-variables.yml")
     helm_repo = os.environ.get("HELM_REPO", default="")
+
+
+class _ModuleConstants(BaseSettings):
+    IOT_DATA_PROCESSOR: Optional[str] = "mqtt-data-processor-worker"
+    IOT_DATA_PROCESSOR_MODULE: Optional[str] = "data-processor-worker"
+
+
+ModuleConstants = _ModuleConstants(_env_file=".env", _env_file_encoding="utf-8")