enh: Code enhancements

helm_automate_script.py

-import shutil
-import time
-
 if __name__ == "__main__":
     from dotenv import load_dotenv
 
@@ -8,16 +5,18 @@ if __name__ == "__main__":
 import argparse
 import logging
 import os
+import shutil
 import sys
+import time
 
 import gitlab
-import jinja2
-import ruamel.yaml
 
 from scripts.core import ILensVersionHandler
 from scripts.core.git_handler import GitHandler
+from scripts.core.helm_handler import HelmHandler
 from scripts.db.psql.databases import get_db_for_func
 from scripts.schemas import GetRequest
+from scripts.utils.common_utils import CommonUtils
 
 default_link = "https://gitlab-pm.knowledgelens.com/"
 git_user_name = os.environ.get("GIT_USERNAME", default="harshavardhan.c")
@@ -32,45 +31,6 @@ HELM_STORE_PATH = "./helm-charts"
 git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
 
 
-def render_helm_chart(data_dict, helm_template_file, template_path, outfile_path):
-    try:
-        environment = jinja2.Environment(
-            loader=jinja2.FileSystemLoader(searchpath=template_path),
-            trim_blocks=True,
-            variable_start_string='<{', variable_end_string='}>')
-        _render = environment.get_template(helm_template_file).render(**data_dict)
-        with open(outfile_path, "w") as fp:
-            fp.write(_render)
-
-    except Exception as e:
-        logging.exception(f"Exception occurred while rendering the helm file  - {e.args}")
-
-
-def convert_yaml_to_json(yaml_file_path):
-    try:
-        if not os.path.exists(yaml_file_path):
-            return {}
-        _yaml = ruamel.yaml.YAML(typ='safe')
-        _yaml.preserve_quotes = True
-        with open(yaml_file_path) as fpi:
-            yaml_dict = _yaml.load(fpi)
-        return yaml_dict
-    except Exception as e:
-        logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
-        return {}
-
-
-def convert_json_to_yaml(json_data: dict, output_file_path):
-    try:
-        yaml = ruamel.yaml.YAML()
-        yaml.preserve_quotes = True
-        with open(output_file_path, 'w') as yaml_file:
-            yaml.dump(json_data, yaml_file)
-    except Exception as e:
-        logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
-        return False
-
-
 def push_helm_deployments(repo_link: str, private_token: str, branch: str, final_helm_path, base_path: str):
     try:
         base_url = os.environ.get("GIT_BASE_URL", default=default_link)
@@ -105,6 +65,8 @@ def push_helm_deployments(repo_link: str, private_token: str, branch: str, final
 
 ap = argparse.ArgumentParser()
 db_handler = ILensVersionHandler()
+common_util = CommonUtils()
+helm_handler = HelmHandler()
 if __name__ == '__main__':
     ap.add_argument(
         "--ilens_version",
@@ -178,10 +140,12 @@ if __name__ == '__main__':
             sys.exit()
         base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
         # global_config_data = convert_yaml_to_json(os.path.join(base_helm_directory_path, global_configmap))
-        global_config_data = convert_yaml_to_json("ilens-global-configmap.yml")
+        global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
 
         variables_file = "variables.yml"
-
+        template_path = os.path.join(GENERAL_TEMP_PATH, "templates")
+        if not os.path.exists(template_path):
+            os.makedirs(template_path)
         for _module in _module_names:
             module_path = os.path.join(GENERAL_TEMP_PATH)
             module_path = os.path.join(module_path, _module)
@@ -197,42 +161,33 @@ if __name__ == '__main__':
                                                                       clone_file_path=variables_file):
                 logging.debug("Failed to clone module!! Skipping Helm File Preparation")
                 continue
-            existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
-            if not os.path.exists(existing_yml_path):
-                logging.debug(f"{existing_yml_path} not found!! Skipping Helm File Preparation")
-                continue
-            existing_data = convert_yaml_to_json(existing_yml_path)
-            _module_data = convert_yaml_to_json(variables_file_path)
-            global_config_vars = global_config_data.get('data', {})
+            _module_data = common_util.convert_yaml_to_json(variables_file_path)
             module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
             module_env_variables = {_v['name']: _v for _v in module_env_variables}
-            existing_env_variables = {_v['name']: _v for _v in
-                                      existing_data.get('deployment', {}).get('environmentVar', [])}
-            existing_data['deployment']['environmentVar'] = []
-            for k, v in module_env_variables.items():
-                if k.lower() in {'port', 'service_port', 'module_port'} and v.get("value"):
-                    global_config_vars["SERVICE_PORT"] = v['value']
-                    continue
-                if "valueFrom" in existing_env_variables.get(k, []):
-                    existing_data['deployment']['environmentVar'].append(existing_env_variables[k])
-                else:
-                    existing_data['deployment']['environmentVar'].append(v)
-            template_path = os.path.join(GENERAL_TEMP_PATH, "templates")
-            if not os.path.exists(template_path):
-                os.makedirs(template_path)
             template_file = os.path.join(template_path, f'{_module}.yml')
-            convert_json_to_yaml(json_data=existing_data, output_file_path=template_file)
+            helm_out_file_path = os.path.join(OUTPUT_PATH, f'{_module}.yml')
             session_obj = get_db_for_func()
             module_info = db_handler.get_module_versions(
                 input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
                                       release_version=_release_version), db=session_obj)
             session_obj.close()
-            existing_data['deployment']['imageName'] = module_info.get("image_tag",
-                                                                       existing_data['deployment']['imageName'])
-            helm_out_path = os.path.join(OUTPUT_PATH, f'{_module}.yml')
-            render_helm_chart(data_dict=global_config_vars, helm_template_file=f'{_module}.yml',
-                              outfile_path=helm_out_path, template_path=template_path)
-        push_helm_deployments(helm_repo, git_access_token, _branch, final_helm_path=OUTPUT_PATH, base_path=helm_path)
+            image_url = ''
+            existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
+            helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url,
+                                                     module_env_variables=module_env_variables,
+                                                     template_file=template_file, template_path=template_path,
+                                                     helm_out_file_path=helm_out_file_path,
+                                                     global_config_data=global_config_data, module_name=_module)
+            celery_yml_path = os.path.join(base_helm_directory_path, f'{_module}-celery.yml')
+            if os.path.exists(celery_yml_path):
+                helm_out_file_path = os.path.join(OUTPUT_PATH, f'{_module}-celery.yml')
+                helm_handler.create_helm_deployment_file(template_yml_path=celery_yml_path, image_tag=image_url,
+                                                         module_env_variables=module_env_variables,
+                                                         template_file=template_file, template_path=template_path,
+                                                         helm_out_file_path=helm_out_file_path,
+                                                         global_config_data=global_config_data,
+                                                         module_name=f'{_module}-celery')
+        # push_helm_deployments(helm_repo, git_access_token, _branch, final_helm_path=OUTPUT_PATH, base_path=helm_path)
 
     except Exception as e:
         logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")

scripts/core/helm_handler.py

+import os
+
+import jinja2
+
+from scripts.logging import logging
+from scripts.utils.common_utils import CommonUtils
+
+
+class HelmHandler:
+    def __init__(self):
+        self.common_utils = CommonUtils()
+        ...
+
+    def create_helm_deployment_file(self, template_yml_path, module_env_variables: dict, global_config_data: dict,
+                                    image_tag: str, module_name: str, template_file: str, helm_out_file_path: str,
+                                    template_path: str):
+        try:
+            if not os.path.exists(template_yml_path):
+                logging.debug(f"{template_yml_path} not found!! Skipping Helm File Preparation")
+                return False
+            existing_data = self.common_utils.convert_yaml_to_json(template_yml_path)
+            global_config_vars = global_config_data.get('data', {})
+            existing_env_variables = {_v['name']: _v for _v in
+                                      existing_data.get('deployment', {}).get('environmentVar', [])}
+            existing_data['deployment']['environmentVar'] = []
+            for k, v in module_env_variables.items():
+                if k.lower() in {'port', 'service_port', 'module_port'} and v.get("value"):
+                    global_config_vars["SERVICE_PORT"] = v['value']
+                    continue
+                if "valueFrom" in existing_env_variables.get(k, []):
+                    existing_data['deployment']['environmentVar'].append(existing_env_variables[k])
+                else:
+                    existing_data['deployment']['environmentVar'].append(v)
+            existing_data['deployment']['imageName'] = image_tag or existing_data['deployment']['imageName']
+
+            self.common_utils.convert_json_to_yaml(json_data=existing_data, output_file_path=template_file)
+            self.render_helm_chart(data_dict=global_config_vars, helm_template_file=f'{module_name}.yml',
+                                   outfile_path=helm_out_file_path, template_path=template_path)
+        except Exception as e:
+            logging.exception(f'Exception occurred while preparing the helm deployment file {e.args}')
+            return False
+
+    @staticmethod
+    def render_helm_chart(data_dict, helm_template_file, template_path, outfile_path):
+        try:
+            environment = jinja2.Environment(
+                loader=jinja2.FileSystemLoader(searchpath=template_path),
+                trim_blocks=True,
+                variable_start_string='<{', variable_end_string='}>')
+            _render = environment.get_template(helm_template_file).render(**data_dict)
+            with open(outfile_path, "w") as fp:
+                fp.write(_render)
+
+        except Exception as e:
+            logging.exception(f"Exception occurred while rendering the helm file  - {e.args}")

scripts/utils/common_utils.py

+import os
+
+import ruamel
+
+from scripts.logging import logging
+
+
+class CommonUtils:
+
+    @staticmethod
+    def convert_yaml_to_json(yaml_file_path):
+        try:
+            if not os.path.exists(yaml_file_path):
+                return {}
+            _yaml = ruamel.yaml.YAML(typ='safe')
+            _yaml.preserve_quotes = True
+            with open(yaml_file_path) as fpi:
+                yaml_dict = _yaml.load(fpi)
+            return yaml_dict
+        except Exception as e:
+            logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
+            return {}
+
+    @staticmethod
+    def convert_json_to_yaml(json_data: dict, output_file_path):
+        try:
+            yaml = ruamel.yaml.YAML()
+            yaml.preserve_quotes = True
+            with open(output_file_path, 'w') as yaml_file:
+                yaml.dump(json_data, yaml_file)
+        except Exception as e:
+            logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
+            return False