enh: updated code for prepartion of helm-chart preparation.

.env

 DB_URI=sqlite:///./ilens_versions.db
 CONFIG_MAP_VARIABLES=MONGO_URI
-HELM_REPO = https://gitlab-pm.knowledgelens.com/faizan.azim/utest
-GIT_USERNAME = faizan.azim
-GIT_TOKEN = MFz2FuGSasbQtBkpdfr6
+HELM_REPO = https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/devops/helm-charts
+GIT_USERNAME = harshavardhan.c
+GIT_TOKEN = FEMA6PnP63fJCs6DrtZJ
 GLOBAL_VARIABLES_FILE=ilens-global-configmap.yml

helm_automate_script.py

+import shutil
+import time
+
 if __name__ == "__main__":
     from dotenv import load_dotenv
 
     load_dotenv()
 import argparse
-import json
 import logging
 import os
-import shutil
 import sys
 
-import git
 import gitlab
 import jinja2
 import ruamel.yaml
 
 from scripts.core import ILensVersionHandler
+from scripts.core.git_handler import GitHandler
 from scripts.db.psql.databases import get_db_for_func
 from scripts.schemas import GetRequest
 
@@ -23,62 +24,26 @@ git_user_name = os.environ.get("GIT_USERNAME", default="harshavardhan.c")
 git_access_token = os.environ.get("GIT_TOKEN", default="FEMA6PnP63fJCs6DrtZJ")
 config_variables = os.environ.get("CONFIG_MAP_VARIABLES", default="").split(",")
 helm_repo = os.environ.get("HELM_REPO", default="")
-global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="")
+global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="ilens-global-configmap.yml")
 
 HELM_PATH = "/ilens-core/ilens-modules"
 HELM_STORE_PATH = "./helm-charts"
 
+git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
 
-def render_helm_chart(helm_name, data_dict, variables_list, helm_template_file):
-    try:
-        output_path = "helm-charts"
-        if not os.path.exists(output_path):
-            os.makedirs(output_path)
-        helm_path = os.path.join(output_path, f'{helm_name}.yaml')
 
+def render_helm_chart(data_dict, helm_template_file, template_path, outfile_path):
+    try:
         environment = jinja2.Environment(
-            loader=jinja2.FileSystemLoader(searchpath='./templates'),
+            loader=jinja2.FileSystemLoader(searchpath=template_path),
             trim_blocks=True,
             variable_start_string='<{', variable_end_string='}>')
-        _render = environment.get_template(helm_template_file).render(**data_dict,
-                                                                      variables=variables_list)
-        with open(helm_path, "w") as fp:
+        _render = environment.get_template(helm_template_file).render(**data_dict)
+        with open(outfile_path, "w") as fp:
             fp.write(_render)
 
     except Exception as e:
-        logging.exception(f"Exception occurred while rendering the helm file {helm_name} - {e.args}")
-
-
-def clone_repository(repo_link, module_output_path, clone_branch):
-    try:
-        if repo_link.split("https://")[-1].startswith("gitlab-pm"):
-            repo_link = repo_link.replace("https://", f"https://{git_user_name}:{git_access_token}@")
-        repo = git.Repo.clone_from(repo_link, module_output_path,
-                                   branch=clone_branch)
-        return True
-    except Exception as e:
-        logging.exception(f"Exception occurred while cloning the git repo - {repo_link} - {e.args}")
-        return False
-
-
-def clone_repository_with_defined_file(repo_link: str, file_output_path, clone_branch, private_token, clone_file_path):
-    try:
-        base_url = os.environ.get("GIT_BASE_URL", default=default_link)
-        repo_link_split = repo_link.split(base_url)
-        if not repo_link_split:
-            return False
-        gl = gitlab.Gitlab(url=base_url, private_token=private_token)
-        search_str = repo_link_split[-1].replace(".git", "")
-        pl = gl.projects.list(search=search_str)
-        if not pl:
-            return False
-        pl = pl[0]
-        with open(file_output_path, 'wb') as f:
-            pl.files.raw(file_path=clone_file_path, ref=clone_branch, streamed=True, action=f.write)
-        return True
-    except Exception as e:
-        logging.exception(f"Exception occurred while cloning the git repo - {repo_link} - {e.args}")
-        return False
+        logging.exception(f"Exception occurred while rendering the helm file  - {e.args}")
 
 
 def convert_yaml_to_json(yaml_file_path):
@@ -86,14 +51,27 @@ def convert_yaml_to_json(yaml_file_path):
         if not os.path.exists(yaml_file_path):
             return {}
         _yaml = ruamel.yaml.YAML(typ='safe')
+        _yaml.preserve_quotes = True
         with open(yaml_file_path) as fpi:
             yaml_dict = _yaml.load(fpi)
         return yaml_dict
     except Exception as e:
         logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
         return {}
-    
-def push_helm_deployments(repo_link:str, private_token:str, branch: str):
+
+
+def convert_json_to_yaml(json_data: dict, output_file_path):
+    try:
+        yaml = ruamel.yaml.YAML()
+        yaml.preserve_quotes = True
+        with open(output_file_path, 'w') as yaml_file:
+            yaml.dump(json_data, yaml_file)
+    except Exception as e:
+        logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
+        return False
+
+
+def push_helm_deployments(repo_link: str, private_token: str, branch: str, final_helm_path, base_path: str):
     try:
         base_url = os.environ.get("GIT_BASE_URL", default=default_link)
         repo_link_split = repo_link.split(base_url)
@@ -101,17 +79,19 @@ def push_helm_deployments(repo_link:str, private_token:str, branch: str):
             return False
         gl = gitlab.Gitlab(url=base_url, private_token=private_token)
         search_str = repo_link_split[-1].replace(".git", "")
-        pl = gl.projects.list(search=search_str)
+        pl = gl.projects.list(search=search_str.split("/")[-1])
         if not pl:
             return False
         pl = pl[0]
         commit_actions = []
-        files_list = os.listdir(HELM_STORE_PATH)
+        files_list = os.listdir(final_helm_path)
+        if not files_list:
+            logging.debug('Files not found for pushing to git.')
         for file in files_list:
             _action = {
                 'action': 'create',
-                'file_path': f'{HELM_PATH}/{file}',
-                'content': open(f'{HELM_STORE_PATH}/{file}').read()
+                'file_path': os.path.join(HELM_PATH, file),
+                'content': open(f'{final_helm_path}/{file}').read()
             }
             commit_actions.append(_action)
         commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
@@ -119,35 +99,6 @@ def push_helm_deployments(repo_link:str, private_token:str, branch: str):
         pl.commits.create(commit_data)
     except Exception as e:
         logging.exception(f'Exception while pushing helm deployments: {e.args}')
-        
-def pull_global_config(repo_link: str, private_token:str, branch: str, file_name: str):
-    base_url = os.environ.get("GIT_BASE_URL", default=default_link)
-    repo_link_split = repo_link.split(base_url)
-    if not repo_link_split:
-        return False
-    gl = gitlab.Gitlab(url=base_url, private_token=private_token)
-    search_str = repo_link_split[-1].replace(".git", "")
-    if pl := gl.projects.list(search=search_str):
-        pl = pl[0]
-    else:
-        return False
-    with open(file_name, 'wb') as f:
-        pl.files.raw(file_path=f'{HELM_PATH[1:]}/{file_name}', ref=branch, streamed=True, action=f.write)
-    return True
-
-def remove_all_files_from_repo(repo_link: str, private_token: str, branch: str, exclude_file: list):
-    base_url = os.environ.get("GIT_BASE_URL", default=default_link)
-    repo_link_split = repo_link.split(base_url)
-    if not repo_link_split:
-        return False
-    gl = gitlab.Gitlab(url=base_url, private_token=private_token)
-    search_str = repo_link_split[-1].replace(".git", "")
-    if pl := gl.projects.list(search=search_str):
-        pl = pl[0]
-    else:
-        return False
-    items = pl.repository_tree(path=HELM_PATH, ref=branch)
-    print(items)
 
 
 ap = argparse.ArgumentParser()
@@ -174,6 +125,13 @@ if __name__ == '__main__':
         default=None,
         help="Client Name Tag"
     )
+    ap.add_argument(
+        "--branch_name",
+        "-b",
+        required=False,
+        default=None,
+        help="Branch Name"
+    )
     ap.add_argument(
         "--git_repos",
         "-gr",
@@ -190,61 +148,93 @@ if __name__ == '__main__':
         help="Module names to be added in helm",
         nargs="+"
     )
-    arguments = vars(ap.parse_args())
-    _ilens_version = arguments["ilens_version"]
-    _release_version = arguments["release_version"]
-    _client_name = arguments['client_name']
-    _git_repos = arguments["git_repos"]
-    _module_names = arguments["module_names"]
-    if not _ilens_version or not _release_version or not _client_name or not (_git_repos or _module_names) or not global_configmap:
-        print("global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
-        sys.exit()
-    _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
-    if not pull_global_config(repo_link=helm_repo, private_token=git_access_token, branch=_branch, file_name=global_configmap):
-        logging.error(f"Cannot clone helm repo with branch: {_branch}")
-        sys.exit()
-    with open("config.json", "r") as f:
-        data = json.load(f)
-    if _git_repos:
-        data['git_modules'] = [x for x in data['git_modules'] if x['git_link'] in _git_repos]
-    else:
-        data['git_modules'] = [x for x in data['git_modules'] if x['module_name'] in _module_names]
-    global_config_data = convert_yaml_to_json(global_configmap)
-    remove_all_files_from_repo(repo_link=helm_repo, private_token=git_access_token, branch=_branch, exclude_file=[])
-    for _data in data.get('git_modules'):
-        _ilens_version = _data.get("ilens_version", _ilens_version)
-        _release_version = _data.get("ilens_version", _release_version)
-        git_link = _data["git_link"]
-        branch = _data["branch"]
-        variables_file = _data.get("variables_file") or "variables.yml"
-
-        module_path = os.path.join("tmp")
-
-        module_name = _data['module_name']
-        module_path = os.path.join(module_path, module_name)
-        if not os.path.exists(module_path):
-            os.makedirs(module_path)
-        variables_file_path = os.path.join(module_path, variables_file)
-        if not clone_repository_with_defined_file(repo_link=git_link, clone_branch=branch,
-                                                  file_output_path=variables_file_path,
-                                                  private_token=git_access_token, clone_file_path=variables_file):
-            logging.debug("Failed to clone module!! Skipping Helm File Preparation")
-            continue
-        _module_data = convert_yaml_to_json(variables_file_path)
-        global_config_vars = global_config_data['data']
-        env_variables_from_yml = _module_data.get('deployment', {}).get('environmentVar', [])
-        env_variables_from_yml = {_v['name']:_v['value'] for _v in env_variables_from_yml if
-                          {'name', 'value'}.issubset(set(list(_v.keys())))}
-        env_variables_from_yml |= global_config_vars
-        template_file = _data.get("template_file") or "helm_service_deployment.yaml"
-        session_obj = get_db_for_func()
-        module_info = db_handler.get_module_versions(
-            input_data=GetRequest(module_name=module_name, client='iLens', ilens_version=_ilens_version,
-                                  release_version=_release_version), db=session_obj)
-        session_obj.close()
-        _data["image_tag"] = module_info.get("image_tag", "-")
-        render_helm_chart(helm_name=module_name, data_dict=_data, variables_list=env_variables_from_yml,
-                          helm_template_file=template_file)
-    push_helm_deployments(helm_repo, git_access_token, _branch)
-    shutil.rmtree("./tmp")
-    shutil.rmtree(HELM_STORE_PATH)
\ No newline at end of file
+    HELM_TEMP_PATH = f"{int(time.time())}_helm_tmp_path"
+    GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
+    OUTPUT_PATH = f"{int(time.time())}_helm-charts"
+    try:
+        arguments = vars(ap.parse_args())
+        _ilens_version = arguments["ilens_version"]
+        _release_version = arguments["release_version"]
+        _client_name = arguments['client_name']
+        _git_repos = arguments["git_repos"]
+        _module_names = arguments["module_names"]
+        _branch_name = arguments['branch_name'] or "master"
+        if not _ilens_version or not _release_version or not _client_name or not (
+                _git_repos or _module_names) or not global_configmap:
+            print(
+                "global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
+            sys.exit()
+        _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
+        if not os.path.exists(HELM_TEMP_PATH):
+            os.makedirs(HELM_TEMP_PATH)
+        if not os.path.exists(OUTPUT_PATH):
+            os.makedirs(OUTPUT_PATH)
+        helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts")
+        if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
+                                                clone_branch=_client_name):
+            logging.error(f"Cannot clone helm repo with branch: {_branch}")
+            sys.exit()
+        base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
+        # global_config_data = convert_yaml_to_json(os.path.join(base_helm_directory_path, global_configmap))
+        global_config_data = convert_yaml_to_json("ilens-global-configmap.yml")
+
+        variables_file = "variables.yml"
+
+        for _module in _module_names:
+            module_path = os.path.join(GENERAL_TEMP_PATH)
+            module_path = os.path.join(module_path, _module)
+            if not os.path.exists(module_path):
+                os.makedirs(module_path)
+            variables_file_path = os.path.join(module_path, variables_file)
+            git_info = git_handler_obj.get_git_url_by_module_name(module_name=_module)
+            if not git_info:
+                logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
+                continue
+            if not git_handler_obj.clone_repository_with_defined_file(repo_link=git_info, clone_branch=_branch_name,
+                                                                      file_output_path=variables_file_path,
+                                                                      clone_file_path=variables_file):
+                logging.debug("Failed to clone module!! Skipping Helm File Preparation")
+                continue
+            existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
+            if not os.path.exists(existing_yml_path):
+                logging.debug(f"{existing_yml_path} not found!! Skipping Helm File Preparation")
+                continue
+            existing_data = convert_yaml_to_json(existing_yml_path)
+            _module_data = convert_yaml_to_json(variables_file_path)
+            global_config_vars = global_config_data.get('data', {})
+            module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
+            module_env_variables = {_v['name']: _v for _v in module_env_variables}
+            existing_env_variables = {_v['name']: _v for _v in
+                                      existing_data.get('deployment', {}).get('environmentVar', [])}
+            existing_data['deployment']['environmentVar'] = []
+            for k, v in module_env_variables.items():
+                if k.lower() in {'port', 'service_port', 'module_port'} and v.get("value"):
+                    global_config_vars["SERVICE_PORT"] = v['value']
+                    continue
+                if "valueFrom" in existing_env_variables.get(k):
+                    existing_data['deployment']['environmentVar'].append(existing_env_variables[k])
+                else:
+                    existing_data['deployment']['environmentVar'].append(v)
+            template_path = os.path.join(GENERAL_TEMP_PATH, "templates")
+            if not os.path.exists(template_path):
+                os.makedirs(template_path)
+            template_file = os.path.join(template_path, f'{_module}.yml')
+            convert_json_to_yaml(json_data=existing_data, output_file_path=template_file)
+            session_obj = get_db_for_func()
+            module_info = db_handler.get_module_versions(
+                input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
+                                      release_version=_release_version), db=session_obj)
+            session_obj.close()
+            existing_data['deployment']['imageName'] = module_info.get("image_tag",
+                                                                       existing_data['deployment']['imageName'])
+            helm_out_path = os.path.join(OUTPUT_PATH, f'{_module}.yml')
+            render_helm_chart(data_dict=global_config_vars, helm_template_file=f'{_module}.yml',
+                              outfile_path=helm_out_path, template_path=template_path)
+        push_helm_deployments(helm_repo, git_access_token, _branch, final_helm_path=OUTPUT_PATH, base_path=helm_path)
+
+    except Exception as e:
+        logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
+    finally:
+        # shutil.rmtree(HELM_STORE_PATH)
+        shutil.rmtree(HELM_TEMP_PATH)
+        shutil.rmtree(GENERAL_TEMP_PATH)

scripts/core/git_handler.py

+import os
+
+import git
+import gitlab
+
+from scripts.logging import logging
+
+default_link = "https://gitlab-pm.knowledgelens.com/"
+HELM_PATH = "/ilens-core/ilens-modules"
+HELM_STORE_PATH = "./helm-charts"
+
+
+class GitHandler:
+
+    def __init__(self, access_token, user_name):
+        self.access_token = access_token
+        self.user_name = user_name
+
+    def clone_repository(self, repo_link, module_output_path, clone_branch):
+
+        try:
+            if repo_link.split("https://")[-1].startswith("gitlab-pm"):
+                repo_link = repo_link.replace("https://", f"https://{self.user_name}:{self.access_token}@")
+            repo = git.Repo.clone_from(repo_link, module_output_path,
+                                       branch=clone_branch)
+            return True
+        except Exception as e:
+            logging.exception(f"Exception occurred while cloning the git repo - {repo_link} - {e.args}")
+            return False
+
+    def clone_repository_with_defined_file(self, repo_link: str, file_output_path, clone_branch, clone_file_path):
+        try:
+            base_url = os.environ.get("GIT_BASE_URL", default=default_link)
+            repo_link_split = repo_link.split(base_url)
+            if not repo_link_split:
+                return False
+            gl = gitlab.Gitlab(url=base_url, private_token=self.access_token)
+            search_str = repo_link_split[-1].replace(".git", "")
+            pl = gl.projects.list(search=search_str)
+            if not pl:
+                return False
+            pl = pl[0]
+            with open(file_output_path, 'wb') as f:
+                pl.files.raw(file_path=clone_file_path, ref=clone_branch, streamed=True, action=f.write)
+            return True
+        except Exception as e:
+            logging.exception(f"Exception occurred while cloning the git repo - {repo_link} - {e.args}")
+            return False
+
+    def get_git_url_by_module_name(self, module_name):
+        try:
+            base_url = os.environ.get("GIT_BASE_URL", default=default_link)
+            gl = gitlab.Gitlab(url=base_url, private_token=self.access_token)
+            pl = gl.projects.list(search=module_name, group="/KnowledgeLens/Products/iLens-2.0")
+            return pl[0].web_url if pl else ''
+        except Exception as e:
+            logging.exception(f"Exception occurred while fetching repo details - {e.args}")
+            return False
+
+    def pull_global_config(self, repo_link: str, branch: str, file_name: str):
+        base_url = os.environ.get("GIT_BASE_URL", default=default_link)
+        repo_link_split = repo_link.split(base_url)
+        if not repo_link_split:
+            return False
+        gl = gitlab.Gitlab(url=base_url, private_token=self.access_token)
+        search_str = repo_link_split[-1].replace(".git", "")
+        if pl := gl.projects.list(search=search_str):
+            pl = pl[0]
+        else:
+            return False
+        with open(file_name, 'wb') as f:
+            pl.files.raw(file_path=f'{HELM_PATH[1:]}/{file_name}', ref=branch, streamed=True, action=f.write)
+        return True
+
+    def remove_all_files_from_repo(self, repo_link: str, branch: str, exclude_file: list):
+        base_url = os.environ.get("GIT_BASE_URL", default=default_link)
+        repo_link_split = repo_link.split(base_url)
+        if not repo_link_split:
+            return False
+        gl = gitlab.Gitlab(url=base_url, private_token=self.access_token)
+        search_str = repo_link_split[-1].replace(".git", "")
+        if pl := gl.projects.list(search=search_str):
+            pl = pl[0]
+        else:
+            return False
+        items = pl.repository_tree(path=HELM_PATH, ref=branch)
+        print(items)