Merge branch 'develop' of...

Merge branch 'develop' of https://gitlab-pm.knowledgelens.com/Harshavardhan.C/helm-automation-script into develop

helm_automate_script.py

@@ -8,8 +8,6 @@ import os
 import sys
 import time
 
-import gitlab
-
 from scripts.core import ILensVersionHandler
 from scripts.core.git_handler import GitHandler
 from scripts.core.helm_handler import HelmHandler
@@ -29,43 +27,6 @@ HELM_STORE_PATH = "./helm-charts"
 
 git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
 
-
-def push_helm_deployments(repo_link: str, private_token: str, branch: str, final_helm_path, base_path: str):
-    try:
-        base_url = os.environ.get("GIT_BASE_URL", default=default_link)
-        repo_link_split = repo_link.split(base_url)
-        if not repo_link_split:
-            return False
-        gl = gitlab.Gitlab(url=base_url, private_token=private_token)
-        search_str = repo_link_split[-1].replace(".git", "")
-        pl = gl.projects.list(search=search_str)
-        if not pl:
-            return False
-        pl = pl[0]
-        commit_actions = []
-        files_list = os.listdir(final_helm_path)
-        branches = pl.branches.list(get_all=True)
-        branches_names = [x.name for x in branches]
-        if branch not in branches_names:
-            pl.branches.create({'branch': branch,
-                                'ref': 'master' if branch.split("_")[0] not in branches_names else branch.split("_")[
-                                    0]})
-        if not files_list:
-            logging.debug('Files not found for pushing to git.')
-        for file in files_list:
-            _action = {
-                'action': 'update',
-                'file_path': f"{HELM_PATH}/{file}",
-                'content': open(f'{final_helm_path}/{file}').read()
-            }
-            commit_actions.append(_action)
-        commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
-
-        pl.commits.create(commit_data)
-    except Exception as e:
-        logging.exception(f'Exception while pushing helm deployments: {e.args}')
-
-
 ap = argparse.ArgumentParser()
 db_handler = ILensVersionHandler()
 common_util = CommonUtils()
@@ -139,7 +100,7 @@ if __name__ == '__main__':
         helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts")
         if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
                                                 clone_branch=_client_name):
-            logging.error(f"Cannot clone helm repo with branch: {_branch}")
+            logging.error(f"Cannot clone helm repo with branch: {_client_name}")
             sys.exit()
         base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
         if len(_module_names) == 1 and _module_names[0].lower() == "all":
@@ -183,13 +144,14 @@ if __name__ == '__main__':
                 input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
                                       release_version=_release_version), db=session_obj)
             session_obj.close()
-            image_url = module_info.get("image_url")
+            image_url = module_info.get("image_url", '') if module_info else ''
             existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
-            helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url,
+            helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url,
                                                               module_env_variables=module_env_variables,
                                                               template_file=template_file, template_path=template_path,
                                                               helm_out_file_path=helm_out_file_path,
-                                                     global_config_data=global_config_data, module_name=_module)
+                                                              global_config_data=global_config_data,
+                                                              module_name=_module)
             files_info = os.listdir(base_helm_directory_path)
             sorted_files = list(filter(lambda f: f.startswith(_module), files_info))
             updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x]
@@ -197,13 +159,14 @@ if __name__ == '__main__':
                 sub_file_path = os.path.join(base_helm_directory_path, _file)
                 helm_out_file_path = os.path.join(OUTPUT_PATH, _file)
                 template_file = os.path.join(template_path, _file)
-                helm_handler.create_helm_deployment_file(template_yml_path=sub_file_path, image_tag=image_url,
+                helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path, image_tag=image_url,
                                                                   module_env_variables=module_env_variables,
-                                                         template_file=template_file, template_path=template_path,
+                                                                  template_file=template_file,
+                                                                  template_path=template_path,
                                                                   helm_out_file_path=helm_out_file_path,
                                                                   global_config_data=global_config_data,
                                                                   module_name=_file.split(".yml")[0])
-        push_helm_deployments(helm_repo, git_access_token, _client_name, final_helm_path=OUTPUT_PATH,
+        git_handler_obj.push_helm_deployments(helm_repo, git_access_token, _branch, final_helm_path=OUTPUT_PATH,
                                               base_path=helm_path)
         git_handler_obj.create_merge_request(repo_link=helm_repo, source_branch=_branch, destination_branch=_client_name)
     except Exception as e:

helm_register.py

+from copy import deepcopy
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+
+    load_dotenv()
+import argparse
+import logging
+import os
+import sys
+import time
+
+from scripts.core import ILensVersionHandler
+from scripts.core.git_handler import GitHandler
+from scripts.core.helm_handler import HelmHandler
+from scripts.utils.common_utils import CommonUtils
+
+default_link = "https://gitlab-pm.knowledgelens.com/"
+git_user_name = os.environ.get("GIT_USERNAME", default="harshavardhan.c")
+git_access_token = os.environ.get("GIT_TOKEN", default="FEMA6PnP63fJCs6DrtZJ")
+config_variables = os.environ.get("CONFIG_MAP_VARIABLES", default="").split(",")
+helm_repo = os.environ.get("HELM_REPO", default="")
+global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="ilens-env-spec-variables.yml")
+
+HELM_PATH = "/ilens-core/ilens-modules"
+HELM_STORE_PATH = "./helm-charts"
+
+git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
+
+ap = argparse.ArgumentParser()
+db_handler = ILensVersionHandler()
+common_util = CommonUtils()
+helm_handler = HelmHandler()
+if __name__ == '__main__':
+    ap.add_argument(
+        "--ilens_version",
+        "-iv",
+        required=False,
+        default=None,
+        help="ILens Version Tag",
+    )
+    ap.add_argument(
+        "--release_version",
+        "-rv",
+        required=False,
+        default=None,
+        help="ILens Release Tag",
+    )
+    ap.add_argument(
+        "--reference_client_name",
+        "-rcn",
+        required=False,
+        default=None,
+        help="Client Name Tag"
+    )
+    ap.add_argument(
+        "--client_name",
+        "-cn",
+        required=False,
+        default=None,
+        help="Client Name Tag"
+    )
+    ap.add_argument(
+        "--branch_name",
+        "-b",
+        required=False,
+        default=None,
+        help="Branch Name"
+    )
+    ap.add_argument(
+        "--git_repos",
+        "-gr",
+        required=False,
+        default=None,
+        help="Git repos to be added in helm",
+        nargs="+"
+    )
+    ap.add_argument(
+        "--module_names",
+        "-mn",
+        required=False,
+        default=None,
+        help="Module names to be added in helm",
+        nargs="+"
+    )
+    ap.add_argument(
+        "--repo_info",
+        "-ri",
+        required=False,
+        default=None,
+        help="Module names to be added in helm",
+        nargs="+"
+    )
+    HELM_TEMP_PATH = f"{int(time.time())}_helm_tmp_path"
+    GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
+    OUTPUT_PATH = f"{int(time.time())}_helm-charts"
+    try:
+        arguments = vars(ap.parse_args())
+        _ilens_version = arguments["ilens_version"]
+        _release_version = arguments["release_version"]
+        _client_name = arguments['client_name']
+        _reference_branch = arguments['reference_client_name']
+        _git_repos = arguments["git_repos"]
+        repo_info = arguments["repo_info"]
+        _module_names = arguments["module_names"]
+        _branch_name = arguments['branch_name'] or "master"
+        if not _ilens_version or not _release_version or not _client_name or not (
+                _git_repos or _module_names) or not global_configmap:
+            print(
+                "global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
+            sys.exit()
+        _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
+        if not os.path.exists(HELM_TEMP_PATH):
+            os.makedirs(HELM_TEMP_PATH)
+        if not os.path.exists(OUTPUT_PATH):
+            os.makedirs(OUTPUT_PATH)
+        helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts")
+        # if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
+        #                                         clone_branch=_reference_branch or _client_name):
+        #     logging.error(f"Cannot clone helm repo with branch: {_reference_branch or _client_name}")
+        base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
+        if os.path.exists(base_helm_directory_path) and len(_module_names) == 1 and _module_names[0].lower() == "all":
+            updated_list = []
+            files_info = os.listdir(base_helm_directory_path)
+            sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
+            _module_names = [_each.replace(".yml", "") for _each in sorted_files]
+
+        global_config_data = common_util.convert_yaml_to_json(os.path.join(base_helm_directory_path, global_configmap))
+        # global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
+
+        variables_file = "variables.yml"
+        template_path = os.path.join(GENERAL_TEMP_PATH, "templates")
+        if not os.path.exists(template_path):
+            os.makedirs(template_path)
+        for _module in _module_names:
+            module_path = os.path.join(GENERAL_TEMP_PATH)
+            module_path = os.path.join(module_path, _module)
+            if not os.path.exists(module_path):
+                os.makedirs(module_path)
+            helm_out_file_path = os.path.join(OUTPUT_PATH, f'{_module}.yml')
+            if os.path.exists(helm_out_file_path):
+                logging.debug(f"Helm Deployment File found for selected the module {_module}")
+                continue
+            variables_file_path = os.path.join(module_path, variables_file)
+            git_info = git_handler_obj.get_git_url_by_module_name(module_name=_module)
+            if not git_info:
+                logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
+                continue
+            if not git_handler_obj.clone_repository_with_defined_file(repo_link=git_info, clone_branch=_branch_name,
+                                                                      file_output_path=variables_file_path,
+                                                                      clone_file_path=variables_file):
+                logging.debug("Failed to clone module!! Skipping Helm File Preparation")
+                continue
+            _module_data = common_util.convert_yaml_to_json(variables_file_path)
+            module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
+
+            module_env_variables = {_v['name']: _v['value'] for _v in module_env_variables if
+                                    {'name', 'value'}.issubset(set(list(_v.keys())))}
+            template_file = os.path.join(template_path, f'{_module}.yml')
+            # session_obj = get_db_for_func()
+            # module_info = db_handler.get_module_versions(
+            #     input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
+            #                           release_version=_release_version), db=session_obj)
+            # session_obj.close()
+            module_info = {}
+            image_url = module_info.get("image_url", '') if module_info else ''
+            existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
+            new_module = deepcopy(_module)
+            if not os.path.exists(existing_yml_path):
+                existing_yml_path = os.path.join("templates", "helm_deployment.yml")
+                new_module = "helm_deployment"
+            worker_type = True if "worker" in _module | "celery" in _module else False
+            helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url,
+                                                     module_env_variables=module_env_variables,
+                                                     helm_out_file_path=helm_out_file_path,
+                                                     global_config_data=global_config_data, module_name=new_module)
+            files_info = os.listdir(base_helm_directory_path)
+            sorted_files = list(filter(lambda f: f.startswith(_module), files_info))
+            updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x]
+            for _file in updated_files_info:
+                sub_file_path = os.path.join(base_helm_directory_path, _file)
+                helm_out_file_path = os.path.join(OUTPUT_PATH, _file)
+                template_file = os.path.join(template_path, _file)
+                worker_type = True if "worker" in _module | "celery" in _module else False
+                helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path, image_tag=image_url,
+                                                                  module_env_variables=module_env_variables,
+                                                                  template_file=template_file,
+                                                                  template_path=template_path,
+                                                                  helm_out_file_path=helm_out_file_path,
+                                                                  global_config_data=global_config_data,
+                                                                  service_type=False,
+                                                                  module_name=_file.split(".yml")[0])
+        git_handler_obj.push_helm_deployments(helm_repo, git_access_token, _client_name, final_helm_path=OUTPUT_PATH,
+                                              base_path=helm_path)
+
+    except Exception as e:
+        logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
+    finally:
+        # shutil.rmtree(HELM_STORE_PATH)
+        # shutil.rmtree(HELM_TEMP_PATH)
+        # shutil.rmtree(GENERAL_TEMP_PATH)
+        # shutil.rmtree(OUTPUT_PATH)
+        ...

scripts/core/git_handler.py

@@ -106,3 +106,40 @@ class GitHandler:
             'target_branch': destination_branch,
             'title': f'{source_branch} merge'
         })
+
+    @staticmethod
+    def push_helm_deployments(repo_link: str, private_token: str, branch: str, final_helm_path, base_path: str):
+        try:
+            base_url = os.environ.get("GIT_BASE_URL", default=default_link)
+            repo_link_split = repo_link.split(base_url)
+            if not repo_link_split:
+                return False
+            gl = gitlab.Gitlab(url=base_url, private_token=private_token)
+            search_str = repo_link_split[-1].replace(".git", "")
+            pl = gl.projects.list(search=search_str)
+            if not pl:
+                return False
+            pl = pl[0]
+            commit_actions = []
+            files_list = os.listdir(final_helm_path)
+            branches = pl.branches.list(get_all=True)
+            branches_names = [x.name for x in branches]
+            if branch not in branches_names:
+                pl.branches.create({'branch': branch,
+                                    'ref': 'master' if branch.split("_")[0] not in branches_names else
+                                    branch.split("_")[
+                                        0]})
+            if not files_list:
+                logging.debug('Files not found for pushing to git.')
+            for file in files_list:
+                _action = {
+                    'action': 'update',
+                    'file_path': f"{HELM_PATH}/{file}",
+                    'content': open(f'{final_helm_path}/{file}').read()
+                }
+                commit_actions.append(_action)
+            commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
+
+            pl.commits.create(commit_data)
+        except Exception as e:
+            logging.exception(f'Exception while pushing helm deployments: {e.args}')

scripts/core/helm_handler.py

@@ -11,9 +11,11 @@ class HelmHandler:
         self.common_utils = CommonUtils()
         ...
 
-    def create_helm_deployment_file(self, template_yml_path, module_env_variables: dict, global_config_data: dict,
-                                    image_tag: str, module_name: str, template_file: str, helm_out_file_path: str,
-                                    template_path: str):
+    def create_existing_helm_deployment_file(self, template_yml_path, module_env_variables: dict,
+                                             global_config_data: dict,
+                                             image_tag: str, module_name: str, template_file: str,
+                                             helm_out_file_path: str,
+                                             template_path: str, service_type=True):
         try:
             if not os.path.exists(template_yml_path):
                 logging.debug(f"{template_yml_path} not found!! Skipping Helm File Preparation")
@@ -34,22 +36,43 @@ class HelmHandler:
 
             self.common_utils.convert_json_to_yaml(json_data=existing_data, output_file_path=template_file)
             self.render_helm_chart(data_dict=global_config_vars, helm_template_file=f'{module_name}.yml',
-                                   outfile_path=helm_out_file_path, template_path=template_path)
+                                   outfile_path=helm_out_file_path, template_path=template_path,
+                                   service_type=service_type)
             return True
         except Exception as e:
             logging.exception(f'Exception occurred while preparing the helm deployment file {e.args}')
             return False
 
     @staticmethod
-    def render_helm_chart(data_dict, helm_template_file, template_path, outfile_path):
+    def render_helm_chart(data_dict, helm_template_file, template_path, outfile_path, **kwargs):
         try:
             environment = jinja2.Environment(
                 loader=jinja2.FileSystemLoader(searchpath=template_path),
                 trim_blocks=True,
                 variable_start_string='<{', variable_end_string='}>', autoescape=True)
-            _render = environment.get_template(helm_template_file).render(**data_dict)
+            _render = environment.get_template(helm_template_file).render(**kwargs, **data_dict)
             with open(outfile_path, "w") as fp:
                 fp.write(_render)
 
         except Exception as e:
             logging.exception(f"Exception occurred while rendering the helm file  - {e.args}")
+
+    def create_helm_deployment_file(self, template_yml_path, module_env_variables: dict, global_config_data: dict,
+                                    image_tag: str, module_name: str, helm_out_file_path: str, service_type=True):
+        try:
+            if not os.path.exists(template_yml_path):
+                logging.debug(f"{template_yml_path} not found!! Skipping Helm File Preparation")
+                return False
+            global_config_vars = global_config_data.get('data', {})
+            for k, v in module_env_variables.items():
+                if global_config_vars.get(k):
+                    continue
+                global_config_vars.update({k: v})
+
+            self.render_helm_chart(data_dict=global_config_vars, helm_template_file=f'{module_name}.yml',
+                                   outfile_path=helm_out_file_path, template_path="templates",
+                                   service_type=service_type, image_url=image_tag)
+            return True
+        except Exception as e:
+            logging.exception(f'Exception occurred while preparing the helm deployment file {e.args}')
+            return False

templates/helm_deployment.yaml

-name: <{ module_name }>
-codeType: backend
-type: core
-
-affinity:
-  enabled: True
-  values: [ "klblrserv24","klblrserv23" ]
-
-podAutoScaler:
-  enabled: False
-  type: "hpa"
-  maxReplicaCount: 3
-  minReplicaCount: 1
-  scalePercentage: 85
-
-deployment:
-  imageName: <{ image_tag }>
-  command: <{ command }>
-  PullSecrets: <{ PullSecrets | default("ilens-azregistry") }>
-  PullPolicy: IfNotPresent <{ branch }>
-  resources:
-    requests:
-      memory: <{ request_memory | default("250Mi") }>
-      cpu: <{ request_cpu | default("250Mi") }>
-    limits:
-      memory: <{ limit_memory | default("750Mi") }>
-      cpu: <{ limit_cpu | default("500Mi") }>
-
-  environmentVar:
-    - name: MONGO_URI
-      valueFrom:
-        secretKeyRef:
-          name: mongo-uri
-          key: MONGO_URI
-{% for k,v in variables.items() %}
-{% if k and v %}
-    - name: '<{ k }>',
-      value: '<{ v }>'
-{% endif %}
-{% endfor %}
-
-
-mountVolume:
-  enabled: <{ enableMountVolume }>
-  volumeMounts:
-    - name: <{ mount_name | default("core-volumes") }>
-      mountPath: "/code/data"
-  volumes:
-    - name: <{ mount_name | default("core-volumes") }>
-      persistentVolumeClaim:
-        claimName: <{ claim_name | default("core-volumes") }>
\ No newline at end of file

templates/helm_service_deployment.yaml → templates/helm_deployment.yml

 name: <{ module_name }>
 codeType: backend
 type: core
+{% if service_type %}
 ports:
   name: port<{ module_port }>
   port: <{ module_port }>
   targetPort: <{ module_port }>
   nodePort: <{ node_port }>
-
+{% endif %}
 
 affinity:
   enabled: True
@@ -20,7 +21,7 @@ podAutoScaler:
   scalePercentage: 85
 
 deployment:
-  imageName: <{ image_tag }>
+  imageName: <{ image_url }>
   command: <{ command }>
   PullSecrets: <{ PullSecrets | default("ilens-azregistry") }>
   PullPolicy: IfNotPresent <{ branch }>
@@ -55,7 +56,7 @@ deployment:
 
 
 mountVolume:
-  enabled: <{ enableMountVolume }>
+  enabled: <{ enableMountVolume | default(True) }>
   volumeMounts:
     - name: <{ mount_name | default("core-volumes") }>
       mountPath: "/code/data"
@@ -64,7 +65,8 @@ mountVolume:
       persistentVolumeClaim:
         claimName: <{ claim_name | default("core-volumes") }>
 
-
+{% if condition %}
 service:
   name: <{ module_name }>
   type: NodePort
+{% endif %}
\ No newline at end of file