Skip to content

H
Helm-Automation-Script
Commit aa7000d1 authored by harshavardhan.c's avatar harshavardhan.c
Browse files
Options

enh: Added support for registering new helm reports.

parent eb6d7d8d
Hide whitespace changes
Inline Side-by-side
Showing with 293 additions and 116 deletions
helm_automate_script.py
View file @ aa7000d1
......@@ -8,8 +8,6 @@ import os
import sys
import time
import gitlab
from scripts.core import ILensVersionHandler
from scripts.core.git_handler import GitHandler
from scripts.core.helm_handler import HelmHandler
......@@ -29,43 +27,6 @@ HELM_STORE_PATH = "./helm-charts"
git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
def push_helm_deployments(repo_link: str, private_token: str, branch: str, final_helm_path, base_path: str):
try:
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url)
if not repo_link_split:
return False
gl = gitlab.Gitlab(url=base_url, private_token=private_token)
search_str = repo_link_split[-1].replace(".git", "")
pl = gl.projects.list(search=search_str)
if not pl:
return False
pl = pl[0]
commit_actions = []
files_list = os.listdir(final_helm_path)
branches = pl.branches.list(get_all=True)
branches_names = [x.name for x in branches]
if branch not in branches_names:
pl.branches.create({'branch': branch,
'ref': 'master' if branch.split("_")[0] not in branches_names else branch.split("_")[
0]})
if not files_list:
logging.debug('Files not found for pushing to git.')
for file in files_list:
_action = {
'action': 'update',
'file_path': f"{HELM_PATH}/{file}",
'content': open(f'{final_helm_path}/{file}').read()
}
commit_actions.append(_action)
commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
pl.commits.create(commit_data)
except Exception as e:
logging.exception(f'Exception while pushing helm deployments: {e.args}')
ap = argparse.ArgumentParser()
db_handler = ILensVersionHandler()
common_util = CommonUtils()
......@@ -139,7 +100,7 @@ if __name__ == '__main__':
helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts")
if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
clone_branch=_client_name):
logging.error(f"Cannot clone helm repo with branch: {_branch}")
logging.error(f"Cannot clone helm repo with branch: {_client_name}")
sys.exit()
base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
if len(_module_names) == 1 and _module_names[0].lower() == "all":
......@@ -183,13 +144,14 @@ if __name__ == '__main__':
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
release_version=_release_version), db=session_obj)
session_obj.close()
image_url = module_info.get("image_url")
image_url = module_info.get("image_url", '') if module_info else ''
existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url,
module_env_variables=module_env_variables,
template_file=template_file, template_path=template_path,
helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data, module_name=_module)
helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url,
module_env_variables=module_env_variables,
template_file=template_file, template_path=template_path,
helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data,
module_name=_module)
files_info = os.listdir(base_helm_directory_path)
sorted_files = list(filter(lambda f: f.startswith(_module), files_info))
updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x]
......@@ -197,14 +159,15 @@ if __name__ == '__main__':
sub_file_path = os.path.join(base_helm_directory_path, _file)
helm_out_file_path = os.path.join(OUTPUT_PATH, _file)
template_file = os.path.join(template_path, _file)
helm_handler.create_helm_deployment_file(template_yml_path=sub_file_path, image_tag=image_url,
module_env_variables=module_env_variables,
template_file=template_file, template_path=template_path,
helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data,
module_name=_file.split(".yml")[0])
push_helm_deployments(helm_repo, git_access_token, _client_name, final_helm_path=OUTPUT_PATH,
base_path=helm_path)
helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path, image_tag=image_url,
module_env_variables=module_env_variables,
template_file=template_file,
template_path=template_path,
helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data,
module_name=_file.split(".yml")[0])
git_handler_obj.push_helm_deployments(helm_repo, git_access_token, _branch, final_helm_path=OUTPUT_PATH,
base_path=helm_path)
except Exception as e:
logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
......
helm_register.py 0 → 100644
View file @ aa7000d1
from copy import deepcopy
if __name__ == "__main__":
from dotenv import load_dotenv
load_dotenv()
import argparse
import logging
import os
import sys
import time
from scripts.core import ILensVersionHandler
from scripts.core.git_handler import GitHandler
from scripts.core.helm_handler import HelmHandler
from scripts.utils.common_utils import CommonUtils
default_link = "https://gitlab-pm.knowledgelens.com/"
git_user_name = os.environ.get("GIT_USERNAME", default="harshavardhan.c")
git_access_token = os.environ.get("GIT_TOKEN", default="FEMA6PnP63fJCs6DrtZJ")
config_variables = os.environ.get("CONFIG_MAP_VARIABLES", default="").split(",")
helm_repo = os.environ.get("HELM_REPO", default="")
global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="ilens-env-spec-variables.yml")
HELM_PATH = "/ilens-core/ilens-modules"
HELM_STORE_PATH = "./helm-charts"
git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
ap = argparse.ArgumentParser()
db_handler = ILensVersionHandler()
common_util = CommonUtils()
helm_handler = HelmHandler()
if __name__ == '__main__':
ap.add_argument(
"--ilens_version",
"-iv",
required=False,
default=None,
help="ILens Version Tag",
)
ap.add_argument(
"--release_version",
"-rv",
required=False,
default=None,
help="ILens Release Tag",
)
ap.add_argument(
"--reference_client_name",
"-rcn",
required=False,
default=None,
help="Client Name Tag"
)
ap.add_argument(
"--client_name",
"-cn",
required=False,
default=None,
help="Client Name Tag"
)
ap.add_argument(
"--branch_name",
"-b",
required=False,
default=None,
help="Branch Name"
)
ap.add_argument(
"--git_repos",
"-gr",
required=False,
default=None,
help="Git repos to be added in helm",
nargs="+"
)
ap.add_argument(
"--module_names",
"-mn",
required=False,
default=None,
help="Module names to be added in helm",
nargs="+"
)
ap.add_argument(
"--repo_info",
"-ri",
required=False,
default=None,
help="Module names to be added in helm",
nargs="+"
)
HELM_TEMP_PATH = f"{int(time.time())}_helm_tmp_path"
GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
OUTPUT_PATH = f"{int(time.time())}_helm-charts"
try:
arguments = vars(ap.parse_args())
_ilens_version = arguments["ilens_version"]
_release_version = arguments["release_version"]
_client_name = arguments['client_name']
_reference_branch = arguments['reference_client_name']
_git_repos = arguments["git_repos"]
repo_info = arguments["repo_info"]
_module_names = arguments["module_names"]
_branch_name = arguments['branch_name'] or "master"
if not _ilens_version or not _release_version or not _client_name or not (
_git_repos or _module_names) or not global_configmap:
print(
"global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
sys.exit()
_branch = f"{_client_name}_{_ilens_version}.{_release_version}"
if not os.path.exists(HELM_TEMP_PATH):
os.makedirs(HELM_TEMP_PATH)
if not os.path.exists(OUTPUT_PATH):
os.makedirs(OUTPUT_PATH)
helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts")
# if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
# clone_branch=_reference_branch or _client_name):
# logging.error(f"Cannot clone helm repo with branch: {_reference_branch or _client_name}")
base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
if os.path.exists(base_helm_directory_path) and len(_module_names) == 1 and _module_names[0].lower() == "all":
updated_list = []
files_info = os.listdir(base_helm_directory_path)
sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
_module_names = [_each.replace(".yml", "") for _each in sorted_files]
global_config_data = common_util.convert_yaml_to_json(os.path.join(base_helm_directory_path, global_configmap))
# global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
variables_file = "variables.yml"
template_path = os.path.join(GENERAL_TEMP_PATH, "templates")
if not os.path.exists(template_path):
os.makedirs(template_path)
for _module in _module_names:
module_path = os.path.join(GENERAL_TEMP_PATH)
module_path = os.path.join(module_path, _module)
if not os.path.exists(module_path):
os.makedirs(module_path)
helm_out_file_path = os.path.join(OUTPUT_PATH, f'{_module}.yml')
if os.path.exists(helm_out_file_path):
logging.debug(f"Helm Deployment File found for selected the module {_module}")
continue
variables_file_path = os.path.join(module_path, variables_file)
git_info = git_handler_obj.get_git_url_by_module_name(module_name=_module)
if not git_info:
logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
continue
if not git_handler_obj.clone_repository_with_defined_file(repo_link=git_info, clone_branch=_branch_name,
file_output_path=variables_file_path,
clone_file_path=variables_file):
logging.debug("Failed to clone module!! Skipping Helm File Preparation")
continue
_module_data = common_util.convert_yaml_to_json(variables_file_path)
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
module_env_variables = {_v['name']: _v['value'] for _v in module_env_variables if
{'name', 'value'}.issubset(set(list(_v.keys())))}
template_file = os.path.join(template_path, f'{_module}.yml')
# session_obj = get_db_for_func()
# module_info = db_handler.get_module_versions(
# input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
# release_version=_release_version), db=session_obj)
# session_obj.close()
module_info = {}
image_url = module_info.get("image_url", '') if module_info else ''
existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
new_module = deepcopy(_module)
if not os.path.exists(existing_yml_path):
existing_yml_path = os.path.join("templates", "helm_deployment.yml")
new_module = "helm_deployment"
worker_type = True if "worker" in _module | "celery" in _module else False
helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url,
module_env_variables=module_env_variables,
helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data, module_name=new_module)
files_info = os.listdir(base_helm_directory_path)
sorted_files = list(filter(lambda f: f.startswith(_module), files_info))
updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x]
for _file in updated_files_info:
sub_file_path = os.path.join(base_helm_directory_path, _file)
helm_out_file_path = os.path.join(OUTPUT_PATH, _file)
template_file = os.path.join(template_path, _file)
worker_type = True if "worker" in _module | "celery" in _module else False
helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path, image_tag=image_url,
module_env_variables=module_env_variables,
template_file=template_file,
template_path=template_path,
helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data,
service_type=False,
module_name=_file.split(".yml")[0])
git_handler_obj.push_helm_deployments(helm_repo, git_access_token, _client_name, final_helm_path=OUTPUT_PATH,
base_path=helm_path)
except Exception as e:
logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
finally:
# shutil.rmtree(HELM_STORE_PATH)
# shutil.rmtree(HELM_TEMP_PATH)
# shutil.rmtree(GENERAL_TEMP_PATH)
# shutil.rmtree(OUTPUT_PATH)
...
scripts/core/git_handler.py
View file @ aa7000d1
......@@ -89,3 +89,40 @@ class GitHandler:
return False
items = pl.repository_tree(path=HELM_PATH, ref=branch)
print(items)
@staticmethod
def push_helm_deployments(repo_link: str, private_token: str, branch: str, final_helm_path, base_path: str):
try:
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url)
if not repo_link_split:
return False
gl = gitlab.Gitlab(url=base_url, private_token=private_token)
search_str = repo_link_split[-1].replace(".git", "")
pl = gl.projects.list(search=search_str)
if not pl:
return False
pl = pl[0]
commit_actions = []
files_list = os.listdir(final_helm_path)
branches = pl.branches.list(get_all=True)
branches_names = [x.name for x in branches]
if branch not in branches_names:
pl.branches.create({'branch': branch,
'ref': 'master' if branch.split("_")[0] not in branches_names else
branch.split("_")[
0]})
if not files_list:
logging.debug('Files not found for pushing to git.')
for file in files_list:
_action = {
'action': 'update',
'file_path': f"{HELM_PATH}/{file}",
'content': open(f'{final_helm_path}/{file}').read()
}
commit_actions.append(_action)
commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
pl.commits.create(commit_data)
except Exception as e:
logging.exception(f'Exception while pushing helm deployments: {e.args}')
scripts/core/helm_handler.py
View file @ aa7000d1
......@@ -11,9 +11,11 @@ class HelmHandler:
self.common_utils = CommonUtils()
...
def create_helm_deployment_file(self, template_yml_path, module_env_variables: dict, global_config_data: dict,
image_tag: str, module_name: str, template_file: str, helm_out_file_path: str,
template_path: str):
def create_existing_helm_deployment_file(self, template_yml_path, module_env_variables: dict,
global_config_data: dict,
image_tag: str, module_name: str, template_file: str,
helm_out_file_path: str,
template_path: str, service_type=True):
try:
if not os.path.exists(template_yml_path):
logging.debug(f"{template_yml_path} not found!! Skipping Helm File Preparation")
......@@ -34,22 +36,43 @@ class HelmHandler:
self.common_utils.convert_json_to_yaml(json_data=existing_data, output_file_path=template_file)
self.render_helm_chart(data_dict=global_config_vars, helm_template_file=f'{module_name}.yml',
outfile_path=helm_out_file_path, template_path=template_path)
outfile_path=helm_out_file_path, template_path=template_path,
service_type=service_type)
return True
except Exception as e:
logging.exception(f'Exception occurred while preparing the helm deployment file {e.args}')
return False
@staticmethod
def render_helm_chart(data_dict, helm_template_file, template_path, outfile_path):
def render_helm_chart(data_dict, helm_template_file, template_path, outfile_path, **kwargs):
try:
environment = jinja2.Environment(
loader=jinja2.FileSystemLoader(searchpath=template_path),
trim_blocks=True,
variable_start_string='<{', variable_end_string='}>', autoescape=True)
_render = environment.get_template(helm_template_file).render(**data_dict)
_render = environment.get_template(helm_template_file).render(**kwargs, **data_dict)
with open(outfile_path, "w") as fp:
fp.write(_render)
except Exception as e:
logging.exception(f"Exception occurred while rendering the helm file - {e.args}")
def create_helm_deployment_file(self, template_yml_path, module_env_variables: dict, global_config_data: dict,
image_tag: str, module_name: str, helm_out_file_path: str, service_type=True):
try:
if not os.path.exists(template_yml_path):
logging.debug(f"{template_yml_path} not found!! Skipping Helm File Preparation")
return False
global_config_vars = global_config_data.get('data', {})
for k, v in module_env_variables.items():
if global_config_vars.get(k):
continue
global_config_vars.update({k: v})
self.render_helm_chart(data_dict=global_config_vars, helm_template_file=f'{module_name}.yml',
outfile_path=helm_out_file_path, template_path="templates",
service_type=service_type, image_url=image_tag)
return True
except Exception as e:
logging.exception(f'Exception occurred while preparing the helm deployment file {e.args}')
return False
templates/helm_deployment.yaml deleted 100644 → 0
View file @ eb6d7d8d
name: <{ module_name }>
codeType: backend
type: core
affinity:
enabled: True
values: [ "klblrserv24","klblrserv23" ]
podAutoScaler:
enabled: False
type: "hpa"
maxReplicaCount: 3
minReplicaCount: 1
scalePercentage: 85
deployment:
imageName: <{ image_tag }>
command: <{ command }>
PullSecrets: <{ PullSecrets | default("ilens-azregistry") }>
PullPolicy: IfNotPresent <{ branch }>
resources:
requests:
memory: <{ request_memory | default("250Mi") }>
cpu: <{ request_cpu | default("250Mi") }>
limits:
memory: <{ limit_memory | default("750Mi") }>
cpu: <{ limit_cpu | default("500Mi") }>
environmentVar:
- name: MONGO_URI
valueFrom:
secretKeyRef:
name: mongo-uri
key: MONGO_URI
{% for k,v in variables.items() %}
{% if k and v %}
- name: '<{ k }>',
value: '<{ v }>'
{% endif %}
{% endfor %}
mountVolume:
enabled: <{ enableMountVolume }>
volumeMounts:
- name: <{ mount_name | default("core-volumes") }>
mountPath: "/code/data"
volumes:
- name: <{ mount_name | default("core-volumes") }>
persistentVolumeClaim:
claimName: <{ claim_name | default("core-volumes") }>
\ No newline at end of file
templates/helm_service_deployment.yaml → templates/helm_deployment.yml
View file @ aa7000d1
name: <{ module_name }>
codeType: backend
type: core
{% if service_type %}
ports:
name: port<{ module_port }>
port: <{ module_port }>
targetPort: <{ module_port }>
nodePort: <{ node_port }>
{% endif %}
affinity:
enabled: True
......@@ -20,7 +21,7 @@ podAutoScaler:
scalePercentage: 85
deployment:
imageName: <{ image_tag }>
imageName: <{ image_url }>
command: <{ command }>
PullSecrets: <{ PullSecrets | default("ilens-azregistry") }>
PullPolicy: IfNotPresent <{ branch }>
......@@ -55,7 +56,7 @@ deployment:
mountVolume:
enabled: <{ enableMountVolume }>
enabled: <{ enableMountVolume | default(True) }>
volumeMounts:
- name: <{ mount_name | default("core-volumes") }>
mountPath: "/code/data"
......@@ -64,7 +65,8 @@ mountVolume:
persistentVolumeClaim:
claimName: <{ claim_name | default("core-volumes") }>
{% if condition %}
service:
name: <{ module_name }>
type: NodePort
\ No newline at end of file
type: NodePort
{% endif %}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment