Skip to content

H
Helm-Automation-Script
Commit 23d70cd4 authored by harshavardhan.c's avatar harshavardhan.c
Browse files
Options

enh: updated code for prepartion of helm-chart preparation.

parent 2c37df7c
Hide whitespace changes
Inline Side-by-side
Showing with 220 additions and 143 deletions
.env
View file @ 23d70cd4
DB_URI=sqlite:///./ilens_versions.db DB_URI=sqlite:///./ilens_versions.db
CONFIG_MAP_VARIABLES=MONGO_URI CONFIG_MAP_VARIABLES=MONGO_URI
HELM_REPO = https://gitlab-pm.knowledgelens.com/faizan.azim/utest HELM_REPO = https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/devops/helm-charts
GIT_USERNAME = faizan.azim GIT_USERNAME = harshavardhan.c
GIT_TOKEN = MFz2FuGSasbQtBkpdfr6 GIT_TOKEN = FEMA6PnP63fJCs6DrtZJ
GLOBAL_VARIABLES_FILE=ilens-global-configmap.yml GLOBAL_VARIABLES_FILE=ilens-global-configmap.yml
helm_automate_script.py
View file @ 23d70cd4
import shutil
import time
if __name__ == "__main__": if __name__ == "__main__":
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv() load_dotenv()
import argparse import argparse
import json
import logging import logging
import os import os
import shutil
import sys import sys
import git
import gitlab import gitlab
import jinja2 import jinja2
import ruamel.yaml import ruamel.yaml
from scripts.core import ILensVersionHandler from scripts.core import ILensVersionHandler
from scripts.core.git_handler import GitHandler
from scripts.db.psql.databases import get_db_for_func from scripts.db.psql.databases import get_db_for_func
from scripts.schemas import GetRequest from scripts.schemas import GetRequest
...@@ -23,62 +24,26 @@ git_user_name = os.environ.get("GIT_USERNAME", default="harshavardhan.c") ...@@ -23,62 +24,26 @@ git_user_name = os.environ.get("GIT_USERNAME", default="harshavardhan.c")
git_access_token = os.environ.get("GIT_TOKEN", default="FEMA6PnP63fJCs6DrtZJ") git_access_token = os.environ.get("GIT_TOKEN", default="FEMA6PnP63fJCs6DrtZJ")
config_variables = os.environ.get("CONFIG_MAP_VARIABLES", default="").split(",") config_variables = os.environ.get("CONFIG_MAP_VARIABLES", default="").split(",")
helm_repo = os.environ.get("HELM_REPO", default="") helm_repo = os.environ.get("HELM_REPO", default="")
global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="") global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="ilens-global-configmap.yml")
HELM_PATH = "/ilens-core/ilens-modules" HELM_PATH = "/ilens-core/ilens-modules"
HELM_STORE_PATH = "./helm-charts" HELM_STORE_PATH = "./helm-charts"
git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
def render_helm_chart(helm_name, data_dict, variables_list, helm_template_file):
try:
output_path = "helm-charts"
if not os.path.exists(output_path):
os.makedirs(output_path)
helm_path = os.path.join(output_path, f'{helm_name}.yaml')
def render_helm_chart(data_dict, helm_template_file, template_path, outfile_path):
try:
environment = jinja2.Environment( environment = jinja2.Environment(
loader=jinja2.FileSystemLoader(searchpath='./templates'), loader=jinja2.FileSystemLoader(searchpath=template_path),
trim_blocks=True, trim_blocks=True,
variable_start_string='<{', variable_end_string='}>') variable_start_string='<{', variable_end_string='}>')
_render = environment.get_template(helm_template_file).render(**data_dict, _render = environment.get_template(helm_template_file).render(**data_dict)
variables=variables_list) with open(outfile_path, "w") as fp:
with open(helm_path, "w") as fp:
fp.write(_render) fp.write(_render)
except Exception as e: except Exception as e:
logging.exception(f"Exception occurred while rendering the helm file {helm_name} - {e.args}") logging.exception(f"Exception occurred while rendering the helm file - {e.args}")
def clone_repository(repo_link, module_output_path, clone_branch):
try:
if repo_link.split("https://")[-1].startswith("gitlab-pm"):
repo_link = repo_link.replace("https://", f"https://{git_user_name}:{git_access_token}@")
repo = git.Repo.clone_from(repo_link, module_output_path,
branch=clone_branch)
return True
except Exception as e:
logging.exception(f"Exception occurred while cloning the git repo - {repo_link} - {e.args}")
return False
def clone_repository_with_defined_file(repo_link: str, file_output_path, clone_branch, private_token, clone_file_path):
try:
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url)
if not repo_link_split:
return False
gl = gitlab.Gitlab(url=base_url, private_token=private_token)
search_str = repo_link_split[-1].replace(".git", "")
pl = gl.projects.list(search=search_str)
if not pl:
return False
pl = pl[0]
with open(file_output_path, 'wb') as f:
pl.files.raw(file_path=clone_file_path, ref=clone_branch, streamed=True, action=f.write)
return True
except Exception as e:
logging.exception(f"Exception occurred while cloning the git repo - {repo_link} - {e.args}")
return False
def convert_yaml_to_json(yaml_file_path): def convert_yaml_to_json(yaml_file_path):
...@@ -86,14 +51,27 @@ def convert_yaml_to_json(yaml_file_path): ...@@ -86,14 +51,27 @@ def convert_yaml_to_json(yaml_file_path):
if not os.path.exists(yaml_file_path): if not os.path.exists(yaml_file_path):
return {} return {}
_yaml = ruamel.yaml.YAML(typ='safe') _yaml = ruamel.yaml.YAML(typ='safe')
_yaml.preserve_quotes = True
with open(yaml_file_path) as fpi: with open(yaml_file_path) as fpi:
yaml_dict = _yaml.load(fpi) yaml_dict = _yaml.load(fpi)
return yaml_dict return yaml_dict
except Exception as e: except Exception as e:
logging.exception(f"Exception Occurred while reading the yaml file {e.args}") logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
return {} return {}
def push_helm_deployments(repo_link:str, private_token:str, branch: str):
def convert_json_to_yaml(json_data: dict, output_file_path):
try:
yaml = ruamel.yaml.YAML()
yaml.preserve_quotes = True
with open(output_file_path, 'w') as yaml_file:
yaml.dump(json_data, yaml_file)
except Exception as e:
logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
return False
def push_helm_deployments(repo_link: str, private_token: str, branch: str, final_helm_path, base_path: str):
try: try:
base_url = os.environ.get("GIT_BASE_URL", default=default_link) base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url) repo_link_split = repo_link.split(base_url)
...@@ -101,17 +79,19 @@ def push_helm_deployments(repo_link:str, private_token:str, branch: str): ...@@ -101,17 +79,19 @@ def push_helm_deployments(repo_link:str, private_token:str, branch: str):
return False return False
gl = gitlab.Gitlab(url=base_url, private_token=private_token) gl = gitlab.Gitlab(url=base_url, private_token=private_token)
search_str = repo_link_split[-1].replace(".git", "") search_str = repo_link_split[-1].replace(".git", "")
pl = gl.projects.list(search=search_str) pl = gl.projects.list(search=search_str.split("/")[-1])
if not pl: if not pl:
return False return False
pl = pl[0] pl = pl[0]
commit_actions = [] commit_actions = []
files_list = os.listdir(HELM_STORE_PATH) files_list = os.listdir(final_helm_path)
if not files_list:
logging.debug('Files not found for pushing to git.')
for file in files_list: for file in files_list:
_action = { _action = {
'action': 'create', 'action': 'create',
'file_path': f'{HELM_PATH}/{file}', 'file_path': os.path.join(HELM_PATH, file),
'content': open(f'{HELM_STORE_PATH}/{file}').read() 'content': open(f'{final_helm_path}/{file}').read()
} }
commit_actions.append(_action) commit_actions.append(_action)
commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions} commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
...@@ -119,35 +99,6 @@ def push_helm_deployments(repo_link:str, private_token:str, branch: str): ...@@ -119,35 +99,6 @@ def push_helm_deployments(repo_link:str, private_token:str, branch: str):
pl.commits.create(commit_data) pl.commits.create(commit_data)
except Exception as e: except Exception as e:
logging.exception(f'Exception while pushing helm deployments: {e.args}') logging.exception(f'Exception while pushing helm deployments: {e.args}')
def pull_global_config(repo_link: str, private_token:str, branch: str, file_name: str):
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url)
if not repo_link_split:
return False
gl = gitlab.Gitlab(url=base_url, private_token=private_token)
search_str = repo_link_split[-1].replace(".git", "")
if pl := gl.projects.list(search=search_str):
pl = pl[0]
else:
return False
with open(file_name, 'wb') as f:
pl.files.raw(file_path=f'{HELM_PATH[1:]}/{file_name}', ref=branch, streamed=True, action=f.write)
return True
def remove_all_files_from_repo(repo_link: str, private_token: str, branch: str, exclude_file: list):
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url)
if not repo_link_split:
return False
gl = gitlab.Gitlab(url=base_url, private_token=private_token)
search_str = repo_link_split[-1].replace(".git", "")
if pl := gl.projects.list(search=search_str):
pl = pl[0]
else:
return False
items = pl.repository_tree(path=HELM_PATH, ref=branch)
print(items)
ap = argparse.ArgumentParser() ap = argparse.ArgumentParser()
...@@ -174,6 +125,13 @@ if __name__ == '__main__': ...@@ -174,6 +125,13 @@ if __name__ == '__main__':
default=None, default=None,
help="Client Name Tag" help="Client Name Tag"
) )
ap.add_argument(
"--branch_name",
"-b",
required=False,
default=None,
help="Branch Name"
)
ap.add_argument( ap.add_argument(
"--git_repos", "--git_repos",
"-gr", "-gr",
...@@ -190,61 +148,93 @@ if __name__ == '__main__': ...@@ -190,61 +148,93 @@ if __name__ == '__main__':
help="Module names to be added in helm", help="Module names to be added in helm",
nargs="+" nargs="+"
) )
arguments = vars(ap.parse_args()) HELM_TEMP_PATH = f"{int(time.time())}_helm_tmp_path"
_ilens_version = arguments["ilens_version"] GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
_release_version = arguments["release_version"] OUTPUT_PATH = f"{int(time.time())}_helm-charts"
_client_name = arguments['client_name'] try:
_git_repos = arguments["git_repos"] arguments = vars(ap.parse_args())
_module_names = arguments["module_names"] _ilens_version = arguments["ilens_version"]
if not _ilens_version or not _release_version or not _client_name or not (_git_repos or _module_names) or not global_configmap: _release_version = arguments["release_version"]
print("global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!") _client_name = arguments['client_name']
sys.exit() _git_repos = arguments["git_repos"]
_branch = f"{_client_name}_{_ilens_version}.{_release_version}" _module_names = arguments["module_names"]
if not pull_global_config(repo_link=helm_repo, private_token=git_access_token, branch=_branch, file_name=global_configmap): _branch_name = arguments['branch_name'] or "master"
logging.error(f"Cannot clone helm repo with branch: {_branch}") if not _ilens_version or not _release_version or not _client_name or not (
sys.exit() _git_repos or _module_names) or not global_configmap:
with open("config.json", "r") as f: print(
data = json.load(f) "global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
if _git_repos: sys.exit()
data['git_modules'] = [x for x in data['git_modules'] if x['git_link'] in _git_repos] _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
else: if not os.path.exists(HELM_TEMP_PATH):
data['git_modules'] = [x for x in data['git_modules'] if x['module_name'] in _module_names] os.makedirs(HELM_TEMP_PATH)
global_config_data = convert_yaml_to_json(global_configmap) if not os.path.exists(OUTPUT_PATH):
remove_all_files_from_repo(repo_link=helm_repo, private_token=git_access_token, branch=_branch, exclude_file=[]) os.makedirs(OUTPUT_PATH)
for _data in data.get('git_modules'): helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts")
_ilens_version = _data.get("ilens_version", _ilens_version) if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
_release_version = _data.get("ilens_version", _release_version) clone_branch=_client_name):
git_link = _data["git_link"] logging.error(f"Cannot clone helm repo with branch: {_branch}")
branch = _data["branch"] sys.exit()
variables_file = _data.get("variables_file") or "variables.yml" base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules")
# global_config_data = convert_yaml_to_json(os.path.join(base_helm_directory_path, global_configmap))
module_path = os.path.join("tmp") global_config_data = convert_yaml_to_json("ilens-global-configmap.yml")
module_name = _data['module_name'] variables_file = "variables.yml"
module_path = os.path.join(module_path, module_name)
if not os.path.exists(module_path): for _module in _module_names:
os.makedirs(module_path) module_path = os.path.join(GENERAL_TEMP_PATH)
variables_file_path = os.path.join(module_path, variables_file) module_path = os.path.join(module_path, _module)
if not clone_repository_with_defined_file(repo_link=git_link, clone_branch=branch, if not os.path.exists(module_path):
file_output_path=variables_file_path, os.makedirs(module_path)
private_token=git_access_token, clone_file_path=variables_file): variables_file_path = os.path.join(module_path, variables_file)
logging.debug("Failed to clone module!! Skipping Helm File Preparation") git_info = git_handler_obj.get_git_url_by_module_name(module_name=_module)
continue if not git_info:
_module_data = convert_yaml_to_json(variables_file_path) logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
global_config_vars = global_config_data['data'] continue
env_variables_from_yml = _module_data.get('deployment', {}).get('environmentVar', []) if not git_handler_obj.clone_repository_with_defined_file(repo_link=git_info, clone_branch=_branch_name,
env_variables_from_yml = {_v['name']:_v['value'] for _v in env_variables_from_yml if file_output_path=variables_file_path,
{'name', 'value'}.issubset(set(list(_v.keys())))} clone_file_path=variables_file):
env_variables_from_yml |= global_config_vars logging.debug("Failed to clone module!! Skipping Helm File Preparation")
template_file = _data.get("template_file") or "helm_service_deployment.yaml" continue
session_obj = get_db_for_func() existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
module_info = db_handler.get_module_versions( if not os.path.exists(existing_yml_path):
input_data=GetRequest(module_name=module_name, client='iLens', ilens_version=_ilens_version, logging.debug(f"{existing_yml_path} not found!! Skipping Helm File Preparation")
release_version=_release_version), db=session_obj) continue
session_obj.close() existing_data = convert_yaml_to_json(existing_yml_path)
_data["image_tag"] = module_info.get("image_tag", "-") _module_data = convert_yaml_to_json(variables_file_path)
render_helm_chart(helm_name=module_name, data_dict=_data, variables_list=env_variables_from_yml, global_config_vars = global_config_data.get('data', {})
helm_template_file=template_file) module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
push_helm_deployments(helm_repo, git_access_token, _branch) module_env_variables = {_v['name']: _v for _v in module_env_variables}
shutil.rmtree("./tmp") existing_env_variables = {_v['name']: _v for _v in
shutil.rmtree(HELM_STORE_PATH) existing_data.get('deployment', {}).get('environmentVar', [])}
\ No newline at end of file existing_data['deployment']['environmentVar'] = []
for k, v in module_env_variables.items():
if k.lower() in {'port', 'service_port', 'module_port'} and v.get("value"):
global_config_vars["SERVICE_PORT"] = v['value']
continue
if "valueFrom" in existing_env_variables.get(k):
existing_data['deployment']['environmentVar'].append(existing_env_variables[k])
else:
existing_data['deployment']['environmentVar'].append(v)
template_path = os.path.join(GENERAL_TEMP_PATH, "templates")
if not os.path.exists(template_path):
os.makedirs(template_path)
template_file = os.path.join(template_path, f'{_module}.yml')
convert_json_to_yaml(json_data=existing_data, output_file_path=template_file)
session_obj = get_db_for_func()
module_info = db_handler.get_module_versions(
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
release_version=_release_version), db=session_obj)
session_obj.close()
existing_data['deployment']['imageName'] = module_info.get("image_tag",
existing_data['deployment']['imageName'])
helm_out_path = os.path.join(OUTPUT_PATH, f'{_module}.yml')
render_helm_chart(data_dict=global_config_vars, helm_template_file=f'{_module}.yml',
outfile_path=helm_out_path, template_path=template_path)
push_helm_deployments(helm_repo, git_access_token, _branch, final_helm_path=OUTPUT_PATH, base_path=helm_path)
except Exception as e:
logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
finally:
# shutil.rmtree(HELM_STORE_PATH)
shutil.rmtree(HELM_TEMP_PATH)
shutil.rmtree(GENERAL_TEMP_PATH)
scripts/core/git_handler.py 0 → 100644
View file @ 23d70cd4
import os
import git
import gitlab
from scripts.logging import logging
default_link = "https://gitlab-pm.knowledgelens.com/"
HELM_PATH = "/ilens-core/ilens-modules"
HELM_STORE_PATH = "./helm-charts"
class GitHandler:
def __init__(self, access_token, user_name):
self.access_token = access_token
self.user_name = user_name
def clone_repository(self, repo_link, module_output_path, clone_branch):
try:
if repo_link.split("https://")[-1].startswith("gitlab-pm"):
repo_link = repo_link.replace("https://", f"https://{self.user_name}:{self.access_token}@")
repo = git.Repo.clone_from(repo_link, module_output_path,
branch=clone_branch)
return True
except Exception as e:
logging.exception(f"Exception occurred while cloning the git repo - {repo_link} - {e.args}")
return False
def clone_repository_with_defined_file(self, repo_link: str, file_output_path, clone_branch, clone_file_path):
try:
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url)
if not repo_link_split:
return False
gl = gitlab.Gitlab(url=base_url, private_token=self.access_token)
search_str = repo_link_split[-1].replace(".git", "")
pl = gl.projects.list(search=search_str)
if not pl:
return False
pl = pl[0]
with open(file_output_path, 'wb') as f:
pl.files.raw(file_path=clone_file_path, ref=clone_branch, streamed=True, action=f.write)
return True
except Exception as e:
logging.exception(f"Exception occurred while cloning the git repo - {repo_link} - {e.args}")
return False
def get_git_url_by_module_name(self, module_name):
try:
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
gl = gitlab.Gitlab(url=base_url, private_token=self.access_token)
pl = gl.projects.list(search=module_name, group="/KnowledgeLens/Products/iLens-2.0")
return pl[0].web_url if pl else ''
except Exception as e:
logging.exception(f"Exception occurred while fetching repo details - {e.args}")
return False
def pull_global_config(self, repo_link: str, branch: str, file_name: str):
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url)
if not repo_link_split:
return False
gl = gitlab.Gitlab(url=base_url, private_token=self.access_token)
search_str = repo_link_split[-1].replace(".git", "")
if pl := gl.projects.list(search=search_str):
pl = pl[0]
else:
return False
with open(file_name, 'wb') as f:
pl.files.raw(file_path=f'{HELM_PATH[1:]}/{file_name}', ref=branch, streamed=True, action=f.write)
return True
def remove_all_files_from_repo(self, repo_link: str, branch: str, exclude_file: list):
base_url = os.environ.get("GIT_BASE_URL", default=default_link)
repo_link_split = repo_link.split(base_url)
if not repo_link_split:
return False
gl = gitlab.Gitlab(url=base_url, private_token=self.access_token)
search_str = repo_link_split[-1].replace(".git", "")
if pl := gl.projects.list(search=search_str):
pl = pl[0]
else:
return False
items = pl.repository_tree(path=HELM_PATH, ref=branch)
print(items)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment