Skip to content

D
docker_health_check
Commit 312a8137 authored by jithu.tagore's avatar jithu.tagore
Browse files
Options

Update infra_as_service_monitor.py

parent 5633ee81
Hide whitespace changes
Inline Side-by-side
Showing with 80 additions and 6 deletions
infra_as_service_monitor.py
View file @ 312a8137
......@@ -20,6 +20,9 @@ import dateutil.parser
import dateutil.parser
import paho.mqtt.client as paho
import docker
import pandas as pd
df=pd.read_csv("check_time.csv",index_col=0,parse_dates=["Time"])
# broker="broker.hivemq.com"
......@@ -40,6 +43,7 @@ restart_status_mail = config.getboolean("service", "include_restart_status")
restart_max_count = config.getint("service", "restart_max_count")
meta_file_path = config.get("kafka", "meta_file_path")
meta_file_name = config.get("kafka", "meta_file_name")
resent_time = config.get("mail_configure", "resent_time")
telegram = config.get("telegram", "enable_alerts")
mqtt = config.get("mqtt", "mqtt_enable")
mqtt_broker = config.get("mqtt", "mqtt_broker")
......@@ -73,6 +77,7 @@ Disk_usage_table.field_names = ["Filesystem", "Size", "Used", "Available", "Used
Ram_usage_table = PrettyTable()
Ram_usage_table.field_names = ["Total [gb]", "Used [gb]", "Free [gb]", "Shared [GB]", "Buff/Cache [GB]"]
logging.basicConfig(filename="infra_status.log",
format='%(asctime)s %(message)s',
filemode='a',
......@@ -193,6 +198,23 @@ def docker_container():
end_date = datetime.datetime.now(tz=datetime.timezone.utc)
calculated_time = (end_date.astimezone(datetime.timezone.utc) - start_date.astimezone(
datetime.timezone.utc)).total_seconds()
container_id = str(container.id)[0:8]
if container_id not in df["Services"].values:
df.loc[len(df.index)] = [container_id, datetime.datetime.now(), 0]
df.to_csv("check_time.csv")
else:
found = list(df[df['Services'] == container_id]["Time"])
timer = datetime.datetime.now() - found[0]
timer = timer.total_seconds()
if timer > resent_time:
df.loc[df['Services'] == container_id, "Ago"] = 0
df.loc[df['Services'] == container_id, "Time"] = datetime.datetime.now()
df.to_csv("check_time.csv")
else:
df.loc[df['Services'] == container_id, "Ago"] = str(timer)
df.to_csv("check_time.csv")
if container.attrs["State"]["Status"] == "running":
start = container.attrs["State"]["StartedAt"]
start_date = dateutil.parser.isoparse(start)
......@@ -238,12 +260,33 @@ def docker_container():
containers = docker_client.containers.list(all=True)
for each in containers:
if each.attrs["State"]["Status"] == "restarting" or each.attrs["State"]["Status"] == "exited":
container_restart.append(each)
end = each.attrs["State"]["FinishedAt"]
start_date = dateutil.parser.isoparse(end)
end_date = datetime.datetime.now(tz=datetime.timezone.utc)
calculated_time = (end_date.astimezone(datetime.timezone.utc) - start_date.astimezone(
datetime.timezone.utc)).total_seconds()
container_id=str(each.id)[0:8]
if container_id not in df["Services"].values:
df.loc[len(df.index)] = [container_id,datetime.datetime.now(),0]
df.to_csv("check_time.csv")
else:
found = list(df[df['Services']==container_id]["Time"])
timer=datetime.datetime.now()-found[0]
timer=timer.total_seconds()
if timer > resent_time:
df.loc[df['Services'] == container_id, "Ago"]=0
container_restart.append(each)
df.loc[df['Services'] == container_id, "Time"]=datetime.datetime.now()
df.to_csv("check_time.csv")
else:
df.loc[df['Services'] == container_id, "Ago"] = str(timer)
df.to_csv("check_time.csv")
if each.attrs["State"]["Status"] == "running":
start = each.attrs["State"]["StartedAt"]
start_date = dateutil.parser.isoparse(start)
......@@ -601,8 +644,24 @@ if __name__ == '__main__':
with open('Ram_uasge.txt', 'r') as f:
lines = [line.rstrip() for line in f]
for li in lines:
send_mail = True
ram_table.append(li.split())
container_id="ram_usage"
if container_id not in df["Services"].values:
df.loc[len(df.index)] = [container_id, datetime.datetime.now(), 0]
Ram_send_mail=1
df.to_csv("check_time.csv")
else:
found = list(df[df['Services'] == container_id]["Time"])
timer = datetime.datetime.now() - found[0]
timer = timer.total_seconds()
if timer > resent_time:
df.loc[df['Services'] == container_id, "Ago"] = 0
df.loc[df['Services'] == container_id, "Time"] = datetime.datetime.now()
send_mail = True
df.to_csv("check_time.csv")
ram_table.append(li.split())
else:
df.loc[df['Services'] == container_id, "Ago"] = str(timer)
df.to_csv("check_time.csv")
if len(ram_table) != 0:
any_service_inactive = True
Ram_usage_table.add_rows(ram_table)
......@@ -614,8 +673,23 @@ if __name__ == '__main__':
with open('usage.txt', 'r') as f:
lines = [line.rstrip() for line in f]
for li in lines:
send_mail = True
disk_table.append(li.split())
container_id="disc_usage"
if container_id not in df["Services"].values:
df.loc[len(df.index)] = [container_id, datetime.datetime.now(), 0]
df.to_csv("check_time.csv")
else:
found = list(df[df['Services'] == container_id]["Time"])
timer = datetime.datetime.now() - found[0]
timer = timer.total_seconds()
if timer > resent_time:
df.loc[df['Services'] == container_id, "Ago"] = 0
df.loc[df['Services'] == container_id, "Time"] = datetime.datetime.now()
df.to_csv("check_time.csv")
send_mail = True
disk_table.append(li.split())
else:
df.loc[df['Services'] == container_id, "Ago"] = str(timer)
df.to_csv("check_time.csv")
if len(disk_table) != 0:
any_service_inactive = True
Disk_usage_table.add_rows(disk_table)
......@@ -637,4 +711,4 @@ if __name__ == '__main__':
send_mail = True
docker_table.add_row([docker_name, docker_id, docker_status, time])
if send_mail:
mail_send()
mail_send()
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment