HWJ 何伟俊 / DMapManager · 提交

提交 23a19983fcdc154690de32ac956f40c08bf2d0b3

作者 qianyingz
1 个父辈 8694e401

监控数据汇总

内嵌 并排对比
正在显示 4 个修改的文件 包含 513 行增加 和 250 行删除
  1 +from datetime import datetime, timedelta
  2 +
  3 +from sqlalchemy.sql.functions import func
  4 +from .models import MonitorLog, db
  5 +from sqlalchemy import and_
  6 +from app.util.component.ApiTemplate import ApiTemplate
  7 +
  8 +
  9 +class Api(ApiTemplate):
  10 + api_name = "统计指标"
  11 +
  12 + def process(self):
  13 +
  14 + # 返回结果
  15 + res = {}
  16 + res["data"] = {}
  17 + logs = []
  18 + try:
  19 + server = self.para.get("server") # server
  20 + # metrics_type = self.para.get("metrics_type")
  21 + grain = self.para.get("grain")
  22 + count = int(self.para.get("count"))
  23 + start_time = self.para.get("start")
  24 + to_time = self.para.get("to")
  25 +
  26 + cur_now = datetime.now()
  27 + start_datetime = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S")
  28 + end_datetime = datetime.strptime(to_time, "%Y-%m-%d %H:%M:%S")
  29 +
  30 + if grain == "day":
  31 + # 粒度是一天
  32 + format = "%Y-%m-%d"
  33 + logs = db.session.query(MonitorLog.date_stamp.label("key"), func.avg(MonitorLog.cpu_usage).label("cpu_usage"),
  34 + func.avg(MonitorLog.total_mem).label(
  35 + "total_mem"),
  36 + func.avg(MonitorLog.available_mem).label(
  37 + "available_mem"),
  38 + func.avg(MonitorLog.used_mem).label(
  39 + "used_mem"),
  40 + func.avg(MonitorLog.disk).label(
  41 + "disk"),
  42 + func.avg(MonitorLog.disk_usage).label(
  43 + "disk_usage"),
  44 + func.avg(MonitorLog.net_recv).label(
  45 + "net_recv"),
  46 + func.avg(MonitorLog.net_send).label("net_send")).filter(and_(
  47 + MonitorLog.date_stamp <= end_datetime.strftime(
  48 + format),
  49 + MonitorLog.date_stamp > start_datetime.strftime(
  50 + format),
  51 + MonitorLog.server == server)).group_by(MonitorLog.date_stamp).order_by(MonitorLog.date_stamp.asc()).all()
  52 +
  53 + keys = []
  54 + keys_map = {}
  55 + while start_datetime <= end_datetime:
  56 + keys.append(self.get_monitor_log(
  57 + start_datetime.strftime('%Y-%m-%d')))
  58 + keys_map[start_datetime.strftime('%Y-%m-%d')] = len(keys)-1
  59 + start_datetime += timedelta(days=1)
  60 +
  61 + res["data"] = self.get_result(logs, keys, keys_map)
  62 + elif grain == "minu_five":
  63 + # 粒度是5分钟
  64 + cur_minu = int(end_datetime.strftime("%M")) % 5
  65 + logs = db.session.query(MonitorLog.hour_stamp.label("hour_stamp"),
  66 + MonitorLog.f_minu_stamp.label(
  67 + "minu_stamp"),
  68 + func.avg(MonitorLog.cpu_usage).label(
  69 + "cpu_usage"),
  70 + func.avg(MonitorLog.total_mem).label(
  71 + "total_mem"),
  72 + func.avg(MonitorLog.available_mem).label(
  73 + "available_mem"),
  74 + func.avg(MonitorLog.used_mem).label(
  75 + "used_mem"),
  76 + func.avg(MonitorLog.disk).label(
  77 + "disk"),
  78 + func.avg(MonitorLog.disk_usage).label(
  79 + "disk_usage"),
  80 + func.avg(MonitorLog.net_recv).label(
  81 + "net_recv"),
  82 + func.avg(MonitorLog.net_send).label("net_send")).filter(and_(
  83 + MonitorLog.time_stamp <= "{}:00".format(
  84 + end_datetime.strftime("%Y-%m-%d %H:%M")),
  85 + MonitorLog.time_stamp >= "{}:00".format(
  86 + start_datetime.strftime("%Y-%m-%d %H:%M")),
  87 + MonitorLog.server == server)).group_by(MonitorLog.date_stamp, MonitorLog.hour_stamp, MonitorLog.f_minu_stamp).order_by(
  88 + MonitorLog.hour_stamp.asc(), MonitorLog.f_minu_stamp.asc()).all()
  89 +
  90 + keys = []
  91 + keys_map = {}
  92 + while start_datetime <= end_datetime:
  93 + key = start_datetime.strftime('%H:%M')
  94 + keys.append(self.get_monitor_log(key))
  95 + keys_map[key] = len(keys)-1
  96 + start_datetime += timedelta(minutes=5)
  97 +
  98 + logs = list(map(lambda log:
  99 + {"key": "%02d:%02d" % (log.hour_stamp, log.minu_stamp*5+cur_minu),
  100 + "cpu_usage": log.cpu_usage, "total_mem": log.total_mem,
  101 + "available_mem": log.available_mem, "used_mem": log.used_mem, "disk": log.disk, "disk_usage": log.disk_usage,
  102 + "net_recv": log.net_recv, "net_send": log.net_send},
  103 + logs))
  104 +
  105 + res["data"] = self.get_result_from_list(logs, keys, keys_map)
  106 + elif grain == "minu_ten":
  107 + # 粒度是10分钟
  108 + cur_minu = int(end_datetime.strftime("%M")) % 10
  109 + logs = db.session.query(MonitorLog.hour_stamp.label("hour_stamp"),
  110 + MonitorLog.d_minu_stamp.label(
  111 + "minu_stamp"),
  112 + func.avg(MonitorLog.cpu_usage).label(
  113 + "cpu_usage"),
  114 + func.avg(MonitorLog.total_mem).label(
  115 + "total_mem"),
  116 + func.avg(MonitorLog.available_mem).label(
  117 + "available_mem"),
  118 + func.avg(MonitorLog.used_mem).label(
  119 + "used_mem"),
  120 + func.avg(MonitorLog.disk).label(
  121 + "disk"),
  122 + func.avg(MonitorLog.disk_usage).label(
  123 + "disk_usage"),
  124 + func.avg(MonitorLog.net_recv).label(
  125 + "net_recv"),
  126 + func.avg(MonitorLog.net_send).label("net_send")).filter(and_(
  127 + MonitorLog.time_stamp <= "{}:00".format(
  128 + end_datetime.strftime("%Y-%m-%d %H:%M")),
  129 + MonitorLog.time_stamp >= "{}:00".format(
  130 + start_datetime.strftime("%Y-%m-%d %H:%M")),
  131 + MonitorLog.server == server)).group_by(MonitorLog.date_stamp,
  132 + MonitorLog.hour_stamp,
  133 + MonitorLog.d_minu_stamp).order_by(MonitorLog.hour_stamp.asc(), MonitorLog.d_minu_stamp.asc()).all()
  134 + keys = []
  135 + keys_map = {}
  136 + while start_datetime <= end_datetime:
  137 + key = start_datetime.strftime('%H:%M')
  138 + keys.append(self.get_monitor_log(key))
  139 + keys_map[key] = len(keys)-1
  140 + start_datetime += timedelta(minutes=10)
  141 +
  142 + logs = list(map(lambda log:
  143 + {"key": "%02d:%02d" % (log.hour_stamp, log.minu_stamp*10+cur_minu),
  144 + "cpu_usage": log.cpu_usage, "total_mem": log.total_mem,
  145 + "available_mem": log.available_mem, "used_mem": log.used_mem, "disk": log.disk, "disk_usage": log.disk_usage,
  146 + "net_recv": log.net_recv, "net_send": log.net_send},
  147 + logs))
  148 +
  149 + res["data"] = self.get_result_from_list(logs, keys, keys_map)
  150 + elif grain == "hour":
  151 + # 粒度是一小时
  152 + logs = db.session.query(MonitorLog.hour_stamp.label("key"), func.avg(MonitorLog.cpu_usage).label("cpu_usage"),
  153 + func.avg(MonitorLog.total_mem).label(
  154 + "total_mem"),
  155 + func.avg(MonitorLog.available_mem).label(
  156 + "available_mem"),
  157 + func.avg(MonitorLog.used_mem).label(
  158 + "used_mem"),
  159 + func.avg(MonitorLog.disk).label(
  160 + "disk"),
  161 + func.avg(MonitorLog.disk_usage).label(
  162 + "disk_usage"),
  163 + func.avg(MonitorLog.net_recv).label(
  164 + "net_recv"),
  165 + func.avg(MonitorLog.net_send).label("net_send")).filter(and_(
  166 + MonitorLog.time_stamp <= "{}:59:59".format(
  167 + end_datetime.strftime("%Y-%m-%d %H")),
  168 + MonitorLog.time_stamp >= "{}:59:59".format(
  169 + start_datetime.strftime("%Y-%m-%d %H")),
  170 + MonitorLog.server == server)).group_by(MonitorLog.hour_stamp).order_by(MonitorLog.hour_stamp.asc()).all()
  171 + keys = []
  172 + keys_map = {}
  173 + while start_datetime <= end_datetime:
  174 + key = int(start_datetime.strftime('%H'))
  175 + keys.append(self.get_monitor_log("%02d时" % key))
  176 + keys_map[str(key)] = len(keys)-1
  177 + start_datetime += timedelta(hours=1)
  178 +
  179 + res["data"] = self.get_result(logs, keys, keys_map)
  180 + else:
  181 + # 按分钟统计
  182 + logs = db.session.query(MonitorLog.hour_stamp.label("hour_stamp"), MonitorLog.minu_stamp.label("minu_stamp"), func.avg(MonitorLog.cpu_usage).label("cpu_usage"),
  183 + func.avg(MonitorLog.total_mem).label(
  184 + "total_mem"),
  185 + func.avg(MonitorLog.available_mem).label(
  186 + "available_mem"),
  187 + func.avg(MonitorLog.used_mem).label("used_mem"),
  188 + func.avg(MonitorLog.disk).label(
  189 + "disk"),
  190 + func.avg(MonitorLog.disk_usage).label(
  191 + "disk_usage"),
  192 + func.avg(MonitorLog.net_recv).label(
  193 + "net_recv"),
  194 + func.avg(MonitorLog.net_send).label("net_send")).filter(and_(
  195 + MonitorLog.time_stamp <= end_datetime.strftime("%Y-%m-%d %H:%M:%S"),
  196 + MonitorLog.time_stamp >= start_datetime.strftime("%Y-%m-%d %H:%M:%S"),
  197 + MonitorLog.server == server)).group_by(MonitorLog.hour_stamp, MonitorLog.minu_stamp).order_by(
  198 + MonitorLog.hour_stamp.asc(), MonitorLog.minu_stamp.asc()).all()
  199 + keys = []
  200 + keys_map = {}
  201 + while start_datetime <= end_datetime:
  202 + key = start_datetime.strftime('%H:%M')
  203 + keys.append(self.get_monitor_log(key))
  204 + keys_map[key] = len(keys)-1
  205 + start_datetime += timedelta(minutes=1)
  206 +
  207 + logs = list(map(lambda log:
  208 + {"key": "%02d:%02d" % (log.hour_stamp, log.minu_stamp),
  209 + "cpu_usage": log.cpu_usage, "total_mem": log.total_mem,
  210 + "available_mem": log.available_mem, "used_mem": log.used_mem, "disk": log.disk, "disk_usage": log.disk_usage,
  211 + "net_recv": log.net_recv, "net_send": log.net_send},
  212 + logs))
  213 +
  214 + res["data"] = self.get_result_from_list(logs, keys, keys_map)
  215 + res["result"] = True
  216 + except Exception as e:
  217 + raise e
  218 + return res
  219 +
  220 + api_doc = {
  221 + "tags": ["监控接口"],
  222 + "parameters": [
  223 + {"name": "server",
  224 + "in": "query",
  225 + "type": "string",
  226 + "description": "服务器地址"},
  227 + {"name": "grain",
  228 + "in": "query",
  229 + "type": "string",
  230 + "description": "指标粒度：minu：分钟，minu_five：5分钟，minu_ten：10分钟，hour：1小时，day：每天"},
  231 + {"name": "count",
  232 + "in": "query",
  233 + "type": "string",
  234 + "description": "个数"},
  235 + {"name": "to",
  236 + "in": "query",
  237 + "type": "string",
  238 + "description": "查询终止时间"},
  239 + {"name": "start",
  240 + "in": "query",
  241 + "type": "string",
  242 + "description": "查询起始时间"}
  243 + ],
  244 + "responses": {
  245 + 200: {
  246 + "schema": {
  247 + "properties": {
  248 + }
  249 + }
  250 + }
  251 + }
  252 + }
  253 +
  254 + def get_monitor_log(self, key):
  255 + return {"key": key,
  256 + "cpu_usage": None, "total_mem": None,
  257 + "available_mem": None, "used_mem": None, "disk": None, "disk_usage": None,
  258 + "net_recv": None, "net_send": None}
  259 +
  260 + def get_result(self, logs, keys, keys_map):
  261 + keys_map_key=keys_map.keys()
  262 + for log in logs:
  263 + if str(log.key) in keys_map_key:
  264 + tmp = keys[keys_map[str(log.key)]]
  265 + if tmp != None:
  266 + tmp['cpu_usage'] = log.cpu_usage
  267 + tmp['total_mem'] = log.total_mem
  268 + tmp['available_mem'] = log.available_mem
  269 + tmp['used_mem'] = log.used_mem
  270 + tmp['disk'] = log.disk
  271 + tmp['disk_usage'] = log.disk_usage
  272 + tmp['net_recv'] = log.net_recv
  273 + tmp['net_send'] = log.net_send
  274 + return keys
  275 +
  276 + def get_result_from_list(self, logs, keys, keys_map):
  277 + keys_map_key=keys_map.keys()
  278 + for log in logs:
  279 + if str(log["key"]) in keys_map_key:
  280 + tmp = keys[keys_map[str(log["key"])]]
  281 + if tmp != None:
  282 + tmp['cpu_usage'] = log["cpu_usage"]
  283 + tmp['total_mem'] = log["total_mem"]
  284 + tmp['available_mem'] = log["available_mem"]
  285 + tmp['used_mem'] = log["used_mem"]
  286 + tmp['disk'] = log["disk"]
  287 + tmp['disk_usage'] = log["disk_usage"]
  288 + tmp['net_recv'] = log["net_recv"]
  289 + tmp['net_send'] = log["net_send"]
  290 + return keys
1 from datetime import datetime, timedelta 1 from datetime import datetime, timedelta
2 - 2 +from operator import or_
  3 +from sqlalchemy.sql.expression import intersect_all
  4 +import re
3 from sqlalchemy.sql.functions import func 5 from sqlalchemy.sql.functions import func
4 -from .models import MonitorLog, db 6 +from .models import db, MonitorInfo
5 from sqlalchemy import and_ 7 from sqlalchemy import and_
6 from app.util.component.ApiTemplate import ApiTemplate 8 from app.util.component.ApiTemplate import ApiTemplate
  9 +import math
  10 +from functools import reduce
7 11
8 12
9 class Api(ApiTemplate): 13 class Api(ApiTemplate):
@@ -13,205 +17,62 @@ class Api(ApiTemplate): @@ -13,205 +17,62 @@ class Api(ApiTemplate):
13 17
14 # 返回结果 18 # 返回结果
15 res = {} 19 res = {}
16 - res["data"] = {} 20 + res["data"] = []
17 logs = [] 21 logs = []
18 try: 22 try:
19 server = self.para.get("server") # server 23 server = self.para.get("server") # server
20 - # metrics_type = self.para.get("metrics_type")  
21 - grain = self.para.get("grain")  
22 - count = int(self.para.get("count")) 24 + metrics_type = self.para.get("metrics_type")
  25 + interval = self.para.get("interval")
23 start_time = self.para.get("start") 26 start_time = self.para.get("start")
24 to_time = self.para.get("to") 27 to_time = self.para.get("to")
25 -  
26 - cur_now = datetime.now()  
27 start_datetime = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S") 28 start_datetime = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S")
28 end_datetime = datetime.strptime(to_time, "%Y-%m-%d %H:%M:%S") 29 end_datetime = datetime.strptime(to_time, "%Y-%m-%d %H:%M:%S")
29 -  
30 - if grain == "day":  
31 - # 粒度是一天  
32 - format = "%Y-%m-%d"  
33 - logs = db.session.query(MonitorLog.date_stamp.label("key"), func.avg(MonitorLog.cpu_usage).label("cpu_usage"),  
34 - func.avg(MonitorLog.total_mem).label(  
35 - "total_mem"),  
36 - func.avg(MonitorLog.available_mem).label(  
37 - "available_mem"),  
38 - func.avg(MonitorLog.used_mem).label(  
39 - "used_mem"),  
40 - func.avg(MonitorLog.disk).label(  
41 - "disk"),  
42 - func.avg(MonitorLog.disk_usage).label(  
43 - "disk_usage"),  
44 - func.avg(MonitorLog.net_recv).label(  
45 - "net_recv"),  
46 - func.avg(MonitorLog.net_send).label("net_send")).filter(and_(  
47 - MonitorLog.date_stamp <= end_datetime.strftime(  
48 - format),  
49 - MonitorLog.date_stamp > start_datetime.strftime(  
50 - format),  
51 - MonitorLog.server == server)).group_by(MonitorLog.date_stamp).order_by(MonitorLog.date_stamp.asc()).all()  
52 -  
53 - keys = []  
54 - keys_map = {}  
55 - while start_datetime <= end_datetime:  
56 - keys.append(self.get_monitor_log(  
57 - start_datetime.strftime('%Y-%m-%d')))  
58 - keys_map[start_datetime.strftime('%Y-%m-%d')] = len(keys)-1  
59 - start_datetime += timedelta(days=1)  
60 -  
61 - res["data"] = self.get_result(logs, keys, keys_map)  
62 - elif grain == "minu_five":  
63 - # 粒度是5分钟  
64 - cur_minu = int(end_datetime.strftime("%M")) % 5  
65 - logs = db.session.query(MonitorLog.hour_stamp.label("hour_stamp"),  
66 - MonitorLog.f_minu_stamp.label(  
67 - "minu_stamp"),  
68 - func.avg(MonitorLog.cpu_usage).label(  
69 - "cpu_usage"),  
70 - func.avg(MonitorLog.total_mem).label(  
71 - "total_mem"),  
72 - func.avg(MonitorLog.available_mem).label(  
73 - "available_mem"),  
74 - func.avg(MonitorLog.used_mem).label(  
75 - "used_mem"),  
76 - func.avg(MonitorLog.disk).label(  
77 - "disk"),  
78 - func.avg(MonitorLog.disk_usage).label(  
79 - "disk_usage"),  
80 - func.avg(MonitorLog.net_recv).label(  
81 - "net_recv"),  
82 - func.avg(MonitorLog.net_send).label("net_send")).filter(and_(  
83 - MonitorLog.time_stamp <= "{}:00".format(  
84 - end_datetime.strftime("%Y-%m-%d %H:%M")),  
85 - MonitorLog.time_stamp >= "{}:00".format(  
86 - start_datetime.strftime("%Y-%m-%d %H:%M")),  
87 - MonitorLog.server == server)).group_by(MonitorLog.date_stamp, MonitorLog.hour_stamp, MonitorLog.f_minu_stamp).order_by(  
88 - MonitorLog.hour_stamp.asc(), MonitorLog.f_minu_stamp.asc()).all()  
89 -  
90 - keys = []  
91 - keys_map = {}  
92 - while start_datetime <= end_datetime:  
93 - key = start_datetime.strftime('%H:%M')  
94 - keys.append(self.get_monitor_log(key))  
95 - keys_map[key] = len(keys)-1  
96 - start_datetime += timedelta(minutes=5)  
97 -  
98 - logs = list(map(lambda log:  
99 - {"key": "%02d:%02d" % (log.hour_stamp, log.minu_stamp*5+cur_minu),  
100 - "cpu_usage": log.cpu_usage, "total_mem": log.total_mem,  
101 - "available_mem": log.available_mem, "used_mem": log.used_mem, "disk": log.disk, "disk_usage": log.disk_usage,  
102 - "net_recv": log.net_recv, "net_send": log.net_send},  
103 - logs))  
104 -  
105 - res["data"] = self.get_result_from_list(logs, keys, keys_map)  
106 - elif grain == "minu_ten":  
107 - # 粒度是10分钟  
108 - cur_minu = int(end_datetime.strftime("%M")) % 10  
109 - logs = db.session.query(MonitorLog.hour_stamp.label("hour_stamp"),  
110 - MonitorLog.d_minu_stamp.label(  
111 - "minu_stamp"),  
112 - func.avg(MonitorLog.cpu_usage).label(  
113 - "cpu_usage"),  
114 - func.avg(MonitorLog.total_mem).label(  
115 - "total_mem"),  
116 - func.avg(MonitorLog.available_mem).label(  
117 - "available_mem"),  
118 - func.avg(MonitorLog.used_mem).label(  
119 - "used_mem"),  
120 - func.avg(MonitorLog.disk).label(  
121 - "disk"),  
122 - func.avg(MonitorLog.disk_usage).label(  
123 - "disk_usage"),  
124 - func.avg(MonitorLog.net_recv).label(  
125 - "net_recv"),  
126 - func.avg(MonitorLog.net_send).label("net_send")).filter(and_(  
127 - MonitorLog.time_stamp <= "{}:00".format(  
128 - end_datetime.strftime("%Y-%m-%d %H:%M")),  
129 - MonitorLog.time_stamp >= "{}:00".format(  
130 - start_datetime.strftime("%Y-%m-%d %H:%M")),  
131 - MonitorLog.server == server)).group_by(MonitorLog.date_stamp,  
132 - MonitorLog.hour_stamp,  
133 - MonitorLog.d_minu_stamp).order_by(MonitorLog.hour_stamp.asc(), MonitorLog.d_minu_stamp.asc()).all()  
134 - keys = []  
135 - keys_map = {}  
136 - while start_datetime <= end_datetime:  
137 - key = start_datetime.strftime('%H:%M')  
138 - keys.append(self.get_monitor_log(key))  
139 - keys_map[key] = len(keys)-1  
140 - start_datetime += timedelta(minutes=10)  
141 -  
142 - logs = list(map(lambda log:  
143 - {"key": "%02d:%02d" % (log.hour_stamp, log.minu_stamp*10+cur_minu),  
144 - "cpu_usage": log.cpu_usage, "total_mem": log.total_mem,  
145 - "available_mem": log.available_mem, "used_mem": log.used_mem, "disk": log.disk, "disk_usage": log.disk_usage,  
146 - "net_recv": log.net_recv, "net_send": log.net_send},  
147 - logs))  
148 -  
149 - res["data"] = self.get_result_from_list(logs, keys, keys_map)  
150 - elif grain == "hour":  
151 - # 粒度是一小时  
152 - logs = db.session.query(MonitorLog.hour_stamp.label("key"), func.avg(MonitorLog.cpu_usage).label("cpu_usage"),  
153 - func.avg(MonitorLog.total_mem).label(  
154 - "total_mem"),  
155 - func.avg(MonitorLog.available_mem).label(  
156 - "available_mem"),  
157 - func.avg(MonitorLog.used_mem).label(  
158 - "used_mem"),  
159 - func.avg(MonitorLog.disk).label(  
160 - "disk"),  
161 - func.avg(MonitorLog.disk_usage).label(  
162 - "disk_usage"),  
163 - func.avg(MonitorLog.net_recv).label(  
164 - "net_recv"),  
165 - func.avg(MonitorLog.net_send).label("net_send")).filter(and_(  
166 - MonitorLog.time_stamp <= "{}:59:59".format(  
167 - end_datetime.strftime("%Y-%m-%d %H")),  
168 - MonitorLog.time_stamp >= "{}:59:59".format(  
169 - start_datetime.strftime("%Y-%m-%d %H")),  
170 - MonitorLog.server == server)).group_by(MonitorLog.hour_stamp).order_by(MonitorLog.hour_stamp.asc()).all()  
171 - keys = []  
172 - keys_map = {}  
173 - while start_datetime <= end_datetime:  
174 - key = int(start_datetime.strftime('%H'))  
175 - keys.append(self.get_monitor_log("%02d时" % key))  
176 - keys_map[str(key)] = len(keys)-1  
177 - start_datetime += timedelta(hours=1)  
178 -  
179 - res["data"] = self.get_result(logs, keys, keys_map) 30 + format = "%Y-%m-%d %H:%M:%S"
  31 +
  32 + interval_size = int(re.findall("\d+", interval)[0])
  33 + interval_unit = re.findall("\D+", interval)[0]
  34 +
  35 + if interval_size == 1 and interval_unit == 'm':
  36 + logs = db.session.query(MonitorInfo.time_stamp.label('key'), MonitorInfo.value).filter(
  37 + and_(MonitorInfo.time_stamp <= end_datetime.strftime(format),
  38 + MonitorInfo.time_stamp > start_datetime.strftime(
  39 + format),
  40 + MonitorInfo.server == server,
  41 + MonitorInfo.metrics == metrics_type)).order_by(MonitorInfo.time_stamp).group_by(
  42 + MonitorInfo.time_stamp, MonitorInfo.value)
  43 + datas = list(map(lambda log:
  44 + {"key": datetime.strftime(log.key, format),
  45 + "value": log.value},
  46 + logs))
  47 + datas = reduce(lambda y, x: y if (x['key'] in [i['key'] for i in y]) else (
  48 + lambda z, u: (z.append(u), z))(y, x)[1], datas, [])
  49 + for data in datas:
  50 + res['data'].append([data['key'], data['value']])
180 else: 51 else:
181 - # 按分钟统计  
182 - logs = db.session.query(MonitorLog.hour_stamp.label("hour_stamp"), MonitorLog.minu_stamp.label("minu_stamp"), func.avg(MonitorLog.cpu_usage).label("cpu_usage"),  
183 - func.avg(MonitorLog.total_mem).label(  
184 - "total_mem"),  
185 - func.avg(MonitorLog.available_mem).label(  
186 - "available_mem"),  
187 - func.avg(MonitorLog.used_mem).label("used_mem"),  
188 - func.avg(MonitorLog.disk).label(  
189 - "disk"),  
190 - func.avg(MonitorLog.disk_usage).label(  
191 - "disk_usage"),  
192 - func.avg(MonitorLog.net_recv).label(  
193 - "net_recv"),  
194 - func.avg(MonitorLog.net_send).label("net_send")).filter(and_(  
195 - MonitorLog.time_stamp <= end_datetime.strftime("%Y-%m-%d %H:%M:%S"),  
196 - MonitorLog.time_stamp >= start_datetime.strftime("%Y-%m-%d %H:%M:%S"),  
197 - MonitorLog.server == server)).group_by(MonitorLog.hour_stamp, MonitorLog.minu_stamp).order_by(  
198 - MonitorLog.hour_stamp.asc(), MonitorLog.minu_stamp.asc()).all()  
199 - keys = []  
200 - keys_map = {}  
201 - while start_datetime <= end_datetime:  
202 - key = start_datetime.strftime('%H:%M')  
203 - keys.append(self.get_monitor_log(key))  
204 - keys_map[key] = len(keys)-1  
205 - start_datetime += timedelta(minutes=1)  
206 -  
207 - logs = list(map(lambda log:  
208 - {"key": "%02d:%02d" % (log.hour_stamp, log.minu_stamp),  
209 - "cpu_usage": log.cpu_usage, "total_mem": log.total_mem,  
210 - "available_mem": log.available_mem, "used_mem": log.used_mem, "disk": log.disk, "disk_usage": log.disk_usage,  
211 - "net_recv": log.net_recv, "net_send": log.net_send},  
212 - logs))  
213 -  
214 - res["data"] = self.get_result_from_list(logs, keys, keys_map) 52 + interval_start_datetime = start_datetime
  53 + interval_end_datatime = self.get_end_interval(
  54 + interval_start_datetime, interval_unit, interval_size)
  55 + res_format = "%Y-%m-%d %H:%M:%S"
  56 +
  57 + while interval_end_datatime <= end_datetime:
  58 + logs = db.session.query(MonitorInfo.time_stamp.label('key'), MonitorInfo.value).filter(
  59 + and_(MonitorInfo.time_stamp <= interval_end_datatime.strftime(format),
  60 + MonitorInfo.time_stamp > interval_start_datetime.strftime(
  61 + format), MonitorInfo.server == server, MonitorInfo.metrics == metrics_type)).order_by(MonitorInfo.time_stamp).group_by(
  62 + MonitorInfo.time_stamp, MonitorInfo.value).all()
  63 + tmp_data = list(map(lambda log:
  64 + {"key": datetime.strftime(log.key, res_format),
  65 + "value": log.value},
  66 + logs))
  67 + datas = self.get_sample_data(tmp_data)
  68 + datas = reduce(lambda y, x: y if (x['key'] in [i['key'] for i in y]) else (
  69 + lambda z, u: (z.append(u), z))(y, x)[1], datas, [])
  70 + for data in datas:
  71 + res['data'].append([data['key'], data['value']])
  72 +
  73 + interval_start_datetime = interval_end_datatime
  74 + interval_end_datatime = self.get_end_interval(
  75 + interval_start_datetime, interval_unit, interval_size)
215 res["result"] = True 76 res["result"] = True
216 except Exception as e: 77 except Exception as e:
217 raise e 78 raise e
@@ -224,14 +85,10 @@ class Api(ApiTemplate): @@ -224,14 +85,10 @@ class Api(ApiTemplate):
224 "in": "query", 85 "in": "query",
225 "type": "string", 86 "type": "string",
226 "description": "服务器地址"}, 87 "description": "服务器地址"},
227 - {"name": "grain",  
228 - "in": "query",  
229 - "type": "string",  
230 - "description": "指标粒度：minu：分钟，minu_five：5分钟，minu_ten：10分钟，hour：1小时，day：每天"},  
231 - {"name": "count", 88 + {"name": "interval",
232 "in": "query", 89 "in": "query",
233 "type": "string", 90 "type": "string",
234 - "description": "个数"}, 91 + "description": "间隔"},
235 {"name": "to", 92 {"name": "to",
236 "in": "query", 93 "in": "query",
237 "type": "string", 94 "type": "string",
@@ -239,7 +96,11 @@ class Api(ApiTemplate): @@ -239,7 +96,11 @@ class Api(ApiTemplate):
239 {"name": "start", 96 {"name": "start",
240 "in": "query", 97 "in": "query",
241 "type": "string", 98 "type": "string",
242 - "description": "查询起始时间"} 99 + "description": "查询起始时间"},
  100 + {"name": "metrics_type",
  101 + "in": "query",
  102 + "type": "string",
  103 + "description": "查询指标"}
243 ], 104 ],
244 "responses": { 105 "responses": {
245 200: { 106 200: {
@@ -251,40 +112,50 @@ class Api(ApiTemplate): @@ -251,40 +112,50 @@ class Api(ApiTemplate):
251 } 112 }
252 } 113 }
253 114
254 - def get_monitor_log(self, key):  
255 - return {"key": key,  
256 - "cpu_usage": None, "total_mem": None,  
257 - "available_mem": None, "used_mem": None, "disk": None, "disk_usage": None,  
258 - "net_recv": None, "net_send": None}  
259 -  
260 - def get_result(self, logs, keys, keys_map):  
261 - keys_map_key=keys_map.keys()  
262 - for log in logs:  
263 - if str(log.key) in keys_map_key:  
264 - tmp = keys[keys_map[str(log.key)]]  
265 - if tmp != None:  
266 - tmp['cpu_usage'] = log.cpu_usage  
267 - tmp['total_mem'] = log.total_mem  
268 - tmp['available_mem'] = log.available_mem  
269 - tmp['used_mem'] = log.used_mem  
270 - tmp['disk'] = log.disk  
271 - tmp['disk_usage'] = log.disk_usage  
272 - tmp['net_recv'] = log.net_recv  
273 - tmp['net_send'] = log.net_send  
274 - return keys 115 + def get_end_interval(self, start, unit, size):
  116 + if unit == 'm':
  117 + return start+timedelta(minutes=size)
  118 + elif unit == 'h':
  119 + return start+timedelta(hours=size)
  120 + elif unit == 'd':
  121 + return start+timedelta(days=size)
  122 + else:
  123 + return None
  124 +
  125 + def get_sample_data(self, orginal):
  126 + res = []
  127 + size = len(orginal)
  128 + orginal_stamp = {'head': 1, 'tail': size}
  129 + if size > 1:
  130 + stamp = {'P0': 1,
  131 + 'P50': math.floor(0.5*size),
  132 + # 'P90': math.floor(0.9*size),
  133 + # 'P95': math.floor(0.95*size),
  134 + # 'P99': math.floor(0.99*size),
  135 + 'P100': size}
  136 + elif size == 1:
  137 + stamp = {'P0': 1,
  138 + 'P50': size,
  139 + # 'P90': size,
  140 + # 'P95': size,
  141 + # 'P99': size,
  142 + 'P100': size}
  143 + else:
  144 + return res
  145 +
  146 + for key in dict.keys(orginal_stamp):
  147 + cur_data = orginal[orginal_stamp[key]-1]
  148 + info = {'key': cur_data['key'], 'value': cur_data['value']}
  149 + res.append(info)
  150 +
  151 + data = sorted(orginal, key=lambda x: x['value'])
  152 + for key in dict.keys(stamp):
  153 + cur_data = data[stamp[key]-1]
  154 + info = {'key': cur_data['key'], 'value': cur_data['value']}
  155 + res.append(info)
  156 +
  157 + res.sort(key=self.takeKey)
  158 + return res
275 159
276 - def get_result_from_list(self, logs, keys, keys_map):  
277 - keys_map_key=keys_map.keys()  
278 - for log in logs:  
279 - if str(log["key"]) in keys_map_key:  
280 - tmp = keys[keys_map[str(log["key"])]]  
281 - if tmp != None:  
282 - tmp['cpu_usage'] = log["cpu_usage"]  
283 - tmp['total_mem'] = log["total_mem"]  
284 - tmp['available_mem'] = log["available_mem"]  
285 - tmp['used_mem'] = log["used_mem"]  
286 - tmp['disk'] = log["disk"]  
287 - tmp['disk_usage'] = log["disk_usage"]  
288 - tmp['net_recv'] = log["net_recv"]  
289 - tmp['net_send'] = log["net_send"]  
290 - return keys 160 + def takeKey(self, elem):
  161 + return elem['key']
@@ -6,7 +6,7 @@ @@ -6,7 +6,7 @@
6 6
7 from datetime import datetime 7 from datetime import datetime
8 from time import time 8 from time import time
9 -from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Time, Float, Binary 9 +from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Time, Float, Binary,TIMESTAMP
10 from sqlalchemy.orm import relationship 10 from sqlalchemy.orm import relationship
11 from sqlalchemy.sql.expression import column 11 from sqlalchemy.sql.expression import column
12 from app.models import db 12 from app.models import db
@@ -44,4 +44,28 @@ class MonitorHost(db.Model): @@ -44,4 +44,28 @@ class MonitorHost(db.Model):
44 user = Column(String(256)) 44 user = Column(String(256))
45 password = Column(String(256)) 45 password = Column(String(256))
46 type = Column(String(256)) 46 type = Column(String(256))
47 - host_name=Column(String(256)) 47 + host_name = Column(String(256))
  48 +
  49 +
  50 +class MonitorCpu(db.Model):
  51 + '''
  52 + '''
  53 + __tablename__ = "dmap_monitor_cpu"
  54 + guid = Column(String(256), primary_key=True)
  55 + server = Column(String(256))
  56 + date_stamp = Column(Text)
  57 + time_stamp = Column(TIMESTAMP) # 创建时间戳
  58 + stamp = Column(Text) # P0,P50,P90,P95,P99,P100
  59 + value = Column(Float)
  60 +
  61 +class MonitorInfo(db.Model):
  62 + '''
  63 + '''
  64 + __tablename__ = "dmap_monitor_info"
  65 + guid = Column(String(256), primary_key=True)
  66 + metrics=Column(String(256))
  67 + server = Column(String(256))
  68 + date_stamp = Column(Text)
  69 + time_stamp = Column(TIMESTAMP) # 创建时间戳
  70 + stamp = Column(Text) # P0,P50,P90,P95,P99,P100
  71 + value = Column(Float)
1 # import schedule 1 # import schedule
2 from flask import json 2 from flask import json
3 -import paramiko  
4 -from sqlalchemy.sql.sqltypes import JSON  
5 -from .models import MonitorHost, MonitorLog 3 +from .models import MonitorHost, MonitorInfo
6 import datetime 4 import datetime
7 import math 5 import math
8 import time 6 import time
@@ -13,6 +11,10 @@ from app.util.component.PGUtil import PGUtil @@ -13,6 +11,10 @@ from app.util.component.PGUtil import PGUtil
13 import configure 11 import configure
14 from app.util.component.StructuredPrint import StructurePrint 12 from app.util.component.StructuredPrint import StructurePrint
15 import traceback 13 import traceback
  14 +import requests
  15 +
  16 +min_size = 60
  17 +size = 60
16 18
17 19
18 def pull_metric(): 20 def pull_metric():
@@ -22,25 +24,50 @@ def pull_metric(): @@ -22,25 +24,50 @@ def pull_metric():
22 configure.SQLALCHEMY_DATABASE_URI) 24 configure.SQLALCHEMY_DATABASE_URI)
23 sys_ds = PGUtil.open_pg_data_source( 25 sys_ds = PGUtil.open_pg_data_source(
24 0, configure.SQLALCHEMY_DATABASE_URI) 26 0, configure.SQLALCHEMY_DATABASE_URI)
25 -  
26 - #拉取服务器信息 27 +
  28 + # 拉取服务器信息
27 29
28 hosts = sys_session.query( 30 hosts = sys_session.query(
29 MonitorHost.host) 31 MonitorHost.host)
30 for host in hosts: 32 for host in hosts:
31 - request_uri="http://{}".format(host.host)  
32 -  
33 -  
34 - 33 + try:
  34 + host_name = host.host
  35 + request_uri = "http://{}".format(host_name)
  36 + response = requests.request("get", request_uri)
  37 + if response.status_code == 200:
  38 + text = response.text
  39 + data = json.loads(text)
  40 + len_metric = len(data)
  41 + if len_metric > min_size:
  42 + metric_data = data[len_metric-min_size:len_metric]
  43 + else:
  44 + continue
  45 +
  46 + type_list = ["cpu_per", "memory_per", "disk_per",
  47 + "disk_read", "disk_write","disk_read_count",
  48 + "disk_write_count","network_sent","network_recv"]
  49 + for type in type_list:
  50 + sample_data = get_sample_data(
  51 + metric_data, type, host_name)
  52 + sys_session.add_all(sample_data)
  53 + sys_session.commit()
  54 +
  55 + except Exception as e:
  56 + StructurePrint().print(e.__str__()+":" + traceback.format_exc(), "error")
35 # 获取数据并汇聚为1min的数据入库 57 # 获取数据并汇聚为1min的数据入库
36 # 结束 58 # 结束
37 - pass  
38 except Exception as e: 59 except Exception as e:
39 StructurePrint().print(e.__str__()+":" + traceback.format_exc(), "error") 60 StructurePrint().print(e.__str__()+":" + traceback.format_exc(), "error")
40 - pass 61 + finally:
  62 + sys_session.rollback()
  63 + if sys_session:
  64 + sys_session.close()
  65 + if sys_ds:
  66 + sys_ds.Destroy()
  67 +
41 68
42 def start_schedule(): 69 def start_schedule():
43 - # # 2分钟巡检一次 70 + # # 1分钟巡检一次
44 try: 71 try:
45 StructurePrint().print("start_schedule") 72 StructurePrint().print("start_schedule")
46 schedule.every(1).minutes.do(pull_metric) 73 schedule.every(1).minutes.do(pull_metric)
@@ -52,3 +79,54 @@ def start_schedule(): @@ -52,3 +79,54 @@ def start_schedule():
52 # # Stop the background thread 79 # # Stop the background thread
53 # time.sleep(10) 80 # time.sleep(10)
54 # stop_run_continuously.set() 81 # stop_run_continuously.set()
  82 +
  83 +
  84 +def get_sample_data(orginal, name, host):
  85 + res = []
  86 + size = len(orginal)
  87 + orginal_stamp = {'head': 1, 'tail': size}
  88 + if size > 1:
  89 + stamp = {'P0': 1,
  90 + 'P50': math.floor(0.5*size),
  91 + # 'P90': math.floor(0.9*size),
  92 + # 'P95': math.floor(0.95*size),
  93 + # 'P99': math.floor(0.99*size),
  94 + 'P100': size}
  95 + elif size == 1:
  96 + stamp = {'P0': 1,
  97 + 'P50': size,
  98 + # 'P90': size,
  99 + # 'P95': size,
  100 + # 'P99': size,
  101 + 'P100': size}
  102 + else:
  103 + return res
  104 +
  105 + for key in dict.keys(orginal_stamp):
  106 + cur_data = orginal[orginal_stamp[key]-1]
  107 + info = get_data(key, host, name, cur_data)
  108 + res.append(info)
  109 +
  110 + data = sorted(orginal, key=lambda x: x[name])
  111 + for key in dict.keys(stamp):
  112 + cur_data = data[stamp[key]-1]
  113 + info = get_data(key, host, name, cur_data)
  114 + res.append(info)
  115 +
  116 + return res
  117 +
  118 +
  119 +def get_data(stamp, host, metrics_name, cur_data):
  120 + time_stamp = datetime.datetime.strptime(
  121 + cur_data['timestamp'], "%Y-%m-%d %H:%M:%S")
  122 + date_stamp = time_stamp.strftime("%Y-%m-%d")
  123 + guid = uuid.uuid1().__str__()
  124 + value = cur_data[metrics_name]
  125 + info = MonitorInfo(guid=guid,
  126 + server=host,
  127 + date_stamp=date_stamp,
  128 + time_stamp=time_stamp,
  129 + stamp=stamp,
  130 + value=value,
  131 + metrics=metrics_name)
  132 + return info
请 注册 或 登录 后发表评论