Skip to content

Commit f1cb130

Browse files
SilianZSilianZ
authored
Merge pull request #10 from TTB-Network/dev/stats
feat: Add Statistics for files, request per seconds...
2 parents 703b4c8 + cb71832 commit f1cb130

File tree

11 files changed

+433
-73
lines changed

11 files changed

+433
-73
lines changed

‎README.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,6 @@
2020

2121
🎉 __*新增功能！*__ 基于 Echart 的 OpenBMCLAPI 仪表盘（Dashboard）。
2222

23-
🎉 __*新增功能！*__ 基于 loguru 的**日志器**。
24-
25-
2623
</div>
2724

2825
# 简介
@@ -109,10 +106,14 @@ web_port: 8800
109106
web_publicport: 8800
110107
```
111108

109+
# 贡献
110+
111+
如果你有能力，你可以向我们的团队[发送邮件](mailto://administrator@ttb-network.top)或团队所有者[发送邮件](mailto://silian_zheng@outlook.com)并申请加入开发者行列。
112+
112113
# 鸣谢
113114

114115
[LiterMC/go-openbmclapi](https://github.com/LiterMC/go-openbmclapi)
115116

116117
[bangbang93/openbmclapi](https://github.com/bangbang93/openbmclapi)
117118

118-
[SALTWOOD/CSharp-OpenBMCLAPI](https://github.com/SALTWOOD/CSharp-OpenBMCLAPI)
119+
[SALTWOOD/CSharp-OpenBMCLAPI](https://github.com/SALTWOOD/CSharp-OpenBMCLAPI)

‎container/bmclapi_dashboard/static/js/index.js‎

Lines changed: 96 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
const UNIT_BYTES = [
2-
"", "K", "M", "G", "T", "E"
2+
"K", "M", "G", "T", "E"
33
];
44
const calc_bits = (v) => {
55
v *= 8
@@ -112,10 +112,9 @@ const calc_more_bytes = (...values) => {
112112
axios.get("master?url=/openbmclapi/metric/dashboard").then(response => {
113113
if (response.status != 200) return
114114
data = response.data
115-
console.log(data)
116115
document.getElementById("t-clusters-nodes").innerText = data.currentNodes
117116
document.getElementById("t-clusters-bandwidth").innerText = data.currentBandwidth.toFixed(2) + " M"
118-
document.getElementById("t-clusters-bytes").innerText = calc_bytes(data.bytes * 1024.0)
117+
document.getElementById("t-clusters-bytes").innerText = calc_bytes(data.bytes)
119118
document.getElementById("t-clusters-req").innerText = (data.hits / 10000).toFixed(2)
120119
nodes = []
121120
bytes = []
@@ -203,6 +202,100 @@ const calc_more_bytes = (...values) => {
203202
).childWidth("33.33%", "33.33%", "33.33%").valueOf()
204203
]
205204
}
205+
},
206+
"dashboard": {
207+
"connect": () => {
208+
if (!("dashboard" in core_modules_locals)) {
209+
core_modules_locals["dashboard"] = {
210+
"refresh": () => {
211+
axios.get("/dashboard").then(resp => {
212+
if (resp.status != 200) return
213+
data = resp.data
214+
req = Array.from({ length: 24 }, (_, __) => null)
215+
hits = Array.from({ length: 24 }, (_, __) => null)
216+
bandwidth = Array.from({ length: 24 }, (_, __) => null)
217+
bytes = Array.from({ length: 24 }, (_, __) => null)
218+
days = data.days[0]
219+
for (day of data.days) {
220+
if (days._day < day._day)
221+
days = day
222+
}
223+
for (hourly of data.hourly) {
224+
const hour = hourly._hour
225+
req[hour] = (hourly.qps / 10000).toFixed(2)
226+
hits[hour] = (hourly.hits / 10000).toFixed(2)
227+
bandwidth[hour] = (hourly.bandwidth * 8 / 1024.0 / 1024.0).toFixed(2)
228+
bytes[hour] = (hourly.bytes / 1024.0 / 1024.0 / 1024.0).toFixed(2)
229+
}
230+
core_modules_locals["dashboard"]["req"] .setOption({title: {text: "每小时请求分布（万）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: req}]})
231+
core_modules_locals["dashboard"]["bytes"] .setOption({title: {text: "每小时流量分布（GiB）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">流量: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'GiB</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bytes}]})
232+
core_modules_locals["dashboard"]["hit"] .setOption({title: {text: "每小时请求文件数（万）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求文件： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: hits}]})
233+
core_modules_locals["dashboard"]["bandwidth"] .setOption({title: {text: "每小时峰值出网带宽（Mbps）"}, tooltip:{formatter: e => e[0].data == null ? '' : '<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">带宽： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'Mbps</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bandwidth}]})
234+
document.getElementById("t-d-req").innerText = (days.qps / 10000).toFixed(2)
235+
document.getElementById("t-d-bytes").innerText = calc_bytes(days.bytes)
236+
document.getElementById("t-d-hit").innerText = (days.hit / 10000).toFixed(2)
237+
document.getElementById("t-d-bandwidth").innerText = (days.bandwidth * 8 / 1024.0 / 1024.0).toFixed(2) + " M"
238+
})
239+
},
240+
"bandwidth": echarts.init(document.getElementById("e-d-bandwidth")),
241+
"bytes": echarts.init(document.getElementById("e-d-bytes")),
242+
"req": echarts.init(document.getElementById("e-d-req")),
243+
"hit": echarts.init(document.getElementById("e-d-hit")),
244+
"load": echarts.init(document.getElementById("e-d-cpu")),
245+
"options": {tooltip:{trigger:"axis",axisPointer:{type:"cross",label:{backgroundColor:"#0FC6C2"}}},grid:{left:"3%",right:"4%",bottom:"3%",containLabel:!0},xAxis:{type:"category",boundaryGap:!1,data:time_hours},yAxis:{type:"value",axisLabel:{formatter:"{value}"}},series:[{name:"",type:"line",stack:"",areaStyle:{},color:"#0FC6C2",symbol:"circle",symbolSize:4,data:[],smooth:!0,animationEasing:"cubicOut",animationDelay:function(t){return 10*t}}]},
246+
}
247+
core_modules_locals["dashboard"]["bandwidth"].setOption(core_modules_locals["dashboard"]["options"])
248+
core_modules_locals["dashboard"]["bytes"] .setOption(core_modules_locals["dashboard"]["options"])
249+
core_modules_locals["dashboard"]["req"] .setOption(core_modules_locals["dashboard"]["options"])
250+
core_modules_locals["dashboard"]["hit"] .setOption(core_modules_locals["dashboard"]["options"])
251+
}
252+
core_modules_locals["dashboard"]["timer"] = setInterval(core_modules_locals["dashboard"].refresh, 30000)
253+
core_modules_locals["dashboard"].refresh()
254+
},
255+
"page": () => [
256+
ExtendFlex().append(
257+
ExtendElement("div").append(
258+
ExtendElement("div").css("panel").append(
259+
ExtendElement("h4").text("当日出网峰值带宽").valueOf(),
260+
ExtendElement("h2").append(
261+
ExtendElement("span").text("0 ").id("t-d-bandwidth").valueOf(),
262+
ExtendElement("span").text("bps").valueOf()
263+
).valueOf(),
264+
ExtendElement("div").id("e-d-bandwidth").style("height: 216px; width: 100%").valueOf()
265+
).valueOf(),
266+
ExtendElement("div").css("panel").append(
267+
ExtendElement("h4").text("当日请求文件数").valueOf(),
268+
ExtendElement("h2").append(
269+
ExtendElement("span").text("0 ").id("t-d-hit").valueOf(),
270+
ExtendElement("span").text("万").valueOf()
271+
).valueOf(),
272+
ExtendElement("div").id("e-d-hit").style("height: 216px; width: 100%").valueOf()
273+
).valueOf(),
274+
),
275+
ExtendElement("div").append(
276+
ExtendElement("div").css("panel").append(
277+
ExtendElement("h4").text("当日总流量").valueOf(),
278+
ExtendElement("h2").append(
279+
ExtendElement("span").text("0 ").id("t-d-bytes").valueOf(),
280+
ExtendElement("span").text("iB").valueOf()
281+
).valueOf(),
282+
ExtendElement("div").id("e-d-bytes").style("height: 216px; width: 100%").valueOf()
283+
).valueOf(),
284+
ExtendElement("div").css("panel").append(
285+
ExtendElement("h4").text("当日请求数").valueOf(),
286+
ExtendElement("h2").append(
287+
ExtendElement("span").text("0 ").id("t-d-req").valueOf(),
288+
ExtendElement("span").text("万").valueOf()
289+
).valueOf(),
290+
ExtendElement("div").id("e-d-req").style("height: 216px; width: 100%").valueOf()
291+
).valueOf(),
292+
),
293+
ExtendElement("div").css("panel").append(
294+
ExtendElement("h4").text("五分钟负载").valueOf(),
295+
ExtendElement("div").id("e-d-cpu").style("height: 98%; width: 100%").valueOf()
296+
),
297+
).childWidth("33.33%", "33.33%", "33.33%").valueOf()
298+
]
206299
}
207300
}
208301
const handler = ((root, key, type) => {

‎container/cluster.py‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@
1717
import utils
1818
import stats
1919
import web
20-
from logger import logger
20+
import logger
2121
from tqdm import tqdm
2222

2323
PY_VERSION = "1.0.0"
2424
VERSION = "1.9.7"
2525
UA = f"openbmclapi-cluster/{VERSION} Python/{PY_VERSION}"
2626
URL = 'https://openbmclapi.bangbang93.com/'
27-
COUNTER = stats.Counters()
28-
27+
COUNTER = stats.counter
28+
LAST_COUNTER = stats.last_counter
2929
@dataclass
3030
class BMCLAPIFile:
3131
path: str
@@ -127,7 +127,7 @@ async def check_file(self):
127127
total = len(filelist)
128128
byte = 0
129129
miss = []
130-
pbar = tqdm(total=total, unit=' file(s)', unit_scale=True)
130+
pbar = tqdm(file=logger.PRINTSTDOUT, total=total, unit=' file(s)', unit_scale=True)
131131
pbar.set_description("Checking files")
132132
for i, file in enumerate(filelist):
133133
filepath = str(self.dir) + f"/{file.hash[:2]}/{file.hash}"
@@ -214,12 +214,12 @@ async def message(self, type, data):
214214
logger.error("Error:" + data[0]['message'])
215215
Timer.delay(self.enable)
216216
elif type == "keep-alive":
217-
COUNTER.hit -= self.cur_counter.hit
218-
COUNTER.bytes -= self.cur_counter.bytes
217+
LAST_COUNTER.hit += self.cur_counter.hit
218+
LAST_COUNTER.bytes += self.cur_counter.bytes
219219
self.keepalive = Timer.delay(self.keepaliveTimer, (), 5)
220220
async def keepaliveTimer(self):
221-
self.cur_counter.hit = COUNTER.hit
222-
self.cur_counter.bytes = COUNTER.bytes
221+
self.cur_counter.hit = COUNTER.hit - LAST_COUNTER.hit
222+
self.cur_counter.bytes = COUNTER.bytes - LAST_COUNTER.bytes
223223
await self.emit("keep-alive", {
224224
"time": time.time(),
225225
"hits": self.cur_counter.hit,
@@ -272,7 +272,7 @@ async def __call__(self) -> io.BytesIO:
272272
if self.size == stat.st_size and self.last_file == stat.st_mtime:
273273
self.last = time.time() + 1440
274274
return self.buf
275-
self.buf.seek(0, os.SEEK_SET)
275+
self.buf = io.BytesIO()
276276
async with aiofiles.open(self.file, "rb") as r:
277277
while (data := await r.read(min(config.IO_BUFFER, stat.st_size - self.buf.tell()))) and self.buf.tell() < stat.st_size:
278278
self.buf.write(data)
@@ -288,6 +288,8 @@ async def init():
288288
global storage
289289
Timer.delay(storage.check_file)
290290
app = web.app
291+
def record_bandwidth(sent: int, recv: int):
292+
COUNTER.bandwidth += sent
291293
@app.get("/measure/{size}")
292294
async def _(request: web.Request, size: int, s: str, e: str):
293295
#if not config.SKIP_SIGN:
@@ -302,12 +304,14 @@ async def _(request: web.Request, hash: str, s: str, e: str):
302304
#if not config.SKIP_SIGN:
303305
# check_sign(request.protocol + "://" + request.host + request.path, config.CLUSTER_SECRET, s, e)
304306
file = Path(str(storage.dir) + "/" + hash[:2] + "/" + hash)
307+
COUNTER.qps += 1
305308
if not file.exists():
306309
return web.Response(status_code=404)
307310
if hash not in cache:
308311
cache[hash] = FileCache(file)
309312
data = await cache[hash]()
310-
COUNTER.bytes += len(data.getbuffer())
313+
COUNTER.bytes += cache[hash].size
314+
request.client.set_log_network(record_bandwidth)
311315
COUNTER.hit += 1
312316
return data.getbuffer()
313317
router: web.Router = web.Router("/bmcl")
@@ -324,6 +328,12 @@ async def _(request: web.Request, url: str):
324328
async with session.get(url) as resp:
325329
content.write(await resp.read())
326330
return content # type: ignore
331+
@router.get("/dashboard")
332+
async def _():
333+
return {
334+
"hourly": stats.hourly(),
335+
"days": stats.days()
336+
}
327337
app.mount(router)
328338

329339
async def clearCache():

‎container/config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ def __init__(self, path: str) -> None:
1111

1212
def load(self):
1313
with open(self.file, "r", encoding="utf-8") as f:
14-
self.cfg = yaml.load(f.read(), Loader=yaml.FullLoader)
14+
self.cfg = yaml.load(f.read(), Loader=yaml.FullLoader) or {}
1515

1616
def get(self, key, default_):
1717
value = self.cfg.get(key, default_)

‎container/logger.py‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,47 @@
1-
from loguru import logger
2-
from pathlib import Path
1+
from enum import Enum
2+
import inspect
3+
import io
4+
import sys
5+
import time
6+
STDOUT = sys.stdout
7+
class Stdout(io.StringIO):
8+
def write(self, __s: str) -> int:
9+
return STDOUT.write(__s)
10+
def flush(self) -> None:
11+
return STDOUT.flush()
12+
def seek(self, __cookie: int, __whence: int = 0) -> int:
13+
return STDOUT.seek(__cookie, __whence)
14+
sys.stdout = Stdout()
315

4-
logger.add(Path("./logs/{time}.log"), rotation="3 hours")
16+
class PrintStdout(io.StringIO):
17+
def write(self, __s: str) -> int:
18+
info(__s.lstrip("\r"), flush=True)
19+
return len(__s)
20+
PRINTSTDOUT = PrintStdout()
21+
22+
class Level(Enum):
23+
DEBUG = 0
24+
INFO = 1
25+
WARNING = 2
26+
ERROR = 3
27+
LevelColors: dict[Level, str] = {
28+
Level.DEBUG: "reset",
29+
Level.INFO: "green",
30+
Level.WARNING: "yellow",
31+
Level.ERROR: "red"
32+
}
33+
34+
def logger(*values, level: Level, flush: bool = False, stack: list[inspect.FrameInfo]):
35+
print(*(f"<<<flush:{flush},time:{time.time()},color:{LevelColors.get(level, 'reset')}>>>[{level.name.upper()}]", *values))
36+
37+
def info(*values, flush: bool = False):
38+
return logger(*values, flush=flush, level=Level.INFO, stack=inspect.stack())
39+
40+
def error(*values, flush: bool = False):
41+
return logger(*values, flush=flush, level=Level.ERROR, stack=inspect.stack())
42+
43+
def warning(*values, flush: bool = False):
44+
return logger(*values, flush=flush, level=Level.WARNING, stack=inspect.stack())
45+
46+
def debug(*values, flush: bool = False):
47+
return logger(*values, flush=flush, level=Level.DEBUG, stack=inspect.stack())

‎container/main.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
from datetime import datetime
2+
import os
3+
import time
4+
cur = time.time()
5+
os.environ["UTC"] = str(int((datetime.fromtimestamp(cur) - datetime.utcfromtimestamp(cur)).total_seconds() / 3600))
6+
17
if __name__ == "__main__":
28
import web
39
web.init()

0 commit comments

Comments
 (0)