Skip to content

Commit c3f5e22

Browse files
tianxiu2b2ttianxiu2b2t
authored
Merge pull request #62 from TTB-Network/dev/downloader
修复下载（添加下载错误时候返回下载地址），修复缓存可能会导致程序oom
2 parents b93bd3c + aad37c3 commit c3f5e22

File tree

9 files changed

+320
-245
lines changed

9 files changed

+320
-245
lines changed

‎bmclapi_dashboard/static/js/index.min.js‎

Lines changed: 62 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -534,7 +534,36 @@ class Application {
534534
border-radius: 4px;
535535
background-color: var(--border-background);
536536
padding-left: 24px;
537-
padding-bottom: 16px;`
537+
padding-bottom: 16px;`,
538+
'.tqdm': [
539+
'display: flex;',
540+
'margin-top: 16px',
541+
],
542+
".tqdm-outline": [
543+
"display: block"
544+
],
545+
'.tqdm .tqdm-progressbar': [
546+
'margin-left: 8px',
547+
'margin-right: 8px',
548+
'display: flex',
549+
'flex-grow: 1;',
550+
'align-items: center'
551+
],
552+
".tqdm.tqdm-outline .tqdm-progressbar": [
553+
"margin: 0"
554+
],
555+
'.tqdm .tqdm-backgroundbar': [
556+
'padding: 2px',
557+
'margin-left: 8px',
558+
'height: 4px',
559+
'background: var(--background)',
560+
'width: 100%'
561+
],
562+
'.tqdm .tqdm-bar': [
563+
'width: 50%',
564+
'height: 4px',
565+
'background: var(--main-color)',
566+
]
538567
}
539568
this.$side = this.createElement("aside").class("side")
540569
this.$container = this.createElement("div").class("main").append(
@@ -1990,7 +2019,7 @@ $I18N.addLangs("zh_cn", {
19902019
"menu.master.rank": "排行榜",
19912020
"menu.config": "配置",
19922021
"menu.config.storage": "存储设置",
1993-
"tqdm": "%value%/%total%, %item%/s",
2022+
"tqdm": "%value%/%total% [%start% < %end%, %item%/s]",
19942023
"storage.webdav": "正在获取WebDav文件中",
19952024
"cluster.want_enable": "正在启用",
19962025
"cluster.enabled.trusted": "正常工作",
@@ -2704,6 +2733,16 @@ app.$Menu.add("dashboard", new class {
27042733
hits: app.createEcharts().style("min-height: 162px;"),
27052734
bytes: app.createEcharts().style("min-height: 162px;").setFormatter((n) => this._format_bytes(n))
27062735
}
2736+
this.pbar = app.createElement("div").class("tqdm").append(
2737+
app.createElement("p"),
2738+
app.createElement("p").class("tqdm-progressbar").append(
2739+
app.createElement("span").setText("100%"),
2740+
app.createElement("div").class("tqdm-backgroundbar").append(
2741+
app.createElement("div").class("tqdm-bar")
2742+
),
2743+
),
2744+
app.createElement("p")
2745+
)
27072746
this.page = [
27082747
app.createElement("div").class("panel").append(
27092748
app.createFlex().append(
@@ -2713,17 +2752,18 @@ app.$Menu.add("dashboard", new class {
27132752
),
27142753
app.createElement("div").append(
27152754
app.createElement("p").class("title").setI18N("dashboard.status"),
2716-
app.createElement("p").append(
2717-
app.createElement("span").class("value").setText("-"),
2718-
app.createElement("span").append(
2719-
app.createElement("span").class("value").setText(" | "),
2720-
app.createElement("span").class("value").setText(""),
2721-
app.createElement("span").class("value").setText(" "),
2722-
app.createElement("span").setText("")
2723-
),
2724-
)
2755+
app.createElement("p").class("value")
27252756
)
2726-
).minWidth(896).child(2)
2757+
).minWidth(896).child(2).addResize(() => {
2758+
this.pbar.removeClass("tqdm-outline")
2759+
var width = this.pbar.getChildren()[1].valueOf().offsetWidth
2760+
if (width >= 84) {
2761+
this.pbar.removeClass("tqdm-outline")
2762+
} else {
2763+
this.pbar.class("tqdm-outline")
2764+
}
2765+
}),
2766+
this.pbar
27272767
),
27282768
app.createElement("div").class("panel nopadding").style("margin-bottom: 0").append(
27292769
app.createFlex(true).class("flex-space-between").child(2).minWidth(512).append(
@@ -3696,15 +3736,21 @@ app.$Menu.add("dashboard", new class {
36963736
}
36973737
}
36983738
setStatus() {
3699-
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].getChildren()[0].setI18N(this.status.key)
3700-
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].getChildren()[1].style(`display: ${this.status.progress ? 'inline' : 'none'}`)
3739+
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].setI18N(this.status.key)
3740+
this.pbar.valueOf().style.display = `${this.status.progress ? '' : 'none'}`
37013741
if (this.status.progress) {
37023742
var value_formatter = this.status.progress.desc == "files.downloading" ? (n) => this._format_bytes(n) : (n) => n
3703-
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].getChildren()[1].getChildren()[1].setI18N(this.status.progress.desc)
3704-
this.page[0].getChildren()[0].getChildren()[1].getChildren()[1].getChildren()[1].getChildren()[3].setI18N("tqdm", {
3743+
var percent = ((this.status.progress.value / this.status.progress.total) * 100)
3744+
percent = isNaN(percent) ? 0 : percent
3745+
this.pbar.getChildren()[0].setI18N(this.status.progress.desc)
3746+
this.pbar.getChildren()[1].getChildren()[0].setText(`${percent.toFixed(0)}%`)
3747+
this.pbar.getChildren()[1].getChildren()[1].getChildren()[0].style(`width: ${percent}%`)
3748+
this.pbar.getChildren()[2].setI18N("tqdm", {
37053749
value: value_formatter(this.status.progress.value),
37063750
total: value_formatter(this.status.progress.total),
37073751
item: value_formatter(this.status.progress.speed),
3752+
start: this.status.progress.start,
3753+
end: this.status.progress.end
37083754
})
37093755
}
37103756
}

‎core/api.py‎

Lines changed: 97 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@
55
import hashlib
66
import io
77
from pathlib import Path
8+
import time
89
from typing import Optional
910
import pyzstd as zstd
1011
import aiofiles
1112

12-
from core import web
13+
from core import logger, scheduler, unit, web
1314
from core.config import Config
14-
from core.const import CACHE_BUFFER_COMPRESSION_MIN_LENGTH
15+
from core.const import CACHE_BUFFER_COMPRESSION_MIN_LENGTH, CACHE_TIME, CHECK_CACHE, CACHE_BUFFER
1516

1617

1718
class FileCheckType(Enum):
@@ -27,6 +28,7 @@ class FileContentType(Enum):
2728
DATA = "data"
2829
URL = "url"
2930
PATH = "path"
31+
EMPTY = "empty"
3032

3133
@dataclass
3234
class BMCLAPIFile:
@@ -47,66 +49,114 @@ def __eq__(self, other):
4749
)
4850
return False
4951

50-
5152
@dataclass
5253
class File:
53-
path: Path | str
5454
hash: str
5555
size: int
56-
last_hit: float = 0
57-
last_access: float = 0
58-
expiry: Optional[float] = None
59-
data: Optional[io.BytesIO] = None
56+
type: FileContentType
57+
data: io.BytesIO | str | Path = None
58+
expiry: float = 0
59+
compressed: bool = False
60+
data_length: int = 0
6061
cache: bool = False
6162
headers: Optional["web.Header"] = None
62-
compressed: bool = False
63-
64-
def is_url(self):
65-
if not isinstance(self.path, str):
66-
return False
67-
return self.path.startswith("http://") or self.path.startswith("https://")
68-
69-
def is_path(self):
70-
return isinstance(self.path, Path)
7163

72-
def get_path(self) -> str | Path:
73-
return self.path
64+
def set_data(self, data: io.BytesIO | str | Path):
65+
if isinstance(data, io.BytesIO):
66+
length = len(data.getbuffer())
67+
if CACHE_BUFFER_COMPRESSION_MIN_LENGTH <= length:
68+
self.data = io.BytesIO(zstd.compress(data.getbuffer()))
69+
self.data_length = len(self.data.getbuffer())
70+
self.compressed = True
71+
else:
72+
self.data = data
73+
self.data_length = len(data.getbuffer())
74+
self.compressed = False
75+
self.type = FileContentType.DATA
76+
elif isinstance(data, str):
77+
self.data_length = len(data)
78+
self.data = data
79+
self.type = FileContentType.URL
80+
elif isinstance(data, Path):
81+
self.data_length = len(str(data))
82+
self.data = data
83+
self.type = FileContentType.PATH
7484

7585
def get_data(self):
76-
if not self.data:
77-
return io.BytesIO()
78-
if not self.compressed:
86+
if self.compressed:
87+
return io.BytesIO(zstd.decompress(self.data.getbuffer()))
88+
else:
7989
return self.data
80-
return io.BytesIO(zstd.decompress(self.data.getbuffer()))
81-
82-
def set_data(self, data: io.BytesIO | memoryview | bytes):
83-
if not isinstance(data, io.BytesIO):
84-
data = io.BytesIO(data)
85-
data_length = len(data.getbuffer())
86-
if data_length >= CACHE_BUFFER_COMPRESSION_MIN_LENGTH:
87-
compressed_data = zstd.compress(data.getbuffer())
88-
if data_length > len(compressed_data):
89-
self.compressed = True
90-
self.data = io.BytesIO(compressed_data)
91-
return
92-
self.compressed = False
93-
self.data = data
94-
95-
90+
def is_url(self):
91+
return self.type == FileContentType.URL
92+
def is_path(self):
93+
return self.type == FileContentType.PATH
94+
def get_path(self) -> Path:
95+
return self.data
9696
@dataclass
9797
class StatsCache:
9898
total: int = 0
9999
bytes: int = 0
100+
data_bytes: int = 0
100101

101102

102103
class Storage(metaclass=abc.ABCMeta):
103104
def __init__(self, name, width: int) -> None:
104105
self.name = name
105106
self.disabled = False
106107
self.width = width
107-
108+
self.cache: dict[str, File] = {}
109+
self.cache_timer = scheduler.repeat(
110+
self.clear_cache, delay=CHECK_CACHE, interval=CHECK_CACHE
111+
)
108112
def get_name(self):
109113
return self.name
114+
115+
def get_cache(self, hash: str) -> Optional[File]:
116+
file = self.cache.get(hash, None)
117+
if file is not None:
118+
file.cache = True
119+
if not file.is_url():
120+
file.expiry = time.time() + CACHE_TIME
121+
return file
122+
123+
def is_cache(self, hash: str) -> Optional[File]:
124+
return hash in self.cache
125+
126+
def set_cache(self, hash: str, file: File):
127+
self.cache[hash] = file
128+
129+
def clear_cache(self):
130+
hashs = set()
131+
data = sorted(
132+
self.cache.copy().items(),
133+
key=lambda x: x[1].expiry, reverse=True)
134+
size = 0
135+
old_size = 0
136+
for hash, file in data:
137+
if file.type == FileContentType.EMPTY:
138+
continue
139+
size += file.data_length
140+
if (size <= CACHE_BUFFER and file.expiry >= time.time()):
141+
continue
142+
hashs.add(hash)
143+
old_size += file.data_length
144+
for hash in hashs:
145+
self.cache.pop(hash)
146+
logger.tinfo(
147+
"cluster.info.clear_cache.count",
148+
name=self.name,
149+
count=unit.format_number(len(hashs)),
150+
size=unit.format_bytes(old_size),
151+
)
152+
153+
def get_cache_stats(self) -> StatsCache:
154+
stat = StatsCache()
155+
for file in self.cache.values():
156+
stat.total += 1
157+
stat.bytes += file.size
158+
stat.data_bytes += file.data_length
159+
return stat
110160

111161
@abc.abstractmethod
112162
async def get(self, file: str, offset: int = 0) -> File:
@@ -175,15 +225,15 @@ async def removes(self, hashs: list[str]) -> int:
175225
"""
176226
raise NotImplementedError
177227

178-
@abc.abstractmethod
179-
async def get_cache_stats(self) -> StatsCache:
180-
"""
181-
dir: path
182-
Getting cache files
183-
return StatsCache
184-
"""
185-
raise NotImplementedError
228+
@dataclass
229+
class OpenbmclapiAgentConfiguration:
230+
source: str
231+
concurrency: int
186232

233+
@dataclass
234+
class ResponseRedirects:
235+
status: int
236+
url: str
187237

188238
def get_hash(org):
189239
if len(org) == 32:

0 commit comments

Comments
 (0)