Skip to content

Commit 2643348

Browse files
wcbingwcbing
committed
refactor: github releases, arch name, multithreading
rewrite github downloader and configuration to store version tag in sqlite. change 'version_tag' to 'releases_tag', 'stripped_version' to 'version'. add 'all' arch, change 'x86_64' to 'amd64'. add multithreading for downloader. rename init_deb.py to init-deb.py
1 parent 07d25c2 commit 2643348

File tree

9 files changed

+304
-228
lines changed

9 files changed

+304
-228
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
data/github-local.json
22
data/deb.db
3-
deb/
3+
deb/
4+
__pycache__/

‎check_downloader.py‎

Lines changed: 47 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,16 @@
22
import subprocess
33
import os
44
import sqlite3
5+
import sys
56
import logging
7+
from threading import Lock
8+
9+
BASE_DIR = "deb"
10+
DB_DIR = "data"
11+
USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)" # from Debian 12
12+
13+
version_lock = Lock()
614

7-
base_dir = "deb"
815
logging.basicConfig(
916
format="%(asctime)s %(message)s",
1017
datefmt="%Y/%m/%d %H:%M:%S",
@@ -13,68 +20,58 @@
1320

1421

1522
def download(url):
16-
file_dir = os.path.join(base_dir, os.path.dirname(url))
17-
if not os.path.exists(file_dir):
18-
os.makedirs(file_dir)
19-
file_path = os.path.join(base_dir, url.split("?")[0])
20-
# 用 curl 模拟 apt 下载文件，User-Agent 来自 Debian 12
21-
subprocess.run(
22-
[
23-
"curl",
24-
"-H",
25-
"User-Agent: Debian APT-HTTP/1.3 (2.6.1)",
26-
"-fsLo",
27-
file_path,
28-
url,
29-
]
30-
)
23+
"""Download file using curl with APT User-Agent."""
24+
file_path = os.path.join(BASE_DIR, url.split("?")[0])
25+
os.makedirs(os.path.dirname(file_path), exist_ok=True)
26+
subprocess.run(["curl", "-H", f"User-Agent: {USER_AGENT}", "-fsLo", file_path, url])
3127

3228

33-
def check_download(name, version, url, arch):
29+
def check_download(name, version, url, arch="amd64"):
30+
"""Check and handle package download/update."""
3431
logging.info("%s:%s = %s", name, arch, version)
3532

36-
# connect to db
37-
with sqlite3.connect(os.path.join("data", f"{base_dir}.db")) as conn:
38-
cur = conn.cursor()
39-
res = cur.execute(
40-
f"SELECT version, url FROM {arch} WHERE name = ?", (name,)
41-
).fetchall()
42-
if len(res):
43-
local_version = res[0][0]
44-
local_url = res[0][1]
45-
if local_version != version:
46-
print(f"Update: {name}:{arch} ({local_version} -> {version})")
47-
download(url)
48-
# wirte to db
49-
cur.execute(
50-
f"UPDATE {arch} SET version = ?, url = ? WHERE name = ?",
33+
db_path = os.path.join("data", f"{BASE_DIR}.db")
34+
# get local version
35+
with version_lock, sqlite3.connect(db_path) as conn:
36+
res = conn.execute(
37+
f"SELECT version, url FROM '{arch}' WHERE name = ?", (name,)
38+
).fetchone()
39+
if res:
40+
local_version, local_url = res
41+
if local_version != version:
42+
print(f"Update: {name}:{arch} ({local_version} -> {version})")
43+
download(url)
44+
# update database
45+
with version_lock, sqlite3.connect(db_path) as conn:
46+
conn.execute(
47+
f"UPDATE '{arch}' SET version = ?, url = ? WHERE name = ?",
5148
(version, url, name),
5249
)
53-
# remove old version
54-
if local_url != url: # 针对固定下载链接
55-
old_file_path = os.path.join(base_dir, local_url.split("?")[0])
56-
if os.path.exists(old_file_path):
57-
os.remove(old_file_path)
58-
else:
59-
print(f"AddNew: {name}:{arch} ({version})")
60-
download(url)
61-
# wirte to db
62-
cur.execute(
63-
f"INSERT INTO {arch}(name, version, url) VALUES (?, ?, ?)",
50+
conn.commit()
51+
# remove old version
52+
if local_url != url: # 防止固定下载链接
53+
old_file_path = os.path.join(BASE_DIR, local_url.split("?")[0])
54+
if os.path.exists(old_file_path):
55+
os.remove(old_file_path)
56+
else:
57+
print(f"AddNew: {name}:{arch} ({version})")
58+
download(url)
59+
# update database
60+
with version_lock, sqlite3.connect(db_path) as conn:
61+
conn.execute(
62+
f"INSERT INTO '{arch}'(name, version, url) VALUES (?, ?, ?)",
6463
(name, version, url),
6564
)
66-
conn.commit()
65+
conn.commit()
6766

6867

6968
if __name__ == "__main__":
70-
args = os.sys.argv
71-
if len(args) == 5:
72-
check_download(args[1], args[2], args[3], args[4])
73-
elif len(args) == 4:
74-
check_download(args[1], args[2], args[3], "x86_64")
69+
args = sys.argv
70+
if len(args) in (4, 5):
71+
check_download(*args[1:])
7572
elif len(args) > 1:
7673
logging.error(f"Unknown Args: {args[1:]}")
7774
else:
7875
print(f"Usage: {args[0]} <package_name> <version> <url> [arch]")
7976
print("options:")
80-
print(" arch: x86_64, arm64. default is x86_64")
77+
print(" arch: amd64, arm64, all. default is amd64")

‎data/github.json‎

Lines changed: 58 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,108 +1,108 @@
11
{
22
"clash-verge": {
33
"repo": "clash-verge-rev/clash-verge-rev",
4-
"file_list": [
5-
"Clash.Verge_{stripped_version}_amd64.deb",
6-
"Clash.Verge_{stripped_version}_arm64.deb"
7-
]
4+
"file_list": {
5+
"amd64": "Clash.Verge_{version}_amd64.deb",
6+
"arm64": "Clash.Verge_{version}_arm64.deb"
7+
}
88
},
99
"mihomo": {
1010
"repo": "MetaCubeX/mihomo",
11-
"file_list": [
12-
"mihomo-linux-amd64-compatible-{version_tag}.deb",
13-
"mihomo-linux-arm64-{version_tag}.deb"
14-
]
11+
"file_list": {
12+
"amd64": "mihomo-linux-amd64-compatible-{releases_tag}.deb",
13+
"arm64": "mihomo-linux-arm64-{releases_tag}.deb"
14+
}
1515
},
1616
"flclash": {
1717
"repo": "chen08209/FlClash",
18-
"file_list": [
19-
"FlClash-{stripped_version}-linux-amd64.deb"
20-
]
18+
"file_list": {
19+
"amd64": "FlClash-{version}-linux-amd64.deb"
20+
}
2121
},
2222
"hugo": {
2323
"repo": "gohugoio/hugo",
24-
"file_list": [
25-
"hugo_extended_{stripped_version}_linux-amd64.deb",
26-
"hugo_extended_{stripped_version}_linux-arm64.deb"
27-
]
24+
"file_list": {
25+
"amd64": "hugo_extended_{version}_linux-amd64.deb",
26+
"arm64": "hugo_extended_{version}_linux-arm64.deb"
27+
}
2828
},
2929
"rustdesk": {
3030
"repo": "rustdesk/rustdesk",
31-
"file_list": [
32-
"rustdesk-{version_tag}-x86_64.deb",
33-
"rustdesk-{version_tag}-aarch64.deb"
34-
]
31+
"file_list": {
32+
"amd64": "rustdesk-{releases_tag}-x86_64.deb",
33+
"arm64": "rustdesk-{releases_tag}-aarch64.deb"
34+
}
3535
},
3636
"obsidian": {
3737
"repo": "obsidianmd/obsidian-releases",
38-
"file_list": [
39-
"obsidian_{stripped_version}_amd64.deb"
40-
]
38+
"file_list": {
39+
"amd64": "obsidian_{version}_amd64.deb"
40+
}
4141
},
4242
"tabby": {
4343
"repo": "Eugeny/tabby",
44-
"file_list": [
45-
"tabby-{stripped_version}-linux-x64.deb",
46-
"tabby-{stripped_version}-linux-arm64.deb"
47-
]
44+
"file_list": {
45+
"amd64": "tabby-{version}-linux-x64.deb",
46+
"arm64": "tabby-{version}-linux-arm64.deb"
47+
}
4848
},
4949
"pandoc": {
5050
"repo": "jgm/pandoc",
51-
"file_list": [
52-
"pandoc-{version_tag}-1-amd64.deb",
53-
"pandoc-{version_tag}-1-arm64.deb"
54-
]
51+
"file_list": {
52+
"amd64": "pandoc-{releases_tag}-1-amd64.deb",
53+
"arm64": "pandoc-{releases_tag}-1-arm64.deb"
54+
}
5555
},
5656
"localsend": {
5757
"repo": "localsend/localsend",
58-
"file_list": [
59-
"LocalSend-{stripped_version}-linux-x86-64.deb",
60-
"LocalSend-{stripped_version}-linux-arm-64.deb"
61-
]
58+
"file_list": {
59+
"amd64": "LocalSend-{version}-linux-x86-64.deb",
60+
"arm64": "LocalSend-{version}-linux-arm-64.deb"
61+
}
6262
},
6363
"motrix": {
6464
"repo": "agalwood/Motrix",
65-
"file_list": [
66-
"Motrix_{stripped_version}_amd64.deb",
67-
"Motrix_{stripped_version}_arm64.deb"
68-
]
65+
"file_list": {
66+
"amd64": "Motrix_{version}_amd64.deb",
67+
"arm64": "Motrix_{version}_arm64.deb"
68+
}
6969
},
7070
"peazip": {
7171
"repo": "peazip/PeaZip",
72-
"file_list": [
73-
"peazip_{version_tag}.LINUX.GTK2-1_amd64.deb"
74-
]
72+
"file_list": {
73+
"amd64": "peazip_{releases_tag}.LINUX.GTK2-1_amd64.deb"
74+
}
7575
},
7676
"neovim": {
7777
"repo": "neovim/neovim-releases",
78-
"file_list": [
79-
"nvim-linux64.deb"
80-
]
78+
"file_list": {
79+
"amd64": "nvim-linux64.deb"
80+
}
8181
},
8282
"hiddify": {
8383
"repo": "hiddify/hiddify-app",
84-
"file_list": [
85-
"Hiddify-Debian-x64.deb"
86-
]
84+
"file_list": {
85+
"amd64": "Hiddify-Debian-x64.deb"
86+
}
8787
},
8888
"cloudflared": {
8989
"repo": "cloudflare/cloudflared",
90-
"file_list": [
91-
"cloudflared-linux-amd64.deb",
92-
"cloudflared-linux-arm64.deb"
93-
]
90+
"file_list": {
91+
"amd64": "cloudflared-linux-amd64.deb",
92+
"arm64": "cloudflared-linux-arm64.deb"
93+
}
9494
},
9595
"caddy": {
9696
"repo": "caddyserver/caddy",
97-
"file_list": [
98-
"caddy_{stripped_version}_linux_amd64.deb",
99-
"caddy_{stripped_version}_linux_arm64.deb"
100-
]
97+
"file_list": {
98+
"amd64": "caddy_{version}_linux_amd64.deb",
99+
"arm64": "caddy_{version}_linux_arm64.deb"
100+
}
101101
},
102102
"foliate": {
103103
"repo": "johnfactotum/foliate",
104-
"file_list": [
105-
"foliate_{version_tag}_all.deb"
106-
]
104+
"file_list": {
105+
"all": "foliate_{releases_tag}_all.deb"
106+
}
107107
}
108108
}

0 commit comments

Comments
 (0)