Skip to content

Commit 847a928

Browse files
committed
feat: add upload script
1 parent 3ea80dc commit 847a928

1 file changed

Lines changed: 58 additions & 0 deletions

File tree

upload.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# /// script
2+
# requires-python = ">=3.11"
3+
# dependencies = [
4+
# "httpx",
5+
# "tqdm",
6+
# ]
7+
# ///
8+
9+
import asyncio
10+
import os
11+
from pathlib import Path
12+
13+
import httpx
14+
from tqdm.asyncio import tqdm as async_tqdm
15+
16+
STORAGE_ZONE = os.getenv("BUNNY_STORAGE_ZONE")
17+
ACCESS_KEY = os.getenv("BUNNY_API_KEY")
18+
MAX_CONCURRENT = 100
19+
20+
21+
def get_files():
22+
files = []
23+
for f in Path("benchmarks").rglob("*"):
24+
if f.is_file() and not f.name.startswith("."):
25+
rel_path = str(f.relative_to("benchmarks")).replace("\\", "/")
26+
files.append((f, rel_path))
27+
return files
28+
29+
30+
async def upload_file(client, sem, local_path, remote_path):
31+
async with sem:
32+
url = f"https://storage.bunnycdn.com/{STORAGE_ZONE}/benchmarks/{remote_path}"
33+
headers = {"AccessKey": ACCESS_KEY, "Content-Type": "application/octet-stream"}
34+
35+
with open(local_path, "rb") as f:
36+
content = f.read()
37+
38+
response = await client.put(url, content=content, headers=headers)
39+
return response.status_code == 201
40+
41+
42+
async def main():
43+
files = get_files()
44+
total = len(files)
45+
print(f"Uploading {total} files...")
46+
47+
sem = asyncio.Semaphore(MAX_CONCURRENT)
48+
49+
async with httpx.AsyncClient(timeout=60.0) as client:
50+
tasks = [upload_file(client, sem, local, remote) for local, remote in files]
51+
results = await async_tqdm.gather(*tasks, desc="Uploading", unit="file")
52+
53+
success = sum(results)
54+
print(f"Done: {success}/{total} uploaded")
55+
56+
57+
if __name__ == "__main__":
58+
asyncio.run(main())

0 commit comments

Comments
 (0)