|
| 1 | +# /// script |
| 2 | +# requires-python = ">=3.11" |
| 3 | +# dependencies = [ |
| 4 | +# "httpx", |
| 5 | +# "tqdm", |
| 6 | +# ] |
| 7 | +# /// |
| 8 | + |
| 9 | +import asyncio |
| 10 | +import os |
| 11 | +from pathlib import Path |
| 12 | + |
| 13 | +import httpx |
| 14 | +from tqdm.asyncio import tqdm as async_tqdm |
| 15 | + |
| 16 | +STORAGE_ZONE = os.getenv("BUNNY_STORAGE_ZONE") |
| 17 | +ACCESS_KEY = os.getenv("BUNNY_API_KEY") |
| 18 | +MAX_CONCURRENT = 100 |
| 19 | + |
| 20 | + |
| 21 | +def get_files(): |
| 22 | + files = [] |
| 23 | + for f in Path("benchmarks").rglob("*"): |
| 24 | + if f.is_file() and not f.name.startswith("."): |
| 25 | + rel_path = str(f.relative_to("benchmarks")).replace("\\", "/") |
| 26 | + files.append((f, rel_path)) |
| 27 | + return files |
| 28 | + |
| 29 | + |
| 30 | +async def upload_file(client, sem, local_path, remote_path): |
| 31 | + async with sem: |
| 32 | + url = f"https://storage.bunnycdn.com/{STORAGE_ZONE}/benchmarks/{remote_path}" |
| 33 | + headers = {"AccessKey": ACCESS_KEY, "Content-Type": "application/octet-stream"} |
| 34 | + |
| 35 | + with open(local_path, "rb") as f: |
| 36 | + content = f.read() |
| 37 | + |
| 38 | + response = await client.put(url, content=content, headers=headers) |
| 39 | + return response.status_code == 201 |
| 40 | + |
| 41 | + |
| 42 | +async def main(): |
| 43 | + files = get_files() |
| 44 | + total = len(files) |
| 45 | + print(f"Uploading {total} files...") |
| 46 | + |
| 47 | + sem = asyncio.Semaphore(MAX_CONCURRENT) |
| 48 | + |
| 49 | + async with httpx.AsyncClient(timeout=60.0) as client: |
| 50 | + tasks = [upload_file(client, sem, local, remote) for local, remote in files] |
| 51 | + results = await async_tqdm.gather(*tasks, desc="Uploading", unit="file") |
| 52 | + |
| 53 | + success = sum(results) |
| 54 | + print(f"Done: {success}/{total} uploaded") |
| 55 | + |
| 56 | + |
| 57 | +if __name__ == "__main__": |
| 58 | + asyncio.run(main()) |
0 commit comments