Skip to content

Commit a7ad3bc

Browse files
MrcSnmMrcSnm
committed
Update:
- Windows is now using winhttp for downloading thus not requiring libcurl. - Updated sdlite dependency - Added missing help info for redub use and install
1 parent f9e9004 commit a7ad3bc

File tree

16 files changed

+241
-30
lines changed

16 files changed

+241
-30
lines changed

‎d_downloader/.gitignore‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
*.exe
2+
*.pdb
3+
*.o
4+
*.obj
5+
*.lst
6+
*.lnk
7+
.history
8+
.dub
9+
docs.json
10+
__dummy.html
11+
docs/
12+
/d_downloader
13+
d_downloader.so
14+
d_downloader.dylib
15+
d_downloader.dll
16+
d_downloader.a
17+
d_downloader.lib
18+
d_downloader-test-*

‎d_downloader/dub.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"authors": [
3+
"Marcelo"
4+
],
5+
"configurations": [
6+
{
7+
"name": "winhttp",
8+
"platforms": ["windows"],
9+
"libs": ["winhttp"]
10+
},
11+
{
12+
"name": "curl"
13+
}
14+
],
15+
"description": "A downloader library which avoids using libcurl so Windows do not needs to link, while use libcurls on other platforms",
16+
"license": "proprietary",
17+
"name": "d_downloader"
18+
}

‎d_downloader/source/d_downloader/curl.d‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
module d_downloader.curl;
2+
version(Posix):
3+
import std.net.curl;
4+
ubyte[] downloadOnBuffer(string file)
5+
{
6+
HTTP http = HTTP(file);
7+
ubyte[] temp;
8+
http.onReceive = (ubyte[] data)
9+
{
10+
temp~= data;
11+
return data.length;
12+
};
13+
http.perform();
14+
return temp;
15+
}
16+
alias downloadToFile = std.net.curl;

‎d_downloader/source/d_downloader/package.d‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module d_downloader;
2+
version(Windows)
3+
public import d_downloader.windows;
4+
else
5+
public import d_downloader.curl;

‎d_downloader/source/d_downloader/windows.d‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
module d_downloader.windows;
2+
version(Windows):
3+
import core.sys.windows.winhttp;
4+
5+
bool download(string link, scope ubyte[] delegate(size_t incomingBytes) bufferSink, scope void delegate(ubyte[] buffer) onDataReceived = null)
6+
{
7+
import std.utf:toUTF16z;
8+
import core.sys.windows.windef;
9+
HINTERNET hSession = WinHttpOpen("D-Downloader/1.0",
10+
WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
11+
WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0);
12+
13+
if (!hSession)
14+
return false;
15+
scope(exit)
16+
WinHttpCloseHandle(hSession);
17+
18+
NetURL url = NetURL(link);
19+
20+
uint[1] protocols = [WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2];
21+
WinHttpSetOption(hSession, WINHTTP_OPTION_SECURE_PROTOCOLS, protocols.ptr, protocols.length);
22+
HINTERNET hConnect = WinHttpConnect(hSession, url.domain.toUTF16z,
23+
INTERNET_DEFAULT_HTTPS_PORT, 0);
24+
scope(exit)
25+
WinHttpCloseHandle(hConnect);
26+
27+
HINTERNET hRequest = WinHttpOpenRequest(hConnect, "GET", url.object.toUTF16z,
28+
null, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES,
29+
WINHTTP_FLAG_SECURE);
30+
31+
if(hRequest is null)
32+
{
33+
import core.sys.windows.winbase;
34+
DWORD code = GetLastError();
35+
string msg;
36+
switch(code)
37+
{
38+
case ERROR_WINHTTP_INCORRECT_HANDLE_TYPE:
39+
msg = `The type of handle supplied is incorrect for this operation.`; break;
40+
case ERROR_WINHTTP_INTERNAL_ERROR:
41+
msg = `An internal error has occurred.`; break;
42+
case ERROR_WINHTTP_INVALID_URL:
43+
msg = `The URL is invalid.`; break;
44+
case ERROR_WINHTTP_OPERATION_CANCELLED:
45+
msg = `The operation was canceled, usually because the handle on which the request was operating was closed before the operation completed.`; break;
46+
case ERROR_WINHTTP_UNRECOGNIZED_SCHEME:
47+
msg = `The URL specified a scheme other than "http:" or "https:".`; break;
48+
case ERROR_NOT_ENOUGH_MEMORY:
49+
msg = `Not enough memory was available to complete the requested operation. (Windows error code)`; break;
50+
default:
51+
wchar* buffer;
52+
DWORD msgLength = FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM |
53+
FORMAT_MESSAGE_IGNORE_INSERTS |
54+
FORMAT_MESSAGE_ALLOCATE_BUFFER, null, code, LANG_NEUTRAL, cast(wchar*)&buffer, 0, null);
55+
if(msgLength)
56+
{
57+
import std.conv:to;
58+
msg = buffer[0..msgLength].to!string;
59+
LocalFree(buffer);
60+
}
61+
break;
62+
}
63+
throw new Exception(msg);
64+
}
65+
66+
scope(exit)
67+
WinHttpCloseHandle(hRequest);
68+
69+
if (WinHttpSendRequest(hRequest, WINHTTP_NO_ADDITIONAL_HEADERS, 0,
70+
WINHTTP_NO_REQUEST_DATA, 0, 0, 0)
71+
&& WinHttpReceiveResponse(hRequest, null))
72+
{
73+
DWORD remainingDataSize = 0;
74+
75+
do {
76+
if (!WinHttpQueryDataAvailable(hRequest, &remainingDataSize))
77+
break;
78+
if (!remainingDataSize)
79+
break;
80+
81+
DWORD dwRead = 0;
82+
ubyte[] buffer = bufferSink(remainingDataSize);
83+
assert(buffer.length < DWORD.max, "Buffer sink length is too big.");
84+
WinHttpReadData(hRequest, buffer.ptr, cast(DWORD)buffer.length, &dwRead);
85+
if(onDataReceived)
86+
onDataReceived(buffer);
87+
} while (remainingDataSize > 0);
88+
}
89+
90+
return true;
91+
}
92+
93+
ubyte[] downloadOnBuffer(string url)
94+
{
95+
ubyte[] buffer;
96+
download(url, (size_t incomingBytes)
97+
{
98+
size_t bufferTail = buffer.length;
99+
buffer.length+= incomingBytes;
100+
return buffer[bufferTail..$];
101+
});
102+
return buffer;
103+
}
104+
105+
void downloadOnFile(string url, string targetFile)
106+
{
107+
import std.stdio;
108+
import std.file;
109+
ubyte[4096] buffer;
110+
string newFile = targetFile;
111+
if(exists(targetFile))
112+
newFile~= ".temp";
113+
114+
File f = File(newFile, "wb");
115+
download(url, (size_t incomingBytes)
116+
{
117+
return incomingBytes > buffer.length ? buffer[0..$] : buffer[0..incomingBytes];
118+
}, (ubyte[] b)
119+
{
120+
f.rawWrite(b);
121+
});
122+
123+
f.close();
124+
if(newFile != targetFile)
125+
std.file.rename(newFile, targetFile);
126+
}
127+
128+
private struct NetURL
129+
{
130+
string protocol;
131+
string domain;
132+
string object;
133+
134+
this(string url)
135+
{
136+
import std.algorithm.searching;
137+
auto protocolIdx = countUntil(url, "://");
138+
if(protocolIdx != -1)
139+
{
140+
protocol = url[0..protocolIdx+3];
141+
url = url[protocolIdx+3..$];
142+
}
143+
auto objectIdx = countUntil(url, "/");
144+
if(objectIdx == -1)
145+
{
146+
object = "/";
147+
domain = url;
148+
}
149+
else
150+
{
151+
object = url[objectIdx..$];
152+
domain = url[0..objectIdx];
153+
}
154+
155+
}
156+
}

‎dub.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@
33
"description": "Dub Based Build System, with parallelization per packages and easier to contribute",
44
"authors": ["Hipreme"],
55
"targetPath": "build",
6-
"stringImportPaths": [
7-
"source"
8-
],
6+
"stringImportPaths": ["source"],
97
"configurations": [
108
{
119
"name": "cli",
@@ -42,6 +40,7 @@
4240
"d_dependencies": {"path": "d_dependencies"},
4341
"dub_sdl_to_json": {"path": "dub_sdl_to_json"},
4442
"package_suppliers": {"path": "package_suppliers"},
43+
"d_downloader": {"path": "d_downloader"},
4544
"xxhash3": "~>0.0.5"
4645
}
4746
}

‎dub.selections.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
"arsd-official": "12.1.0",
88
"colorize": {"path": "colorize"},
99
"d_dependencies": {"path": "d_dependencies"},
10+
"d_downloader": {"path": "d_downloader"},
1011
"dub_sdl_to_json": {"path": "dub_sdl_to_json"},
11-
"sdlite": "1.3.3",
12+
"sdlite": "1.3.4",
1213
"taggedalgebraic": "1.0.1",
1314
"fswatch": "0.6.1",
1415
"package_suppliers": {"path": "package_suppliers"},

‎package_suppliers/dub.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
],
55
"dependencies": {
66
"hipjson": {"path": "../hipjson"},
7-
"semver": {"path": "../semver"}
7+
"semver": {"path": "../semver"},
8+
"d_downloader": {"path": "../d_downloader"}
89
},
910
"copyright": "Copyright © 2024, Marcelo",
1011
"description": "A minimal D application.",

‎package_suppliers/source/redub/libs/package_suppliers/dub_registry.d‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ class RegistryPackageSupplier
7777

7878
void loadPackagesMetadata(string[] packages)
7979
{
80-
import std.net.curl;
80+
import d_downloader;
8181
static string getMetadataUrl(string registryUrl, string[] packages)
8282
{
8383
import std.string:join;
@@ -86,7 +86,7 @@ class RegistryPackageSupplier
8686
encodeComponent(`["`~packageName~`"]`)~
8787
"&include_dependencies=true&minimize=true";
8888
}
89-
string data = cast(string)downloadFile(getMetadataUrl(registryUrl, packages));
89+
string data = cast(string)downloadOnBuffer(getMetadataUrl(registryUrl, packages));
9090
JSONValue parsed = parseJSON(data);
9191

9292
synchronized(metadataMutex.writer)

‎package_suppliers/source/redub/libs/package_suppliers/utils.d‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,6 @@ string getFirstFileInDirectory(string inputDir)
1717
return entries.front;
1818
}
1919

20-
ubyte[] downloadFile(string file)
21-
{
22-
import std.net.curl;
23-
HTTP http = HTTP(file);
24-
ubyte[] temp;
25-
http.onReceive = (ubyte[] data)
26-
{
27-
temp~= data;
28-
return data.length;
29-
};
30-
http.perform();
31-
return temp;
32-
}
33-
3420
bool extractZipToFolder(ubyte[] data, string outputDirectory)
3521
{
3622
import std.file;

0 commit comments

Comments
 (0)
⚡