chore: upgrade to reqwest 0.12.4 and rustls 0.22 (denoland#24056)

This commit updates Deno to use `reqwest` at 0.12.4
and `rustls` at 0.22. Other related crates were updated
as well to match versions accepted by `reqwest` and `rustls`.

Note: we are not using the latest available `rustls` yet,
but this upgrade was non-trivial already, so a bump to
0.23 for `rustls` will be done in a separate commit.

Closes denoland#23370

---------

Signed-off-by: Ryan Dahl <ry@tinyclouds.org>
Signed-off-by: Bartek Iwańczuk <biwanczuk@gmail.com>
Co-authored-by: Ryan Dahl <ry@tinyclouds.org>
Co-authored-by: Divy Srivastava <dj.srivastava23@gmail.com>

Author: 3 people

Cargo.lock

@@ -55,10 +55,10 @@ deno_terminal = "0.1.1"
 napi_sym = { version = "0.86.0", path = "./cli/napi/sym" }
 test_util = { package = "test_server", path = "./tests/util/server" }
 
-denokv_proto = "0.7.0"
-denokv_remote = "0.7.0"
+denokv_proto = "0.8.1"
+denokv_remote = "0.8.1"
 # denokv_sqlite brings in bundled sqlite if we don't disable the default features
-denokv_sqlite = { default-features = false, version = "0.7.0" }
+denokv_sqlite = { default-features = false, version = "0.8.1" }
 
 # exts
 deno_broadcast_channel = { version = "0.150.0", path = "./ext/broadcast_channel" }
@@ -117,8 +117,8 @@ http = "1.0"
 http-body-util = "0.1"
 http_v02 = { package = "http", version = "0.2.9" }
 httparse = "1.8.0"
-hyper = { version = "=1.1.0", features = ["full"] }
-hyper-util = { version = "=0.1.2", features = ["tokio", "server", "server-auto"] }
+hyper = { version = "=1.3.1", features = ["full"] }
+hyper-util = { version = "=0.1.5", features = ["tokio", "server", "server-auto"] }
 hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] }
 indexmap = { version = "2", features = ["serde"] }
 jsonc-parser = { version = "=0.23.0", features = ["serde"] }
@@ -145,14 +145,13 @@ prost = "0.11"
 prost-build = "0.11"
 rand = "=0.8.5"
 regex = "^1.7.0"
-reqwest = { version = "=0.11.20", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json"] } # pinned because of https://github.com/seanmonstar/reqwest/pull/1955
+reqwest = { version = "=0.12.4", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json", "http2"] } # pinned because of https://github.com/seanmonstar/reqwest/pull/1955
 ring = "^0.17.0"
 rusqlite = { version = "=0.29.0", features = ["unlock_notify", "bundled"] }
-# pinned because it was causing issues on cargo publish
-rustls = "=0.21.11"
-rustls-pemfile = "1.0.0"
-rustls-tokio-stream = "=0.2.24"
-rustls-webpki = "0.101.4"
+rustls = "0.22.4"
+rustls-pemfile = "2"
+rustls-tokio-stream = "=0.2.23"
+rustls-webpki = "0.102"
 rustyline = "=13.0.0"
 saffron = "=0.1.0"
 scopeguard = "1.2.0"
@@ -178,7 +177,7 @@ twox-hash = "=1.6.3"
 # Upgrading past 2.4.1 may cause WPT failures
 url = { version = "< 2.5.0", features = ["serde", "expose_internals"] }
 uuid = { version = "1.3.0", features = ["v4"] }
-webpki-roots = "0.25.2"
+webpki-roots = "0.26"
 zeromq = { version = "=0.3.4", default-features = false, features = ["tcp-transport", "tokio-runtime"] }
 zstd = "=0.12.4"
 

Cargo.toml

@@ -696,21 +696,13 @@ pub fn get_root_cert_store(
   for store in ca_stores.iter() {
     match store.as_str() {
       "mozilla" => {
-        root_cert_store.add_trust_anchors(
-          webpki_roots::TLS_SERVER_ROOTS.iter().map(|ta| {
-            rustls::OwnedTrustAnchor::from_subject_spki_name_constraints(
-              ta.subject,
-              ta.spki,
-              ta.name_constraints,
-            )
-          }),
-        );
+        root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.to_vec());
       }
       "system" => {
         let roots = load_native_certs().expect("could not load platform certs");
         for root in roots {
           root_cert_store
-            .add(&rustls::Certificate(root.0))
+            .add(rustls::pki_types::CertificateDer::from(root.0))
             .expect("Failed to add platform cert to root cert store");
         }
       }
@@ -734,17 +726,17 @@ pub fn get_root_cert_store(
           RootCertStoreLoadError::CaFileOpenError(err.to_string())
         })?;
         let mut reader = BufReader::new(certfile);
-        rustls_pemfile::certs(&mut reader)
+        rustls_pemfile::certs(&mut reader).collect::<Result<Vec<_>, _>>()
       }
       CaData::Bytes(data) => {
         let mut reader = BufReader::new(Cursor::new(data));
-        rustls_pemfile::certs(&mut reader)
+        rustls_pemfile::certs(&mut reader).collect::<Result<Vec<_>, _>>()
       }
     };
 
     match result {
       Ok(certs) => {
-        root_cert_store.add_parsable_certificates(&certs);
+        root_cert_store.add_parsable_certificates(certs);
       }
       Err(e) => {
         return Err(RootCertStoreLoadError::FailedAddPemFile(e.to_string()));

cli/args/mod.rs

@@ -587,7 +587,7 @@ mod test {
   use std::collections::HashSet;
   use std::hash::RandomState;
 
-  use deno_runtime::deno_tls::RootCertStore;
+  use deno_runtime::deno_tls::rustls::RootCertStore;
 
   use crate::version;
 

cli/http_util.rs

@@ -20,7 +20,7 @@ deno_core.workspace = true
 deno_permissions.workspace = true
 deno_tls.workspace = true
 dyn-clone = "1"
-http_v02.workspace = true
+http.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true

ext/fetch/Cargo.toml

@@ -31,7 +31,7 @@ impl FetchHandler for FsFetchHandler {
       let file = tokio::fs::File::open(path).map_err(|_| ()).await?;
       let stream = ReaderStream::new(file);
       let body = reqwest::Body::wrap_stream(stream);
-      let response = http_v02::Response::builder()
+      let response = http::Response::builder()
         .status(StatusCode::OK)
         .body(body)
         .map_err(|_| ())?

ext/fetch/fs_fetch_handler.rs

@@ -47,8 +47,8 @@ use data_url::DataUrl;
 use deno_tls::TlsKey;
 use deno_tls::TlsKeys;
 use deno_tls::TlsKeysHolder;
-use http_v02::header::CONTENT_LENGTH;
-use http_v02::Uri;
+use http::header::CONTENT_LENGTH;
+use http::Uri;
 use reqwest::header::HeaderMap;
 use reqwest::header::HeaderName;
 use reqwest::header::HeaderValue;
@@ -449,12 +449,9 @@ where
         .decode_to_vec()
         .map_err(|e| type_error(format!("{e:?}")))?;
 
-      let response = http_v02::Response::builder()
-        .status(http_v02::StatusCode::OK)
-        .header(
-          http_v02::header::CONTENT_TYPE,
-          data_url.mime_type().to_string(),
-        )
+      let response = http::Response::builder()
+        .status(http::StatusCode::OK)
+        .header(http::header::CONTENT_TYPE, data_url.mime_type().to_string())
         .body(reqwest::Body::from(body))?;
 
       let fut = async move { Ok(Ok(Response::from(response))) };

ext/fetch/lib.rs

@@ -17,6 +17,7 @@ path = "lib.rs"
 anyhow.workspace = true
 async-trait.workspace = true
 base64.workspace = true
+bytes.workspace = true
 chrono = { workspace = true, features = ["now"] }
 deno_core.workspace = true
 deno_fetch.workspace = true
@@ -27,6 +28,7 @@ denokv_proto.workspace = true
 denokv_remote.workspace = true
 denokv_sqlite.workspace = true
 faster-hex.workspace = true
+http.workspace = true
 log.workspace = true
 num-bigint.workspace = true
 prost.workspace = true

ext/kv/Cargo.toml

@@ -8,17 +8,23 @@ use std::sync::Arc;
 use crate::DatabaseHandler;
 use anyhow::Context;
 use async_trait::async_trait;
+use bytes::Bytes;
 use deno_core::error::type_error;
 use deno_core::error::AnyError;
+use deno_core::futures::Stream;
+use deno_core::futures::TryStreamExt as _;
 use deno_core::OpState;
 use deno_fetch::create_http_client;
+use deno_fetch::reqwest;
 use deno_fetch::CreateHttpClientOptions;
 use deno_tls::rustls::RootCertStore;
 use deno_tls::Proxy;
 use deno_tls::RootCertStoreProvider;
 use deno_tls::TlsKeys;
 use denokv_remote::MetadataEndpoint;
 use denokv_remote::Remote;
+use denokv_remote::RemoteResponse;
+use denokv_remote::RemoteTransport;
 use url::Url;
 
 #[derive(Clone)]
@@ -102,11 +108,44 @@ impl<P: RemoteDbHandlerPermissions + 'static> denokv_remote::RemotePermissions
   }
 }
 
+#[derive(Clone)]
+pub struct ReqwestClient(reqwest::Client);
+pub struct ReqwestResponse(reqwest::Response);
+
+impl RemoteTransport for ReqwestClient {
+  type Response = ReqwestResponse;
+  async fn post(
+    &self,
+    url: Url,
+    headers: http::HeaderMap,
+    body: Bytes,
+  ) -> Result<(Url, http::StatusCode, Self::Response), anyhow::Error> {
+    let res = self.0.post(url).headers(headers).body(body).send().await?;
+    let url = res.url().clone();
+    let status = res.status();
+    Ok((url, status, ReqwestResponse(res)))
+  }
+}
+
+impl RemoteResponse for ReqwestResponse {
+  async fn bytes(self) -> Result<Bytes, anyhow::Error> {
+    Ok(self.0.bytes().await?)
+  }
+  fn stream(
+    self,
+  ) -> impl Stream<Item = Result<Bytes, anyhow::Error>> + Send + Sync {
+    self.0.bytes_stream().map_err(|e| e.into())
+  }
+  async fn text(self) -> Result<String, anyhow::Error> {
+    Ok(self.0.text().await?)
+  }
+}
+
 #[async_trait(?Send)]
 impl<P: RemoteDbHandlerPermissions + 'static> DatabaseHandler
   for RemoteDbHandler<P>
 {
-  type DB = Remote<PermissionChecker<P>>;
+  type DB = Remote<PermissionChecker<P>, ReqwestClient>;
 
   async fn open(
     &self,
@@ -162,13 +201,14 @@ impl<P: RemoteDbHandlerPermissions + 'static> DatabaseHandler
         http2: true,
       },
     )?;
+    let reqwest_client = ReqwestClient(client);
 
     let permissions = PermissionChecker {
       state: state.clone(),
       _permissions: PhantomData,
     };
 
-    let remote = Remote::new(client, permissions, metadata_endpoint);
+    let remote = Remote::new(reqwest_client, permissions, metadata_endpoint);
 
     Ok(remote)
   }

ext/kv/remote.rs

@@ -31,11 +31,11 @@ use deno_tls::create_client_config;
 use deno_tls::load_certs;
 use deno_tls::load_private_keys;
 use deno_tls::new_resolver;
-use deno_tls::rustls::Certificate;
+use deno_tls::rustls::pki_types::ServerName;
 use deno_tls::rustls::ClientConnection;
-use deno_tls::rustls::PrivateKey;
 use deno_tls::rustls::ServerConfig;
-use deno_tls::rustls::ServerName;
+use deno_tls::webpki::types::CertificateDer;
+use deno_tls::webpki::types::PrivateKeyDer;
 use deno_tls::ServerConfigProvider;
 use deno_tls::SocketUse;
 use deno_tls::TlsKey;
@@ -48,7 +48,6 @@ use serde::Deserialize;
 use std::borrow::Cow;
 use std::cell::RefCell;
 use std::convert::From;
-use std::convert::TryFrom;
 use std::fs::File;
 use std::io::BufReader;
 use std::io::ErrorKind;
@@ -304,14 +303,14 @@ where
 {
   let rid = args.rid;
   let hostname = match &*args.hostname {
-    "" => "localhost",
-    n => n,
+    "" => "localhost".to_string(),
+    n => n.to_string(),
   };
 
   {
     let mut s = state.borrow_mut();
     let permissions = s.borrow_mut::<NP>();
-    permissions.check_net(&(hostname, Some(0)), "Deno.startTls()")?;
+    permissions.check_net(&(&hostname, Some(0)), "Deno.startTls()")?;
   }
 
   let ca_certs = args
@@ -320,8 +319,8 @@ where
     .map(|s| s.into_bytes())
     .collect::<Vec<_>>();
 
-  let hostname_dns =
-    ServerName::try_from(hostname).map_err(|_| invalid_hostname(hostname))?;
+  let hostname_dns = ServerName::try_from(hostname.to_string())
+    .map_err(|_| invalid_hostname(&hostname))?;
 
   let unsafely_ignore_certificate_errors = state
     .borrow()
@@ -422,9 +421,9 @@ where
     .borrow::<DefaultTlsOptions>()
     .root_cert_store()?;
   let hostname_dns = if let Some(server_name) = args.server_name {
-    ServerName::try_from(server_name.as_str())
+    ServerName::try_from(server_name)
   } else {
-    ServerName::try_from(&*addr.hostname)
+    ServerName::try_from(addr.hostname.clone())
   }
   .map_err(|_| invalid_hostname(&addr.hostname))?;
   let connect_addr = resolve_addr(&addr.hostname, addr.port)
@@ -466,15 +465,17 @@ where
   Ok((rid, IpAddr::from(local_addr), IpAddr::from(remote_addr)))
 }
 
-fn load_certs_from_file(path: &str) -> Result<Vec<Certificate>, AnyError> {
+fn load_certs_from_file(
+  path: &str,
+) -> Result<Vec<CertificateDer<'static>>, AnyError> {
   let cert_file = File::open(path)?;
   let reader = &mut BufReader::new(cert_file);
   load_certs(reader)
 }
 
 fn load_private_keys_from_file(
   path: &str,
-) -> Result<Vec<PrivateKey>, AnyError> {
+) -> Result<Vec<PrivateKeyDer<'static>>, AnyError> {
   let key_bytes = std::fs::read(path)?;
   load_private_keys(&key_bytes)
 }
@@ -523,7 +524,6 @@ where
     TlsKeys::Null => Err(anyhow!("Deno.listenTls requires a key")),
     TlsKeys::Static(TlsKey(cert, key)) => {
       let mut tls_config = ServerConfig::builder()
-        .with_safe_defaults()
         .with_no_client_auth()
         .with_single_cert(cert, key)
         .map_err(|e| anyhow!(e))?;