Read incrementally to enforce max_response_body_size_

Author: stephenberry

include/glaze/net/http_client.hpp

@@ -1710,7 +1710,7 @@ namespace glz
 
             if (is_chunked) {
                // Read chunked transfer-encoded body
-               for (;;) {
+               while (true) {
                   // Read chunk size line (hex digits followed by CRLF)
                   asio::error_code read_ec;
                   std::visit([&](auto& sock) { asio::read_until(*sock, response_buffer, "\r\n", read_ec); },
@@ -1747,7 +1747,7 @@ namespace glz
 
                   if (chunk_size == 0) {
                      // Terminal chunk; skip optional trailers and final CRLF (RFC 7230 §4.1)
-                     for (;;) {
+                     while (true) {
                         std::visit([&](auto& sock) { asio::read_until(*sock, response_buffer, "\r\n", read_ec); },
                                    socket_var);
                         if (read_ec) {
@@ -1795,20 +1795,24 @@ namespace glz
             }
             else if (!has_content_length) {
                // No Content-Length and not chunked: read until connection close (RFC 7230 §3.3.3)
+               // Read incrementally to enforce max_response_body_size_ without unbounded allocation
                asio::error_code read_ec;
-               std::visit(
-                  [&](auto& sock) {
-                     asio::read(*sock, response_buffer, asio::transfer_all(), read_ec);
-                  },
-                  socket_var);
-               if (read_ec && read_ec != asio::error::eof) {
-                  detail::close_socket(socket_var, connection_pool->graceful_ssl_shutdown());
-                  return std::unexpected(read_ec);
+               while (true) {
+                  std::visit(
+                     [&](auto& sock) {
+                        asio::read(*sock, response_buffer, asio::transfer_at_least(1), read_ec);
+                     },
+                     socket_var);
+                  if (read_ec) break;
+                  if (max_response_body_size_ > 0 && response_buffer.size() > max_response_body_size_) {
+                     detail::close_socket(socket_var, connection_pool->graceful_ssl_shutdown());
+                     return std::unexpected(make_error_code(http_client_error::response_too_large));
+                  }
                }
 
-               if (max_response_body_size_ > 0 && response_buffer.size() > max_response_body_size_) {
+               if (read_ec != asio::error::eof) {
                   detail::close_socket(socket_var, connection_pool->graceful_ssl_shutdown());
-                  return std::unexpected(make_error_code(http_client_error::response_too_large));
+                  return std::unexpected(read_ec);
                }
 
                response_body.assign(static_cast<const char*>(response_buffer.data().data()),
@@ -2206,6 +2210,52 @@ namespace glz
             *socket_var);
       }
 
+      // Async EOF-delimited body reading: reads incrementally until connection close,
+      // checking max_response_body_size_ after each read to prevent unbounded allocation.
+      template <typename CompletionHandler>
+      void async_read_eof_body(std::shared_ptr<socket_variant> socket_var, std::shared_ptr<asio::streambuf> buffer,
+                               const url_parts& url, bool use_https, int status_code,
+                               std::unordered_map<std::string, std::string> response_headers,
+                               CompletionHandler&& handler)
+      {
+         std::visit(
+            [&, this](auto& sock) {
+               asio::async_read(
+                  *sock, *buffer, asio::transfer_at_least(1),
+                  [this, socket_var, buffer, url, use_https, status_code,
+                   response_headers = std::move(response_headers),
+                   handler = std::forward<CompletionHandler>(handler)](asio::error_code ec, std::size_t) mutable {
+                     if (ec && ec != asio::error::eof) {
+                        handler(std::unexpected(ec));
+                        return;
+                     }
+
+                     if (max_response_body_size_ > 0 && buffer->size() > max_response_body_size_) {
+                        handler(std::unexpected(make_error_code(http_client_error::response_too_large)));
+                        return;
+                     }
+
+                     if (ec == asio::error::eof) {
+                        // Connection closed — body is complete
+                        std::string body(static_cast<const char*>(buffer->data().data()), buffer->size());
+
+                        response resp;
+                        resp.status_code = status_code;
+                        resp.response_headers = std::move(response_headers);
+                        resp.response_body = std::move(body);
+
+                        handler(std::move(resp));
+                        return;
+                     }
+
+                     // More data available — continue reading
+                     async_read_eof_body(socket_var, buffer, url, use_https, status_code,
+                                         std::move(response_headers), std::move(handler));
+                  });
+            },
+            *socket_var);
+      }
+
       template <typename CompletionHandler>
       void parse_and_read_body(std::shared_ptr<socket_variant> socket_var, std::shared_ptr<asio::streambuf> buffer,
                                size_t header_size, const url_parts& url, bool use_https, CompletionHandler&& handler)
@@ -2284,35 +2334,8 @@ namespace glz
          }
          else if (!has_content_length) {
             // No Content-Length and not chunked: read until connection close (RFC 7230 §3.3.3)
-            std::visit(
-               [&, this](auto& sock) {
-                  asio::async_read(
-                     *sock, *buffer, asio::transfer_all(),
-                     [this, socket_var, buffer, url, use_https, status_code = parsed_status->status_code,
-                      response_headers = std::move(response_headers),
-                      handler = std::forward<CompletionHandler>(handler)](asio::error_code ec, std::size_t) mutable {
-                        if (ec && ec != asio::error::eof) {
-                           handler(std::unexpected(ec));
-                           return;
-                        }
-
-                        if (max_response_body_size_ > 0 && buffer->size() > max_response_body_size_) {
-                           handler(std::unexpected(make_error_code(http_client_error::response_too_large)));
-                           return;
-                        }
-
-                        std::string body(static_cast<const char*>(buffer->data().data()), buffer->size());
-
-                        response resp;
-                        resp.status_code = status_code;
-                        resp.response_headers = std::move(response_headers);
-                        resp.response_body = std::move(body);
-
-                        // Connection is done after EOF-delimited body, don't return to pool
-                        handler(std::move(resp));
-                     });
-               },
-               *socket_var);
+            async_read_eof_body(socket_var, buffer, url, use_https, parsed_status->status_code,
+                                std::move(response_headers), std::forward<CompletionHandler>(handler));
          }
          else {
             // Read the rest of the body based on content-length.

tests/networking_tests/http_client_test/http_client_test.cpp

@@ -1091,18 +1091,9 @@ class eof_delimited_server
    bool start(const std::string& response_body)
    {
       try {
-         for (uint16_t test_port = 18300; test_port < 18400; ++test_port) {
-            try {
-               acceptor_ = std::make_unique<asio::ip::tcp::acceptor>(
-                  io_, asio::ip::tcp::endpoint(asio::ip::make_address("127.0.0.1"), test_port));
-               port_ = test_port;
-               break;
-            }
-            catch (...) {
-               continue;
-            }
-         }
-         if (port_ == 0) return false;
+         acceptor_ = std::make_unique<asio::ip::tcp::acceptor>(
+            io_, asio::ip::tcp::endpoint(asio::ip::make_address("127.0.0.1"), 0));
+         port_ = acceptor_->local_endpoint().port();
 
          server_thread_ = std::thread([this, response_body]() {
             try {
@@ -1136,6 +1127,47 @@ class eof_delimited_server
       }
    }
 
+   bool start_with_reset()
+   {
+      try {
+         acceptor_ = std::make_unique<asio::ip::tcp::acceptor>(
+            io_, asio::ip::tcp::endpoint(asio::ip::make_address("127.0.0.1"), 0));
+         port_ = acceptor_->local_endpoint().port();
+
+         server_thread_ = std::thread([this]() {
+            try {
+               asio::ip::tcp::socket socket(io_);
+               acceptor_->accept(socket);
+
+               asio::streambuf request_buf;
+               asio::read_until(socket, request_buf, "\r\n\r\n");
+
+               // Send headers and partial body, then reset the connection
+               std::string http_response =
+                  "HTTP/1.1 200 OK\r\n"
+                  "Content-Type: application/json\r\n"
+                  "Connection: close\r\n"
+                  "\r\n"
+                  "partial";
+
+               asio::write(socket, asio::buffer(http_response));
+
+               // Force a RST by setting linger to 0 and closing
+               asio::socket_base::linger option(true, 0);
+               socket.set_option(option);
+               socket.close();
+            }
+            catch (...) {
+            }
+         });
+
+         return true;
+      }
+      catch (...) {
+         return false;
+      }
+   }
+
    void stop()
    {
       if (acceptor_) acceptor_->close();
@@ -1197,6 +1229,64 @@ suite eof_delimited_tests = [] {
 
       server.stop();
    };
+
+   "sync_eof_delimited_max_body_size"_test = [] {
+      // Body larger than the limit
+      const std::string large_body(1024, 'X');
+      eof_delimited_server server;
+      expect(server.start(large_body)) << "EOF-delimited server should start";
+
+      glz::http_client client;
+      client.max_response_body_size(100); // limit well below body size
+      auto result = client.get(server.base_url() + "/api/0/config");
+
+      expect(!result.has_value()) << "GET should fail when body exceeds max size";
+
+      server.stop();
+   };
+
+   "async_eof_delimited_max_body_size"_test = [] {
+      const std::string large_body(1024, 'X');
+      eof_delimited_server server;
+      expect(server.start(large_body)) << "EOF-delimited server should start";
+
+      glz::http_client client;
+      client.max_response_body_size(100);
+
+      std::promise<glz::expected<glz::response, std::error_code>> promise;
+      auto future = promise.get_future();
+
+      client.get_async(server.base_url() + "/api/0/config", {},
+                       [&](glz::expected<glz::response, std::error_code> result) { promise.set_value(std::move(result)); });
+
+      auto status = future.wait_for(std::chrono::seconds(5));
+      expect(status == std::future_status::ready) << "Async GET should complete";
+
+      if (status == std::future_status::ready) {
+         auto result = future.get();
+         expect(!result.has_value()) << "Async GET should fail when body exceeds max size";
+      }
+
+      server.stop();
+   };
+
+   "sync_eof_delimited_connection_reset"_test = [] {
+      // Server that resets the connection after sending partial data
+      eof_delimited_server server;
+      expect(server.start_with_reset()) << "Server should start";
+
+      glz::http_client client;
+      auto result = client.get(server.base_url() + "/api/0/config");
+
+      // RST behavior is platform-dependent:
+      // - On most platforms, the RST is received as a connection error (result is an error)
+      // - On some platforms, data may arrive before the RST, resulting in a partial body with EOF
+      if (result.has_value()) {
+         expect(result->response_body == "partial") << "If RST is seen as EOF, partial body should be returned";
+      }
+
+      server.stop();
+   };
 };
 
 int main()