Skip to content

Commit 26314e7

Browse files
committed
imp(API): faster null-value removal
Previously reserialization of token streams with removed null values was performed on a byte-per-byte basis, which was quite inefficient to say the least. Now it uses `io::copy` to copy in chunks of 65kb, which makes out our throughput and should deliver about 150MB/s at least.
1 parent 3efa4f2 commit 26314e7

1 file changed

Lines changed: 12 additions & 7 deletions

File tree

src/mako/api/lib/mbuild.mako

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -483,6 +483,9 @@ match result {
483483
% if URL_ENCODE in special_cases:
484484
use url::percent_encoding::{percent_encode, FORM_URLENCODED_ENCODE_SET};
485485
% endif
486+
% if request_value:
487+
use json_tools::{TokenReader, Lexer, BufferType, TokenType, FilterTypedKeyValuePairs};
488+
% endif
486489
use std::io::{Read, Seek};
487490
use hyper::header::{ContentType, ContentLength, Authorization, UserAgent, Location};
488491
let mut dd = DefaultDelegate;
@@ -666,14 +669,16 @@ else {
666669
let mut request_value_reader =
667670
{
668671
let json_cache = json::to_string(&self.${property(REQUEST_VALUE_PROPERTY_NAME)}).unwrap();
669-
io::Cursor::new(json_tools::TokenReader::new(
670-
json_tools::FilterTypedKeyValuePairs::new(
671-
json_tools::Lexer::new(
672+
let mut mem_dst = io::Cursor::new(Vec::with_capacity(json_cache.len()));
673+
io::copy(&mut TokenReader::new(
674+
FilterTypedKeyValuePairs::new(
675+
Lexer::new(
672676
json_cache.bytes(),
673-
json_tools::BufferType::Span),
674-
json_tools::TokenType::Null),
675-
Some(&json_cache)).bytes().filter_map(|v|v.ok()).collect::${'<Vec<u8>>'}()
676-
)
677+
BufferType::Span),
678+
TokenType::Null),
679+
Some(&json_cache)), &mut mem_dst).unwrap();
680+
mem_dst.seek(io::SeekFrom::Start(0)).unwrap();
681+
mem_dst
677682
};
678683
let request_size = request_value_reader.seek(io::SeekFrom::End(0)).unwrap();
679684
request_value_reader.seek(io::SeekFrom::Start(0)).unwrap();

0 commit comments

Comments
 (0)