diff --git a/benchmark/buffers/buffer-write-string-utf8.js b/benchmark/buffers/buffer-write-string-utf8.js new file mode 100644 index 000000000000..930b83dcd55b --- /dev/null +++ b/benchmark/buffers/buffer-write-string-utf8.js @@ -0,0 +1,36 @@ +'use strict'; + +// buf.write(string, 'utf8') for strings whose in-memory representation is +// one-byte (Latin-1) or two-byte (UTF-16), which take different encoder paths. +const common = require('../common.js'); +const bench = common.createBenchmark(main, { + chars: ['one-byte', 'two-byte', 'two-byte-astral', 'two-byte-lone-surrogate'], + len: [16, 256, 2048, 65536], + n: [5e5], +}); + +function makeString(chars, len) { + switch (chars) { + case 'one-byte': + return 'aé'.repeat(len / 2); + case 'two-byte': + return 'aé€日'.repeat(len / 4); + case 'two-byte-astral': + return 'aé€日\u{1F600}'.repeat(len / 6).padEnd(len, 'a'); + case 'two-byte-lone-surrogate': + return 'aé€日'.repeat(len / 4 - 1) + 'ab\ud800c'; + default: + throw new Error(chars); + } +} + +function main({ chars, len, n }) { + const string = makeString(chars, len); + const buf = Buffer.allocUnsafe(Buffer.byteLength(string)); + if (len >= 65536) n = Math.floor(n / 32); + bench.start(); + for (let i = 0; i < n; ++i) { + buf.write(string, 0, 'utf8'); + } + bench.end(n); +} diff --git a/src/string_bytes.cc b/src/string_bytes.cc index b39d6fcc2ecb..88b499fb4289 100644 --- a/src/string_bytes.cc +++ b/src/string_bytes.cc @@ -310,9 +310,37 @@ size_t StringBytes::Write(Isolate* isolate, input_view.length(), buf, buflen); - } else { + } else if (input_view.length() <= 32) { + // V8 is as fast for tiny strings (same threshold TextEncoder uses). nbytes = str->WriteUtf8V2( isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8); + } else { + // Use simdutf for two-byte strings as well whenever the UTF-8 form + // is guaranteed to fit; truncating writes (which must stop at a + // character boundary) keep going through V8 so that their output + // stays byte-for-byte identical. + const char16_t* data = + reinterpret_cast(input_view.data16()); + const size_t length = input_view.length(); + MaybeStackBuffer well_formed; + if (!simdutf::validate_utf16(data, length)) { + // Unpaired surrogates: encode a copy in which each of them has been + // replaced with U+FFFD, which is what kReplaceInvalidUtf8 produces. + well_formed.AllocateSufficientStorage(length); + simdutf::to_well_formed_utf16(data, length, well_formed.out()); + data = well_formed.out(); + } + // A UTF-16 code unit never expands to more than 3 UTF-8 bytes, so + // 3 * length is what StorageSize() hands most callers; only compute + // the exact length when the buffer is smaller than that. + if (buflen >= 3 * length || + buflen >= simdutf::utf8_length_from_utf16(data, length)) { + nbytes = simdutf::convert_utf16_to_utf8(data, length, buf); + } else { + // Does not fit: let V8 truncate at a character boundary. + nbytes = str->WriteUtf8V2( + isolate, buf, buflen, String::WriteFlags::kReplaceInvalidUtf8); + } } break; diff --git a/src/string_decoder.cc b/src/string_decoder.cc index 4787a83aafe3..33f3ad94599a 100644 --- a/src/string_decoder.cc +++ b/src/string_decoder.cc @@ -28,23 +28,21 @@ MaybeLocal MakeString(Isolate* isolate, const char* data, size_t length, enum encoding encoding) { - MaybeLocal ret; - if (encoding == UTF8) { - MaybeLocal utf8_string; - if (length <= static_cast(v8::String::kMaxLength)) { - utf8_string = String::NewFromUtf8( - isolate, data, v8::NewStringType::kNormal, length); - } - if (utf8_string.IsEmpty()) { - isolate->ThrowException(node::ERR_STRING_TOO_LONG(isolate)); - return MaybeLocal(); - } else { - return utf8_string; - } - } else { - ret = StringBytes::Encode(isolate, data, length, encoding); + // StringBytes::Encode() would report an over-long UTF-8 input as + // ERR_BUFFER_TOO_LARGE (or clamp it); keep reporting it the way this + // decoder always has. + if (encoding == UTF8 && length > static_cast(v8::String::kMaxLength)) + [[unlikely]] { + isolate->ThrowException(node::ERR_STRING_TOO_LONG(isolate)); + return MaybeLocal(); } + // For UTF-8 this takes the simdutf-backed ASCII / Latin-1 / UTF-16 paths and + // only falls back to v8::String::NewFromUtf8() (the previous unconditional + // path here) for input containing invalid sequences, so U+FFFD replacement + // is unchanged. + MaybeLocal ret = StringBytes::Encode(isolate, data, length, encoding); + if (ret.IsEmpty()) { return {}; } diff --git a/test/parallel/test-buffer-write-utf8-two-byte.js b/test/parallel/test-buffer-write-utf8-two-byte.js new file mode 100644 index 000000000000..5d68cc90e7e5 --- /dev/null +++ b/test/parallel/test-buffer-write-utf8-two-byte.js @@ -0,0 +1,128 @@ +'use strict'; +// UTF-8 encoding of two-byte (UTF-16) JS strings through the Buffer write +// paths (Buffer.from, Buffer#write, Buffer.byteLength) must: +// - produce standard UTF-8 for well-formed input of any size, +// - replace lone surrogates with U+FFFD (EF BF BD), +// - never write a partial character when the target is too small, +// independent of which internal fast path handles the string. +require('../common'); +const assert = require('assert'); + +// Reference encoder written out longhand so the test does not depend on the +// implementation under test (TextEncoder shares code with it). +function utf8Reference(str) { + const out = []; + for (let i = 0; i < str.length; i++) { + let cp = str.charCodeAt(i); + if (cp >= 0xd800 && cp <= 0xdbff) { + const next = i + 1 < str.length ? str.charCodeAt(i + 1) : 0; + if (next >= 0xdc00 && next <= 0xdfff) { + cp = 0x10000 + ((cp - 0xd800) << 10) + (next - 0xdc00); + i++; + } else { + cp = 0xfffd; + } + } else if (cp >= 0xdc00 && cp <= 0xdfff) { + cp = 0xfffd; + } + if (cp < 0x80) { + out.push(cp); + } else if (cp < 0x800) { + out.push(0xc0 | (cp >> 6), 0x80 | (cp & 0x3f)); + } else if (cp < 0x10000) { + out.push(0xe0 | (cp >> 12), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f)); + } else { + out.push(0xf0 | (cp >> 18), 0x80 | ((cp >> 12) & 0x3f), + 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f)); + } + } + return Buffer.from(out); +} + +function checkFull(str, label) { + const expected = utf8Reference(str); + assert.deepStrictEqual(Buffer.from(str, 'utf8'), expected, `${label}: Buffer.from`); + assert.strictEqual(Buffer.byteLength(str, 'utf8'), expected.length, `${label}: byteLength`); + // Exact-size target. + const exact = Buffer.alloc(expected.length); + assert.strictEqual(exact.write(str, 'utf8'), expected.length, `${label}: write exact`); + assert.deepStrictEqual(exact, expected, `${label}: write exact bytes`); + // Oversized target (3 bytes per code unit is what most internal callers allocate). + const big = Buffer.alloc(str.length * 3 + 7, 0xaa); + assert.strictEqual(big.write(str, 2, 'utf8'), expected.length, `${label}: write big`); + assert.deepStrictEqual(big.subarray(2, 2 + expected.length), expected, `${label}: write big bytes`); + assert.strictEqual(big[0], 0xaa); + assert.strictEqual(big[2 + expected.length], 0xaa, `${label}: no overrun`); +} + +// Truncating writes must stop before the first character that does not fit. +function checkTruncation(str, label) { + const expected = utf8Reference(str); + for (let size = 0; size <= Math.min(expected.length, 70); size++) { + const target = Buffer.alloc(size + 1, 0xaa); + const n = target.write(str, 0, size, 'utf8'); + assert.ok(n <= size, `${label}: size=${size} wrote ${n}`); + assert.deepStrictEqual(target.subarray(0, n), expected.subarray(0, n), `${label}: prefix size=${size}`); + assert.strictEqual(target[size], 0xaa, `${label}: overrun size=${size}`); + // What was written must be a whole number of characters: the next byte in + // the reference (if any) has to be a lead byte, not a continuation byte. + if (n < expected.length) { + assert.notStrictEqual(expected[n] & 0xc0, 0x80, `${label}: split char at size=${size}`); + // And it stopped only because the next character really did not fit. + let next = n + 1; + while (next < expected.length && (expected[next] & 0xc0) === 0x80) next++; + assert.ok(next > size, `${label}: stopped early at size=${size} (n=${n}, next=${next})`); + } + } +} + +// Force a two-byte representation even for ASCII/Latin-1 content by building +// the string from a two-byte seed and slicing (V8 keeps the representation). +function twoByte(str) { + const s = ('\u{1F600}' + str).slice(2); + assert.strictEqual(s, str); + return s; +} + +const samples = { + ascii: 'The quick brown fox jumps over the lazy dog 0123456789', + latin1: 'français élan über naïve façade ÿ', + bmp: '日本語テキストとハングル한국어', + astral: 'emoji \u{1F600}\u{1F4A9} math \u{1D49C} han \u{20BB7}', + mixed: 'a é 日 \u{1F600} b ü 本 \u{1F4A9}', +}; + +for (const [name, base] of Object.entries(samples)) { + for (const repeat of [1, 3, 40, 700, 12000]) { + const str = twoByte(base.repeat(repeat)); + checkFull(str, `${name} x${repeat}`); + } + checkTruncation(twoByte(base.repeat(3)), `${name} truncation`); +} + +// Lone surrogates in various positions and sizes -> U+FFFD, rest intact. +const high = '\ud83d'; +const low = '\ude00'; +const surrogateCases = { + 'lone high': `ab${high}cd`, + 'lone low': `ab${low}cd`, + 'reversed pair': `ab${low}${high}cd`, + 'high at end': `abcd${high}`, + 'low at start': `${low}abcd`, + 'high high low': `${high}${high}${low}x`, + 'pair then lone': `${high}${low}${high}`, + 'only lone': high, +}; +for (const [name, base] of Object.entries(surrogateCases)) { + for (const pad of ['', 'x'.repeat(50), 'é'.repeat(300), '日'.repeat(30000)]) { + const str = pad + base + pad; + checkFull(str, `${name} pad=${pad.length}`); + } + checkTruncation(base + 'zz', `${name} truncation`); +} + +// Buffer.from of a large two-byte string equals TextEncoder output. +{ + const str = twoByte(samples.mixed.repeat(50000)); + assert.deepStrictEqual(Buffer.from(str), Buffer.from(new TextEncoder().encode(str))); +} diff --git a/test/parallel/test-string-decoder-utf8-large.js b/test/parallel/test-string-decoder-utf8-large.js new file mode 100644 index 000000000000..ccbabf26a27c --- /dev/null +++ b/test/parallel/test-string-decoder-utf8-large.js @@ -0,0 +1,81 @@ +'use strict'; +// The UTF-8 StringDecoder shares its byte->string conversion with +// Buffer#toString(): ASCII, Latin-1-representable and general inputs take +// different (SIMD) paths depending on content and size, and invalid input +// falls back to a replacing decoder. This test pins the decoder's output for +// inputs that cross those size thresholds, for chunkings that split multibyte +// sequences, and for invalid bytes embedded in otherwise large valid input. +require('../common'); +const assert = require('assert'); +const { StringDecoder } = require('string_decoder'); + +function decodeInChunks(buf, chunkSize) { + const decoder = new StringDecoder('utf8'); + let out = ''; + for (let i = 0; i < buf.length; i += chunkSize) { + out += decoder.write(buf.subarray(i, i + chunkSize)); + } + return out + decoder.end(); +} + +function check(str, label) { + const buf = Buffer.from(str, 'utf8'); + // Sanity: the expectation itself round-trips. + assert.strictEqual(buf.toString('utf8'), str, `${label}: toString`); + for (const chunkSize of [1, 2, 3, 4, 5, 7, 31, 32, 33, 255, 256, 257, + 4095, 4096, 65536, buf.length]) { + if (chunkSize > buf.length) continue; + // Keep the test fast: byte-sized chunks only for the smaller inputs. + if (buf.length > 100_000 && chunkSize < 4095) continue; + assert.strictEqual(decodeInChunks(buf, chunkSize), str, + `${label}: chunkSize=${chunkSize}`); + } +} + +const sizes = [31, 32, 33, 255, 256, 257, 4096, 70000, (1 << 20) + 5]; +for (const size of sizes) { + check('a'.repeat(size), `ascii ${size}`); + // Latin-1 range only (one-byte string in V8, two bytes each in UTF-8). + check('é'.repeat(size), `latin1 ${size}`); + // ASCII with a single Latin-1 character at the end / start. + check('a'.repeat(size - 1) + 'ÿ', `ascii+latin1 tail ${size}`); + check('Ä' + 'a'.repeat(size - 1), `latin1 head+ascii ${size}`); + // BMP beyond Latin-1 (three-byte sequences). + check('日'.repeat(size), `cjk ${size}`); + // Mixed, including astral plane characters (surrogate pairs, 4 bytes). + check(('abé日\u{1F600}').repeat(Math.ceil(size / 6)), `mixed ${size}`); +} + +// Invalid bytes inside otherwise valid input of every size class must still be +// replaced with U+FFFD exactly as before, regardless of chunking. +for (const size of [8, 40, 300, 5000, (1 << 20) + 5]) { + const valid = Buffer.from('a'.repeat(size)); + for (const bad of [[0xff], [0xc0, 0xaf], [0xe2, 0x28, 0xa1], + [0xed, 0xa0, 0x80] /* encoded surrogate */, + [0xf0, 0x9f, 0x98] /* truncated 4-byte */]) { + const buf = Buffer.concat([valid, Buffer.from(bad), valid]); + const expected = buf.toString('utf8'); + assert.ok(expected.includes('�'), `size=${size} bad=${bad}`); + for (const chunkSize of [1, 3, 64, size, size + 1, buf.length]) { + if (buf.length > 100_000 && chunkSize < size) continue; + assert.strictEqual(decodeInChunks(buf, chunkSize), expected, + `invalid ${bad} in ${size}, chunkSize=${chunkSize}`); + } + } +} + +// A lone continuation / lead byte split across the size classes at the very +// end is buffered by the decoder and flushed as U+FFFD by end(). +{ + const decoder = new StringDecoder('utf8'); + const big = Buffer.concat([Buffer.from('a'.repeat(300)), Buffer.from([0xe2, 0x82])]); + assert.strictEqual(decoder.write(big), 'a'.repeat(300)); + assert.strictEqual(decoder.end(), '�'); +} +{ + const decoder = new StringDecoder('utf8'); + const big = Buffer.concat([Buffer.from('é'.repeat(300)), Buffer.from([0xe2, 0x82])]); + assert.strictEqual(decoder.write(big), 'é'.repeat(300)); + assert.strictEqual(decoder.write(Buffer.from([0xac])), '€'); + assert.strictEqual(decoder.end(), ''); +}