diff --git a/Polyfills/TextDecoder/README.md b/Polyfills/TextDecoder/README.md index d11cb9ac..bd34b5b3 100644 --- a/Polyfills/TextDecoder/README.md +++ b/Polyfills/TextDecoder/README.md @@ -6,19 +6,20 @@ A C++ implementation of the [WHATWG Encoding API](https://encoding.spec.whatwg.o ### Supported -- Decoding `Uint8Array`, `Int8Array`, and other typed array views from a UTF-8 encoded byte sequence. +- Decoding `Uint8Array`, `Int8Array`, and other typed array views from a UTF-8 or UTF-16 encoded byte sequence. - Decoding raw `ArrayBuffer` objects. - Constructing `TextDecoder` with no argument (defaults to `utf-8`). -- Constructing `TextDecoder` with the explicit encoding label `"utf-8"` or `"UTF-8"`. +- Constructing `TextDecoder` with any WHATWG label for UTF-8 (`"utf-8"`, `"utf8"`, `"unicode-1-1-utf-8"`, `"unicode11utf8"`, `"unicode20utf8"`, `"x-unicode20utf8"`), UTF-16LE (`"utf-16"`, `"utf-16le"`, `"ucs-2"`, `"unicode"`, `"unicodeFEFF"`, `"csunicode"`, `"iso-10646-ucs-2"`) or UTF-16BE (`"utf-16be"`, `"unicodeFFFE"`). Labels are matched case-insensitively and ignore surrounding whitespace. +- Stripping a leading byte order mark when decoding UTF-16. - Calling `decode()` with no argument or `undefined` returns an empty string (matches the Web API). ### Not Supported -- Encodings other than UTF-8 — passing any other label (e.g. `"utf-16"`, `"iso-8859-1"`) throws a JavaScript `Error`. +- Encodings other than UTF-8 and UTF-16 — passing any other label (e.g. `"iso-8859-1"`) throws a JavaScript `Error`. - `DataView` is not accepted by `decode()` — due to missing `Napi::DataView` support in the underlying JSI layer. - Passing a non-BufferSource value (e.g. a string or number) to `decode()` throws a `TypeError`. -- The `fatal` option: decoding errors are not detected and do not throw a `TypeError`. -- The `ignoreBOM` option: the byte order mark is not stripped. +- The `fatal` option: decoding errors are not detected and do not throw a `TypeError`. A trailing odd byte in a UTF-16 sequence is dropped rather than decoded as U+FFFD. +- The `ignoreBOM` option: a leading UTF-16 byte order mark is always stripped and cannot be retained. A UTF-8 byte order mark is never stripped. - Streaming decode (passing `{ stream: true }` to `decode()`) — each call is stateless. - The `encoding` property on the `TextDecoder` instance is not exposed. @@ -30,10 +31,12 @@ const decoder = new TextDecoder("utf-8"); // explicit, also fine const bytes = new Uint8Array([72, 101, 108, 108, 111]); decoder.decode(bytes); // "Hello" + +new TextDecoder("utf-16le").decode(new Uint8Array([0x48, 0x00, 0x69, 0x00])); // "Hi" ``` Passing an unsupported encoding throws: ```javascript -new TextDecoder("utf-16"); // Error: TextDecoder: unsupported encoding 'utf-16', only 'utf-8' is supported +new TextDecoder("iso-8859-1"); // Error: TextDecoder: unsupported encoding 'iso-8859-1', only UTF-8 and UTF-16 are supported ``` diff --git a/Polyfills/TextDecoder/Source/TextDecoder.cpp b/Polyfills/TextDecoder/Source/TextDecoder.cpp index e209fbf0..640e5d25 100644 --- a/Polyfills/TextDecoder/Source/TextDecoder.cpp +++ b/Polyfills/TextDecoder/Source/TextDecoder.cpp @@ -38,6 +38,16 @@ namespace class TextDecoder final : public Napi::ObjectWrap { + // Only the encodings the runtime actually needs. UTF-16 shows up in Emscripten output + // (UTF16ToString creates a `new TextDecoder('utf-16le')` at module scope), so refusing + // it makes whole WebAssembly modules unloadable. + enum class Encoding + { + Utf8, + Utf16LittleEndian, + Utf16BigEndian + }; + public: static void Initialize(Napi::Env env) { @@ -67,19 +77,63 @@ namespace // Several labels (e.g. "utf8", "unicode-1-1-utf-8") all map to UTF-8 after // normalization; callers such as the glTF/Draco loader pass "utf8". const std::string label = NormalizeEncodingLabel(encoding); - if (label != "utf-8" && - label != "utf8" && - label != "unicode-1-1-utf-8" && - label != "unicode11utf8" && - label != "unicode20utf8" && - label != "x-unicode20utf8") + if (label == "utf-8" || + label == "utf8" || + label == "unicode-1-1-utf-8" || + label == "unicode11utf8" || + label == "unicode20utf8" || + label == "x-unicode20utf8") + { + m_encoding = Encoding::Utf8; + } + else if (label == "utf-16" || + label == "utf-16le" || + label == "ucs-2" || + label == "unicode" || + label == "unicodefeff" || + label == "csunicode" || + label == "iso-10646-ucs-2") { - throw Napi::Error::New(Env(), "TextDecoder: unsupported encoding '" + encoding + "', only UTF-8 is supported"); + m_encoding = Encoding::Utf16LittleEndian; + } + else if (label == "utf-16be" || + label == "unicodefffe") + { + m_encoding = Encoding::Utf16BigEndian; + } + else + { + throw Napi::Error::New(Env(), "TextDecoder: unsupported encoding '" + encoding + "', only UTF-8 and UTF-16 are supported"); } } } private: + Encoding m_encoding{Encoding::Utf8}; + + Napi::Value DecodeUtf16(Napi::Env env, const std::string& data) const + { + // Trailing odd byte is dropped: the WHATWG decoder would emit U+FFFD for it, but + // every producer we care about hands over whole code units. + const size_t unitCount = data.size() / 2; + std::u16string units(unitCount, u'\0'); + for (size_t index = 0; index < unitCount; ++index) + { + const auto first = static_cast(data[index * 2]); + const auto second = static_cast(data[index * 2 + 1]); + units[index] = m_encoding == Encoding::Utf16LittleEndian + ? static_cast(first | (second << 8)) + : static_cast(second | (first << 8)); + } + + if (!units.empty() && units.front() == u'\uFEFF') + { + units.erase(0, 1); + } + + return Napi::String::New(env, units); + } + Napi::Value Decode(const Napi::CallbackInfo& info) { if (info.Length() < 1 || info[0].IsUndefined()) @@ -116,7 +170,7 @@ namespace throw Napi::TypeError::New(Env(), "TextDecoder.decode: input must be a BufferSource (ArrayBuffer or TypedArray)"); } - return Napi::String::New(info.Env(), data); + return m_encoding == Encoding::Utf8 ? Napi::String::New(info.Env(), data) : DecodeUtf16(info.Env(), data); } }; } diff --git a/Tests/UnitTests/Scripts/tests.ts b/Tests/UnitTests/Scripts/tests.ts index 1dc2aa4c..99770909 100644 --- a/Tests/UnitTests/Scripts/tests.ts +++ b/Tests/UnitTests/Scripts/tests.ts @@ -1736,7 +1736,7 @@ describe("TextDecoder", function () { // many times to create many dangling wraps, then allocate/decode to // exercise the heap and surface any corruption within this test run. for (let i = 0; i < 100; ++i) { - expect(() => new TextDecoder("utf-16")).to.throw(); + expect(() => new TextDecoder("iso-8859-2")).to.throw(); } const decoder = new TextDecoder("utf-8"); expect(decoder.decode(new Uint8Array([79, 75]))).to.equal("OK"); @@ -1763,7 +1763,39 @@ describe("TextDecoder", function () { }); it("should still throw for a genuinely unsupported encoding", function () { - expect(() => new TextDecoder("utf-16")).to.throw(); + expect(() => new TextDecoder("iso-8859-2")).to.throw(); + }); + + it("should decode utf-16le", function () { + const decoder = new TextDecoder("utf-16le"); + // "Hi" as UTF-16LE code units. + expect(decoder.decode(new Uint8Array([0x48, 0x00, 0x69, 0x00]))).to.equal("Hi"); + }); + + it("should decode utf-16be", function () { + const decoder = new TextDecoder("utf-16be"); + expect(decoder.decode(new Uint8Array([0x00, 0x48, 0x00, 0x69]))).to.equal("Hi"); + }); + + it("should accept the other WHATWG utf-16 aliases as little endian", function () { + for (const label of ["utf-16", "ucs-2", "unicode", "unicodeFEFF", "csunicode", "iso-10646-ucs-2"]) { + const decoder = new TextDecoder(label); + expect(decoder.decode(new Uint8Array([0x4F, 0x00, 0x4B, 0x00]))).to.equal("OK"); + } + expect(new TextDecoder("unicodeFFFE").decode(new Uint8Array([0x00, 0x4F, 0x00, 0x4B]))).to.equal("OK"); + }); + + it("should strip a leading byte order mark", function () { + expect(new TextDecoder("utf-16le").decode(new Uint8Array([0xFF, 0xFE, 0x48, 0x00]))).to.equal("H"); + expect(new TextDecoder("utf-16be").decode(new Uint8Array([0xFE, 0xFF, 0x00, 0x48]))).to.equal("H"); + }); + + it("should decode utf-16 outside the BMP and preserve null code units", function () { + // U+1F600 as a surrogate pair, then U+0000, then "A". + const decoder = new TextDecoder("utf-16le"); + const result = decoder.decode(new Uint8Array([0x3D, 0xD8, 0x00, 0xDE, 0x00, 0x00, 0x41, 0x00])); + expect(result).to.equal("\u{1F600}\0A"); + expect(result.length).to.equal(4); }); });