TextDecoder: support UTF-16LE and UTF-16BE

Polyfills/TextDecoder/README.md

@@ -6,19 +6,20 @@ A C++ implementation of the [WHATWG Encoding API](https://encoding.spec.whatwg.o
 
 ### Supported
 
-- Decoding `Uint8Array`, `Int8Array`, and other typed array views from a UTF-8 encoded byte sequence.
+- Decoding `Uint8Array`, `Int8Array`, and other typed array views from a UTF-8 or UTF-16 encoded byte sequence.
 - Decoding raw `ArrayBuffer` objects.
 - Constructing `TextDecoder` with no argument (defaults to `utf-8`).
-- Constructing `TextDecoder` with the explicit encoding label `"utf-8"` or `"UTF-8"`.
+- Constructing `TextDecoder` with any WHATWG label for UTF-8 (`"utf-8"`, `"utf8"`, `"unicode-1-1-utf-8"`, `"unicode11utf8"`, `"unicode20utf8"`, `"x-unicode20utf8"`), UTF-16LE (`"utf-16"`, `"utf-16le"`, `"ucs-2"`, `"unicode"`, `"unicodeFEFF"`, `"csunicode"`, `"iso-10646-ucs-2"`) or UTF-16BE (`"utf-16be"`, `"unicodeFFFE"`). Labels are matched case-insensitively and ignore surrounding whitespace.
+- Stripping a leading byte order mark when decoding UTF-16.
 - Calling `decode()` with no argument or `undefined` returns an empty string (matches the Web API).
 
 ### Not Supported
 
-- Encodings other than UTF-8 — passing any other label (e.g. `"utf-16"`, `"iso-8859-1"`) throws a JavaScript `Error`.
+- Encodings other than UTF-8 and UTF-16 — passing any other label (e.g. `"iso-8859-1"`) throws a JavaScript `Error`.
 - `DataView` is not accepted by `decode()` — due to missing `Napi::DataView` support in the underlying JSI layer.
 - Passing a non-BufferSource value (e.g. a string or number) to `decode()` throws a `TypeError`.
-- The `fatal` option: decoding errors are not detected and do not throw a `TypeError`.
-- The `ignoreBOM` option: the byte order mark is not stripped.
+- The `fatal` option: decoding errors are not detected and do not throw a `TypeError`. A trailing odd byte in a UTF-16 sequence is dropped rather than decoded as U+FFFD.
+- The `ignoreBOM` option: a leading UTF-16 byte order mark is always stripped and cannot be retained. A UTF-8 byte order mark is never stripped.
 - Streaming decode (passing `{ stream: true }` to `decode()`) — each call is stateless.
 - The `encoding` property on the `TextDecoder` instance is not exposed.
 
@@ -30,10 +31,12 @@ const decoder = new TextDecoder("utf-8");       // explicit, also fine
 
 const bytes = new Uint8Array([72, 101, 108, 108, 111]);
 decoder.decode(bytes); // "Hello"
+
+new TextDecoder("utf-16le").decode(new Uint8Array([0x48, 0x00, 0x69, 0x00])); // "Hi"
 ```
 
 Passing an unsupported encoding throws:
 
 ```javascript
-new TextDecoder("utf-16"); // Error: TextDecoder: unsupported encoding 'utf-16', only 'utf-8' is supported
+new TextDecoder("iso-8859-1"); // Error: TextDecoder: unsupported encoding 'iso-8859-1', only UTF-8 and UTF-16 are supported
 ```

Polyfills/TextDecoder/Source/TextDecoder.cpp

@@ -38,6 +38,16 @@ namespace
 
     class TextDecoder final : public Napi::ObjectWrap<TextDecoder>
     {
+        // Only the encodings the runtime actually needs. UTF-16 shows up in Emscripten output
+        // (UTF16ToString creates a `new TextDecoder('utf-16le')` at module scope), so refusing
+        // it makes whole WebAssembly modules unloadable.
+        enum class Encoding
+        {
+            Utf8,
+            Utf16LittleEndian,
+            Utf16BigEndian
+        };
+
     public:
         static void Initialize(Napi::Env env)
         {
@@ -67,19 +77,63 @@ namespace
                 // Several labels (e.g. "utf8", "unicode-1-1-utf-8") all map to UTF-8 after
                 // normalization; callers such as the glTF/Draco loader pass "utf8".
                 const std::string label = NormalizeEncodingLabel(encoding);
-                if (label != "utf-8" &&
-                    label != "utf8" &&
-                    label != "unicode-1-1-utf-8" &&
-                    label != "unicode11utf8" &&
-                    label != "unicode20utf8" &&
-                    label != "x-unicode20utf8")
+                if (label == "utf-8" ||
+                    label == "utf8" ||
+                    label == "unicode-1-1-utf-8" ||
+                    label == "unicode11utf8" ||
+                    label == "unicode20utf8" ||
+                    label == "x-unicode20utf8")
+                {
+                    m_encoding = Encoding::Utf8;
+                }
+                else if (label == "utf-16" ||
+                         label == "utf-16le" ||
+                         label == "ucs-2" ||
+                         label == "unicode" ||
+                         label == "unicodefeff" ||
+                         label == "csunicode" ||
+                         label == "iso-10646-ucs-2")
                 {
-                    throw Napi::Error::New(Env(), "TextDecoder: unsupported encoding '" + encoding + "', only UTF-8 is supported");
+                    m_encoding = Encoding::Utf16LittleEndian;
+                }
+                else if (label == "utf-16be" ||
+                         label == "unicodefffe")
+                {
+                    m_encoding = Encoding::Utf16BigEndian;
+                }
+                else
+                {
+                    throw Napi::Error::New(Env(), "TextDecoder: unsupported encoding '" + encoding + "', only UTF-8 and UTF-16 are supported");
                 }
             }
         }
 
     private:
+        Encoding m_encoding{Encoding::Utf8};
+
+        Napi::Value DecodeUtf16(Napi::Env env, const std::string& data) const
+        {
+            // Trailing odd byte is dropped: the WHATWG decoder would emit U+FFFD for it, but
+            // every producer we care about hands over whole code units.
+            const size_t unitCount = data.size() / 2;
+            std::u16string units(unitCount, u'\0');
+            for (size_t index = 0; index < unitCount; ++index)
+            {
+                const auto first = static_cast<unsigned char>(data[index * 2]);
+                const auto second = static_cast<unsigned char>(data[index * 2 + 1]);
+                units[index] = m_encoding == Encoding::Utf16LittleEndian
+                    ? static_cast<char16_t>(first | (second << 8))
+                    : static_cast<char16_t>(second | (first << 8));
+            }
+
+            if (!units.empty() && units.front() == u'\uFEFF')
+            {
+                units.erase(0, 1);
+            }
+
+            return Napi::String::New(env, units);
+        }
+
         Napi::Value Decode(const Napi::CallbackInfo& info)
         {
             if (info.Length() < 1 || info[0].IsUndefined())
@@ -116,7 +170,7 @@ namespace
                 throw Napi::TypeError::New(Env(), "TextDecoder.decode: input must be a BufferSource (ArrayBuffer or TypedArray)");
             }
 
-            return Napi::String::New(info.Env(), data);
+            return m_encoding == Encoding::Utf8 ? Napi::String::New(info.Env(), data) : DecodeUtf16(info.Env(), data);
         }
     };
 }

Tests/UnitTests/Scripts/tests.ts

@@ -1736,7 +1736,7 @@ describe("TextDecoder", function () {
         // many times to create many dangling wraps, then allocate/decode to
         // exercise the heap and surface any corruption within this test run.
         for (let i = 0; i < 100; ++i) {
-            expect(() => new TextDecoder("utf-16")).to.throw();
+            expect(() => new TextDecoder("iso-8859-2")).to.throw();
         }
         const decoder = new TextDecoder("utf-8");
         expect(decoder.decode(new Uint8Array([79, 75]))).to.equal("OK");
@@ -1763,7 +1763,39 @@ describe("TextDecoder", function () {
     });
 
     it("should still throw for a genuinely unsupported encoding", function () {
-        expect(() => new TextDecoder("utf-16")).to.throw();
+        expect(() => new TextDecoder("iso-8859-2")).to.throw();
+    });
+
+    it("should decode utf-16le", function () {
+        const decoder = new TextDecoder("utf-16le");
+        // "Hi" as UTF-16LE code units.
+        expect(decoder.decode(new Uint8Array([0x48, 0x00, 0x69, 0x00]))).to.equal("Hi");
+    });
+
+    it("should decode utf-16be", function () {
+        const decoder = new TextDecoder("utf-16be");
+        expect(decoder.decode(new Uint8Array([0x00, 0x48, 0x00, 0x69]))).to.equal("Hi");
+    });
+
+    it("should accept the other WHATWG utf-16 aliases as little endian", function () {
+        for (const label of ["utf-16", "ucs-2", "unicode", "unicodeFEFF", "csunicode", "iso-10646-ucs-2"]) {
+            const decoder = new TextDecoder(label);
+            expect(decoder.decode(new Uint8Array([0x4F, 0x00, 0x4B, 0x00]))).to.equal("OK");
+        }
+        expect(new TextDecoder("unicodeFFFE").decode(new Uint8Array([0x00, 0x4F, 0x00, 0x4B]))).to.equal("OK");
+    });
+
+    it("should strip a leading byte order mark", function () {
+        expect(new TextDecoder("utf-16le").decode(new Uint8Array([0xFF, 0xFE, 0x48, 0x00]))).to.equal("H");
+        expect(new TextDecoder("utf-16be").decode(new Uint8Array([0xFE, 0xFF, 0x00, 0x48]))).to.equal("H");
+    });
+
+    it("should decode utf-16 outside the BMP and preserve null code units", function () {
+        // U+1F600 as a surrogate pair, then U+0000, then "A".
+        const decoder = new TextDecoder("utf-16le");
+        const result = decoder.decode(new Uint8Array([0x3D, 0xD8, 0x00, 0xDE, 0x00, 0x00, 0x41, 0x00]));
+        expect(result).to.equal("\u{1F600}\0A");
+        expect(result.length).to.equal(4);
     });
 });