src: fix TextDecoder large-input and error paths

src/node_i18n.cc

@@ -66,6 +66,7 @@
 #include <unicode/utypes.h>
 #include <unicode/uvernum.h>
 #include <unicode/uversion.h>
+#include <algorithm>
 #include "nbytes.h"
 
 #ifdef NODE_HAVE_SMALL_ICU
@@ -446,18 +447,22 @@ void ConverterObject::Decode(const FunctionCallbackInfo<Value>& args) {
 
   UBool flush = (flags & CONVERTER_FLAGS_FLUSH) == CONVERTER_FLAGS_FLUSH;
 
-  // When flushing the final chunk, the limit is the maximum
-  // of either the input buffer length or the number of pending
-  // characters times the min char size, multiplied by 2 as unicode may
-  // take up to 2 UChars to encode a character
-  size_t limit = 2 * converter->min_char_size() *
-      (!flush ?
-          input.length() :
-          std::max(
-              input.length(),
-              static_cast<size_t>(
-                  ucnv_toUCountPending(converter->conv(), &status))));
+  // ucnv_toUnicode() rejects target ranges larger than this in its
+  // argument validation. See deps/icu-small/source/common/ucnv.cpp.
+  constexpr size_t kMaxTargetUChars = 0x3fffffff;
+
+  // Each character consumes at least min_char_size() bytes and produces at
+  // most 2 UChars (a surrogate pair). Count the bytes the converter is
+  // still holding from previous chunks too: they belong to a character
+  // whose remaining bytes may arrive in this chunk. Clamping to the ICU
+  // cap loses nothing: any result that can become a V8 string needs at
+  // most String::kMaxLength UChars, well under the cap.
+  int32_t pending = ucnv_toUCountPending(converter->conv(), &status);
   status = U_ZERO_ERROR;
+  size_t limit = std::min(
+      2 * (input.length() + (pending > 0 ? static_cast<size_t>(pending) : 0)) /
+          converter->min_char_size(),
+      kMaxTargetUChars);
 
   if (limit > 0)
     result.AllocateSufficientStorage(limit);
@@ -519,8 +524,10 @@ void ConverterObject::Decode(const FunctionCallbackInfo<Value>& args) {
     if (StringBytes::Encode(env->isolate(), value, length, UCS2)
             .ToLocal(&ret)) {
       args.GetReturnValue().Set(ret);
-      return;
     }
+    // If Encode() failed, it has already scheduled an exception; do not
+    // replace it with ERR_ENCODING_INVALID_ENCODED_DATA below.
+    return;
   }
 
   node::THROW_ERR_ENCODING_INVALID_ENCODED_DATA(

test/pummel/test-whatwg-encoding-custom-textdecoder-large.js

@@ -0,0 +1,68 @@
+'use strict';
+const common = require('../common');
+
+// Input large enough that the old 4x target bound exceeded ICU's
+// 0x3fffffff UChar limit; also needs more than a 32-bit heap.
+common.skipIf32Bits();
+
+if (!common.hasIntl)
+  common.skip('missing Intl');
+
+// Peak RSS is around 1.6 GiB: the input, the ICU target buffer, and two
+// result strings.
+if (require('os').totalmem() < 8 * 2 ** 30)
+  common.skip('less than 8 GiB of total memory');
+
+const assert = require('assert');
+
+const size = 2 ** 27;
+
+let input;
+
+try {
+  input = Buffer.allocUnsafe(size * 2);
+} catch (e) {
+  if (
+    e.code === 'ERR_MEMORY_ALLOCATION_FAILED' ||
+    /Array buffer allocation failed/.test(e.message)
+  ) {
+    common.skip('insufficient space for Buffer.allocUnsafe');
+  }
+
+  throw e;
+}
+
+// Non-uniform repeating pattern of A, a U+1F600 surrogate pair and 中,
+// written as explicit little-endian bytes so the input is identical on
+// big-endian hosts. Corrupted or misplaced output cannot match it.
+input.fill(Buffer.from([0x41, 0x00, 0x3D, 0xD8, 0x00, 0xDE, 0x2D, 0x4E]));
+
+const decoder = new TextDecoder('utf-16le');
+
+// 2 ** 27 UTF-16 code units used to fail with
+// ERR_ENCODING_INVALID_ENCODED_DATA because the target buffer request
+// exceeded ICU's internal targetLimit validation.
+// Refs: https://github.com/nodejs/node/issues/47645
+const result = decoder.decode(input);
+assert.strictEqual(result.length, size);
+assert.strictEqual(result[0], 'A');
+assert.strictEqual(result[1], '\uD83D');
+assert.strictEqual(result[2], '\uDE00');
+assert.strictEqual(result[size / 2], 'A');
+assert.strictEqual(result[size - 1], '中');
+
+// Guard against over-correction: one code unit below the failure boundary
+// decodes at HEAD too and must keep doing so. The truncation removes the
+// trailing 中, so it does not split a surrogate pair.
+assert.strictEqual(decoder.decode(input.subarray(0, size * 2 - 2)).length,
+                   size - 1);
+
+// Streaming with an odd byte split lands mid-code-unit, so one byte stays
+// pending in the converter across the chunk boundary. The full content is
+// compared against the non-streaming result, so any corruption at the
+// boundary fails the test.
+const split = 2 ** 26 + 1;
+const streamed = decoder.decode(input.subarray(0, split), { stream: true }) +
+                 decoder.decode(input.subarray(split));
+assert.strictEqual(streamed.length, result.length);
+assert.strictEqual(streamed, result);

test/pummel/test-whatwg-encoding-custom-textdecoder-toolong.js

@@ -0,0 +1,79 @@
+'use strict';
+const common = require('../common');
+
+// The working set is around 3 GiB, far beyond a 32-bit heap.
+common.skipIf32Bits();
+
+if (!common.hasIntl)
+  common.skip('missing Intl');
+
+// Peak RSS is around 3 GiB: a 1 GiB input, a 2 GiB ICU target buffer, and
+// a transient 1 GiB StringBytes copy.
+if (require('os').totalmem() < 8 * 2 ** 30)
+  common.skip('less than 8 GiB of total memory');
+
+const assert = require('assert');
+const kStringMaxLength = require('buffer').constants.MAX_STRING_LENGTH;
+
+function allocOrSkip(bytes) {
+  try {
+    return Buffer.allocUnsafe(bytes);
+  } catch (e) {
+    if (
+      e.code === 'ERR_MEMORY_ALLOCATION_FAILED' ||
+      /Array buffer allocation failed/.test(e.message)
+    ) {
+      common.skip('insufficient space for Buffer.allocUnsafe');
+    }
+
+    throw e;
+  }
+}
+
+function assertThrowsTooLong(fn) {
+  assert.throws(fn, (e) => {
+    // Constrained machines can fail the 1 GiB copy StringBytes makes while
+    // building the string, before the length limit is reached.
+    if (e.code === 'ERR_MEMORY_ALLOCATION_FAILED')
+      common.skip('insufficient memory for the StringBytes copy');
+
+    assert.strictEqual(e.code, 'ERR_STRING_TOO_LONG');
+    return true;
+  });
+}
+
+{
+  // One UTF-16 code unit beyond the maximum string length: the decode
+  // completes inside ICU but the resulting kStringMaxLength + 1 characters
+  // cannot be materialised as a string, which must surface as
+  // ERR_STRING_TOO_LONG rather than ERR_ENCODING_INVALID_ENCODED_DATA.
+  // The ICU target buffer request is 2 * (size / 2) = size UChars, which
+  // must stay <= 0x3fffffff (ICU's targetLimit cap) for the conversion to
+  // run at all; size = 2 * kStringMaxLength + 2 = 1073741778 satisfies
+  // that.
+  const size = 2 * kStringMaxLength + 2;
+  const input = allocOrSkip(size);
+  input.fill(0x20);
+  assertThrowsTooLong(() => new TextDecoder('utf-16le').decode(input));
+}
+
+{
+  // Same limit through a min_char_size() == 1 encoding: pure-ASCII input
+  // of kStringMaxLength + 1 bytes decodes to kStringMaxLength + 1
+  // characters. The 2x target bound is clamped to ICU's cap, so only the
+  // output length decides the outcome. gb18030 needs full-icu; skip the
+  // case silently on small-icu builds (the utf-16le case above ran).
+  let decoder;
+  try {
+    decoder = new TextDecoder('gb18030');
+  } catch (e) {
+    if (e.code !== 'ERR_ENCODING_NOT_SUPPORTED')
+      throw e;
+  }
+
+  if (decoder !== undefined) {
+    const input = allocOrSkip(kStringMaxLength + 1);
+    input.fill(0x41);
+    assertThrowsTooLong(() => decoder.decode(input));
+  }
+}