Skip to content

Catch OutOfMemoryError at WasmGC allocation sites #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
andreaTP wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Catch OutOfMemoryError at WasmGC allocation sites #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d8eae45
Catch OutOfMemoryError at WasmGC allocation sites
andreaTP Aug 12, 2026
c391a7a
Merge branch 'main' into catch-oom-wasmgc
andreaTP Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 98 additions & 0 deletions compiler-tests/src/test/java/run/endive/testing/GcOutOfMemoryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package run.endive.testing;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

import org.junit.jupiter.api.Test;
import run.endive.compiler.MachineFactoryCompiler;
import run.endive.corpus.CorpusResources;
import run.endive.runtime.Instance;
import run.endive.runtime.InterpreterMachine;
import run.endive.wasm.Parser;
import run.endive.wasm.WasmEngineException;
import run.endive.wasm.WasmModule;

public class GcOutOfMemoryTest {

private static final WasmModule MODULE =
Parser.parse(CorpusResources.getResource("compiled/gc_oom_test.wat.wasm"));

private Instance interpreterInstance() {
return Instance.builder(MODULE).withMachineFactory(InterpreterMachine::new).build();
}

private Instance compilerInstance() {
return Instance.builder(MODULE).withMachineFactory(MachineFactoryCompiler::compile).build();
}

private void assertOom(Instance instance, String funcName) {
var fn = instance.export(funcName);
var ex = assertThrows(WasmEngineException.class, () -> fn.apply(Integer.MAX_VALUE));
assertTrue(ex.getMessage().contains("out of memory"));
}

// array.new_default with i32 elements

@Test
public void arrayNewDefaultI32Interpreter() {
assertOom(interpreterInstance(), "array_new_default_i32");
}

@Test
public void arrayNewDefaultI32Compiler() {
assertOom(compilerInstance(), "array_new_default_i32");
}

// array.new with i32 fill value

@Test
public void arrayNewI32Interpreter() {
assertOom(interpreterInstance(), "array_new_i32");
}

@Test
public void arrayNewI32Compiler() {
assertOom(compilerInstance(), "array_new_i32");
}

// array.new_default with reference elements

@Test
public void arrayNewDefaultRefInterpreter() {
assertOom(interpreterInstance(), "array_new_default_ref");
}

@Test
public void arrayNewDefaultRefCompiler() {
assertOom(compilerInstance(), "array_new_default_ref");
}

// array.new with reference fill value

@Test
public void arrayNewRefInterpreter() {
assertOom(interpreterInstance(), "array_new_ref");
}

@Test
public void arrayNewRefCompiler() {
assertOom(compilerInstance(), "array_new_ref");
}

// struct.new_default works normally (no OOM on small struct)

@Test
public void structNewDefaultNormalInterpreter() {
var instance = interpreterInstance();
var result = instance.export("struct_new_default").apply();
assertEquals(0, result[0]);
}

@Test
public void structNewDefaultNormalCompiler() {
var instance = compilerInstance();
var result = instance.export("struct_new_default").apply();
assertEquals(0, result[0]);
}
}
178 changes: 117 additions & 61 deletions compiler/src/main/java/run/endive/compiler/internal/Shaded.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,10 @@ public static RuntimeException throwCallStackExhausted(StackOverflowError e) {
throw new WasmEngineException("call stack exhausted", e);
}

public static RuntimeException throwOutOfMemory(OutOfMemoryError e) {
throw new WasmEngineException("out of memory", e);
}

public static RuntimeException throwIndirectCallTypeMismatch() {
return new WasmEngineException("indirect call type mismatch");
}
Expand Down Expand Up @@ -951,22 +955,38 @@ public static void memoryAtomicFence(Memory memory) {

public static Object structNew(
long[] fields, Object[] fieldRefs, int typeIdx, Instance instance) {
return WasmStruct.builder().typeIdx(typeIdx).fields(fields).fieldRefs(fieldRefs).build();
try {
return WasmStruct.builder()
.typeIdx(typeIdx)
.fields(fields)
.fieldRefs(fieldRefs)
.build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
}

public static Object structNewDefault(int typeIdx, Instance instance) {
var st = instance.module().typeSection().getSubType(typeIdx).compType().structType();
var fields = new long[st.fieldTypes().length];
var fieldRefs = new Object[st.fieldTypes().length];
for (int i = 0; i < st.fieldTypes().length; i++) {
var ft = st.fieldTypes()[i];
if (ft.storageType().valType() != null
&& ft.storageType().valType().isReference()
&& !ft.storageType().isObjectRef()) {
fields[i] = REF_NULL_VALUE;
try {
var st = instance.module().typeSection().getSubType(typeIdx).compType().structType();
var fields = new long[st.fieldTypes().length];
var fieldRefs = new Object[st.fieldTypes().length];
for (int i = 0; i < st.fieldTypes().length; i++) {
var ft = st.fieldTypes()[i];
if (ft.storageType().valType() != null
&& ft.storageType().valType().isReference()
&& !ft.storageType().isObjectRef()) {
fields[i] = REF_NULL_VALUE;
}
}
return WasmStruct.builder()
.typeIdx(typeIdx)
.fields(fields)
.fieldRefs(fieldRefs)
.build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
return WasmStruct.builder().typeIdx(typeIdx).fields(fields).fieldRefs(fieldRefs).build();
}

public static long structGet(Object ref, int typeIdx, int fieldIdx, Instance instance) {
Expand Down Expand Up @@ -1037,82 +1057,118 @@ public static void structSetRef(
}

public static Object arrayNew(long initVal, int len, int typeIdx, Instance instance) {
var elems = new long[len];
Arrays.fill(elems, initVal);
return WasmArray.builder().typeIdx(typeIdx).elements(elems).build();
try {
var elems = new long[len];
Arrays.fill(elems, initVal);
return WasmArray.builder().typeIdx(typeIdx).elements(elems).build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
}

public static Object arrayNewRef(Object initVal, int len, int typeIdx, Instance instance) {
var elems = new long[len];
var elemRefs = new Object[len];
Arrays.fill(elemRefs, initVal);
return WasmArray.builder().typeIdx(typeIdx).elements(elems).elementRefs(elemRefs).build();
}

public static Object arrayNewDefault(int len, int typeIdx, Instance instance) {
var elems = new long[len];
var at = instance.module().typeSection().getSubType(typeIdx).compType().arrayType();
var ft = at.fieldType();
if (ft.storageType().valType() != null && ft.storageType().isObjectRef()) {
try {
var elems = new long[len];
var elemRefs = new Object[len];
Arrays.fill(elemRefs, initVal);
return WasmArray.builder()
.typeIdx(typeIdx)
.elements(elems)
.elementRefs(new Object[len])
.elementRefs(elemRefs)
.build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
if (ft.storageType().valType() != null
&& ft.storageType().valType().isReference()
&& !ft.storageType().isObjectRef()) {
Arrays.fill(elems, REF_NULL_VALUE);
}

public static Object arrayNewDefault(int len, int typeIdx, Instance instance) {
try {
var elems = new long[len];
var at = instance.module().typeSection().getSubType(typeIdx).compType().arrayType();
var ft = at.fieldType();
if (ft.storageType().valType() != null && ft.storageType().isObjectRef()) {
return WasmArray.builder()
.typeIdx(typeIdx)
.elements(elems)
.elementRefs(new Object[len])
.build();
}
if (ft.storageType().valType() != null
&& ft.storageType().valType().isReference()
&& !ft.storageType().isObjectRef()) {
Arrays.fill(elems, REF_NULL_VALUE);
}
return WasmArray.builder().typeIdx(typeIdx).elements(elems).build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
return WasmArray.builder().typeIdx(typeIdx).elements(elems).build();
}

public static Object arrayNewFixed(long[] vals, int typeIdx, Instance instance) {
return WasmArray.builder().typeIdx(typeIdx).elements(vals).build();
try {
return WasmArray.builder().typeIdx(typeIdx).elements(vals).build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
}

public static Object arrayNewFixedRefs(Object[] vals, int typeIdx, Instance instance) {
var elems = new long[vals.length];
return WasmArray.builder().typeIdx(typeIdx).elements(elems).elementRefs(vals).build();
try {
var elems = new long[vals.length];
return WasmArray.builder().typeIdx(typeIdx).elements(elems).elementRefs(vals).build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
}

public static Object arrayNewData(
int offset, int len, int typeIdx, int dataIdx, Instance instance) {
var at = instance.module().typeSection().getSubType(typeIdx).compType().arrayType();
var elemSize = at.fieldType().storageType().byteSize();
var data = instance.dataSegmentData(dataIdx);
if ((long) offset + (long) len * elemSize > data.length) {
throw new TrapException("out of bounds memory access");
}
var elems = new long[len];
for (int i = 0; i < len; i++) {
var byteOff = offset + i * elemSize;
elems[i] = readFromData(data, byteOff, elemSize);
try {
var at = instance.module().typeSection().getSubType(typeIdx).compType().arrayType();
var elemSize = at.fieldType().storageType().byteSize();
var data = instance.dataSegmentData(dataIdx);
if ((long) offset + (long) len * elemSize > data.length) {
throw new TrapException("out of bounds memory access");
}
var elems = new long[len];
for (int i = 0; i < len; i++) {
var byteOff = offset + i * elemSize;
elems[i] = readFromData(data, byteOff, elemSize);
}
return WasmArray.builder().typeIdx(typeIdx).elements(elems).build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
return WasmArray.builder().typeIdx(typeIdx).elements(elems).build();
}

public static Object arrayNewElem(
int offset, int len, int typeIdx, int elemIdx, Instance instance) {
var element = instance.element(elemIdx);
if (element == null || offset + len > element.elementCount()) {
throw new TrapException("out of bounds table access");
}
var at = instance.module().typeSection().getSubType(typeIdx).compType().arrayType();
boolean isRef = at.fieldType().storageType().isObjectRef();
var elems = new long[len];
var elemRefs = new Object[len];
for (int i = 0; i < len; i++) {
var init = element.initializers().get(offset + i);
var result = ConstantEvaluators.computeConstant(instance, init);
if (isRef) {
elemRefs[i] = result.ref();
} else {
elems[i] = result.longValue();
try {
var element = instance.element(elemIdx);
if (element == null || offset + len > element.elementCount()) {
throw new TrapException("out of bounds table access");
}
var at = instance.module().typeSection().getSubType(typeIdx).compType().arrayType();
boolean isRef = at.fieldType().storageType().isObjectRef();
var elems = new long[len];
var elemRefs = new Object[len];
for (int i = 0; i < len; i++) {
var init = element.initializers().get(offset + i);
var result = ConstantEvaluators.computeConstant(instance, init);
if (isRef) {
elemRefs[i] = result.ref();
} else {
elems[i] = result.longValue();
}
}
return WasmArray.builder()
.typeIdx(typeIdx)
.elements(elems)
.elementRefs(elemRefs)
.build();
} catch (OutOfMemoryError e) {
throw throwOutOfMemory(e);
}
return WasmArray.builder().typeIdx(typeIdx).elements(elems).elementRefs(elemRefs).build();
}

public static long arrayGet(Object ref, int idx, int typeIdx, Instance instance) {
Expand Down
Loading
Loading