From 96249013a2dbbe4a9ac4aaff95b06f2ee24de006 Mon Sep 17 00:00:00 2001 From: prygunovx Date: Mon, 17 Aug 2026 11:30:17 +0200 Subject: [PATCH 1/2] AVRO-4332: [java] Add support for enum default values in IDL serialization IdlUtils.writeSchema did not emit the enum default when serializing a schema to IDL, even though IdlReader parses it. As a result, an enum default was silently dropped when round-tripping a schema through IDL. Append "= ;" after the enum body when the schema has a default, matching the IDL grammar (RBrace defaultSymbol=enumDefault?), and add tests covering the with-default, without-default and write-then-parse round-trip cases. --- .../java/org/apache/avro/idl/IdlUtils.java | 7 ++- .../org/apache/avro/idl/IdlUtilsTest.java | 55 +++++++++++++++++++ .../avro/idl/idl_utils_test_enum_default.avdl | 10 ++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_enum_default.avdl diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlUtils.java b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlUtils.java index 29c787c9e27..80499a27093 100644 --- a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlUtils.java +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlUtils.java @@ -285,7 +285,12 @@ private static void writeSchema(Schema schema, boolean insideProtocol, Writer wr } else { throw new AvroRuntimeException("Enum schema must have at least a symbol " + schema); } - writer.append(NEWLINE).append(indent).append("}").append(NEWLINE); + writer.append(NEWLINE).append(indent).append("}"); + String enumDefault = schema.getEnumDefault(); + if (enumDefault != null) { + writer.append(" = ").append(enumDefault).append(";"); + } + writer.append(NEWLINE); } else /* (type == Schema.Type.FIXED) */ { writer.append(indent).append("fixed ").append(schemaName).append('(') .append(Integer.toString(schema.getFixedSize())).append(");").append(NEWLINE); diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/IdlUtilsTest.java b/lang/java/idl/src/test/java/org/apache/avro/idl/IdlUtilsTest.java index ef2a81d3ffd..7b3f2fd91d1 100644 --- a/lang/java/idl/src/test/java/org/apache/avro/idl/IdlUtilsTest.java +++ b/lang/java/idl/src/test/java/org/apache/avro/idl/IdlUtilsTest.java @@ -17,6 +17,7 @@ */ package org.apache.avro.idl; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -33,11 +34,15 @@ import org.apache.avro.Schema; import org.junit.jupiter.api.Test; +import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class IdlUtilsTest { @Test @@ -103,6 +108,56 @@ public void cannotWriteProtocolWithUnnamedTypes() { () -> IdlUtils.writeIdlProtocol(new StringWriter(), Schema.create(Schema.Type.STRING))); } + @Test + public void enumDefaultIsWrittenToIdl() throws IOException { + Schema withDefault = Schema.createEnum("Status", null, "naming", asList("ACTIVE", "INACTIVE"), "ACTIVE"); + Schema withoutDefault = Schema.createEnum("Status", null, "naming", asList("ACTIVE", "INACTIVE")); + + StringWriter withDefaultWriter = new StringWriter(); + IdlUtils.writeIdlProtocol(withDefaultWriter, withDefault); + StringWriter withoutDefaultWriter = new StringWriter(); + IdlUtils.writeIdlProtocol(withoutDefaultWriter, withoutDefault); + + assertTrue(withDefaultWriter.toString().contains("} = ACTIVE;"), + "Enum with default should serialize default value"); + assertFalse(withoutDefaultWriter.toString().contains("="), "Enum without default should not serialize a default"); + } + + @Test + public void enumDefaultSurvivesWriteThenParse() throws IOException { + Schema withDefault = Schema.createEnum("Status", null, "naming", asList("ACTIVE", "INACTIVE"), "ACTIVE"); + + StringWriter withDefaultWriter = new StringWriter(); + IdlUtils.writeIdlProtocol(withDefaultWriter, withDefault); + + // The written IDL must parse back into an equivalent schema: this is the bug + // that was reported. + IdlReader reader = new IdlReader(); + Schema roundTripped; + try (InputStream in = new ByteArrayInputStream(withDefaultWriter.toString().getBytes(StandardCharsets.UTF_8))) { + roundTripped = reader.parse(in).getNamedSchemas().get("naming.Status"); + } + assertEquals("ACTIVE", roundTripped.getEnumDefault()); + } + + @Test + public void enumDefaultIsReadFromIdlFile() throws IOException { + Protocol protocol = parseIdlResource("idl_utils_test_enum_default.avdl").getProtocol(); + + assertEquals("ACTIVE", protocol.getType("naming.Status").getEnumDefault()); + assertNull(protocol.getType("naming.Color").getEnumDefault()); + } + + @Test + public void idlFileWithEnumDefaultIsWrittenBackUnchanged() throws IOException { + Protocol protocol = parseIdlResource("idl_utils_test_enum_default.avdl").getProtocol(); + + StringWriter buffer = new StringWriter(); + IdlUtils.writeIdlProtocol(buffer, protocol); + + assertEquals(getResourceAsString("idl_utils_test_enum_default.avdl"), buffer.toString()); + } + @Test public void cannotWriteEmptyEnums() { assertThrows(AvroRuntimeException.class, diff --git a/lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_enum_default.avdl b/lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_enum_default.avdl new file mode 100644 index 00000000000..d2c2a0d5eb2 --- /dev/null +++ b/lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_enum_default.avdl @@ -0,0 +1,10 @@ +@namespace("naming") +protocol EnumDefaults { + enum Status { + ACTIVE, INACTIVE + } = ACTIVE; + + enum Color { + RED, GREEN + } +} From a052a75ea7e960ab61b63cb8f0d10be5daa6ac22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 17 Aug 2026 16:37:02 +0200 Subject: [PATCH 2/2] AVRO-4332: [java] Enable and fix the dormant IdlUtils tests The IdlUtils test class was never executed: Surefire only includes classes matching **/Test** (name starting with "Test"), but the class was named IdlUtilsTest, so its tests silently rotted since AVRO-3677. Rename IdlUtilsTest to TestIdlUtils so the suite runs, and fix the problems this uncovers: - byte[] values serialized to an empty string because the byte[] serializer discarded MAPPER.writeValueAsString(...) instead of writing to the generator; write the value to the generator. - The callToJson test helper had the same discard bug, so every *ToJson assertion previously compared against an empty string. - The happy-flow fixtures lived under org/apache/avro/util and were unreachable from the org.apache.avro.idl package; move them beside the test and regenerate them from the current writer output. - getMainSchema() now returns the record directly, so drop the obsolete union unwrapping in validateHappyFlowForSingleSchema. - Map/collection JSON expectations now include the ", " separator the MAPPER emits. --- .../java/org/apache/avro/idl/IdlUtils.java | 2 +- .../{IdlUtilsTest.java => TestIdlUtils.java} | 20 +++++------ .../idl_utils_test_protocol.avdl | 6 ++-- .../avro/idl/idl_utils_test_schema.avdl | 35 +++++++++++++++++++ .../avro/util/idl_utils_test_schema.avdl | 35 ------------------- 5 files changed, 50 insertions(+), 48 deletions(-) rename lang/java/idl/src/test/java/org/apache/avro/idl/{IdlUtilsTest.java => TestIdlUtils.java} (93%) rename lang/java/idl/src/test/resources/org/apache/avro/{util => idl}/idl_utils_test_protocol.avdl (90%) create mode 100644 lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_schema.avdl delete mode 100644 lang/java/idl/src/test/resources/org/apache/avro/util/idl_utils_test_schema.avdl diff --git a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlUtils.java b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlUtils.java index 80499a27093..39614f5868e 100644 --- a/lang/java/idl/src/main/java/org/apache/avro/idl/IdlUtils.java +++ b/lang/java/idl/src/main/java/org/apache/avro/idl/IdlUtils.java @@ -84,7 +84,7 @@ public void serialize(JsonProperties.Null value, JsonGenerator gen, SerializerPr module.addSerializer(new StdSerializer(byte[].class) { @Override public void serialize(byte[] value, JsonGenerator gen, SerializerProvider provider) throws IOException { - MAPPER.writeValueAsString(new String(value, StandardCharsets.ISO_8859_1)); + gen.writeString(new String(value, StandardCharsets.ISO_8859_1)); } }); diff --git a/lang/java/idl/src/test/java/org/apache/avro/idl/IdlUtilsTest.java b/lang/java/idl/src/test/java/org/apache/avro/idl/TestIdlUtils.java similarity index 93% rename from lang/java/idl/src/test/java/org/apache/avro/idl/IdlUtilsTest.java rename to lang/java/idl/src/test/java/org/apache/avro/idl/TestIdlUtils.java index 7b3f2fd91d1..6e026c1d54f 100644 --- a/lang/java/idl/src/test/java/org/apache/avro/idl/IdlUtilsTest.java +++ b/lang/java/idl/src/test/java/org/apache/avro/idl/TestIdlUtils.java @@ -27,7 +27,7 @@ import java.util.LinkedHashMap; import java.util.Map; -import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.avro.AvroRuntimeException; import org.apache.avro.JsonProperties; import org.apache.avro.Protocol; @@ -44,7 +44,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -public class IdlUtilsTest { +public class TestIdlUtils { @Test public void idlUtilsUtilitiesThrowRuntimeExceptionsOnProgrammerError() { assertThrows(IllegalStateException.class, () -> IdlUtils.getField(Object.class, "noSuchField"), "Programmer error"); @@ -97,7 +97,7 @@ public void validateHappyFlowForSingleSchema() throws IOException { Schema mainSchema = idlFile.getMainSchema(); StringWriter buffer = new StringWriter(); - IdlUtils.writeIdlSchema(buffer, mainSchema.getTypes().iterator().next()); + IdlUtils.writeIdlSchema(buffer, mainSchema); assertEquals(getResourceAsString("idl_utils_test_schema.avdl"), buffer.toString()); } @@ -181,12 +181,12 @@ public void validateMapToJson() throws IOException { Map data = new LinkedHashMap<>(); data.put("key", "name"); data.put("value", 81763); - assertEquals("{\"key\":\"name\",\"value\":81763}", callToJson(data)); + assertEquals("{\"key\":\"name\", \"value\":81763}", callToJson(data)); } @Test public void validateCollectionToJson() throws IOException { - assertEquals("[123,\"abc\"]", callToJson(Arrays.asList(123, "abc"))); + assertEquals("[123, \"abc\"]", callToJson(Arrays.asList(123, "abc"))); } @Test @@ -234,12 +234,12 @@ public void validateUnknownCannotBeWrittenAsJson() { assertThrows(AvroRuntimeException.class, () -> callToJson(new Object())); } - private String callToJson(Object datum) throws IOException { - StringWriter buffer = new StringWriter(); - try (JsonGenerator generator = IdlUtils.MAPPER.createGenerator(buffer)) { - IdlUtils.MAPPER.writeValueAsString(datum); + private String callToJson(Object datum) { + try { + return IdlUtils.MAPPER.writeValueAsString(datum); + } catch (JsonProcessingException e) { + throw new AvroRuntimeException(e); } - return buffer.toString(); } private enum SingleValue { diff --git a/lang/java/idl/src/test/resources/org/apache/avro/util/idl_utils_test_protocol.avdl b/lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_protocol.avdl similarity index 90% rename from lang/java/idl/src/test/resources/org/apache/avro/util/idl_utils_test_protocol.avdl rename to lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_protocol.avdl index fa59f5b3568..54b572d1f62 100644 --- a/lang/java/idl/src/test/resources/org/apache/avro/util/idl_utils_test_protocol.avdl +++ b/lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_protocol.avdl @@ -6,7 +6,7 @@ protocol HappyFlow { @aliases(["naming.OldMessage"]) record NewMessage { string @generator("uuid-type1") id; - @my-key("my-value") string? @aliases(["text","msg"]) message = null; + @my-key("my-value") string? @aliases(["text", "msg"]) message = null; @my-key("my-value") map @order("DESCENDING") flags; Counter mainCounter; /** A list of counters. */ @@ -20,7 +20,9 @@ protocol HappyFlow { } @namespace("common") - enum Flag {ON, OFF, CANARY} + enum Flag { + ON, OFF, CANARY + } record Counter { string name; diff --git a/lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_schema.avdl b/lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_schema.avdl new file mode 100644 index 00000000000..97fa466510e --- /dev/null +++ b/lang/java/idl/src/test/resources/org/apache/avro/idl/idl_utils_test_schema.avdl @@ -0,0 +1,35 @@ +namespace naming; + +schema NewMessage; + +/** A sample record type. */ +@version(2) +@aliases(["naming.OldMessage"]) +record NewMessage { + string @generator("uuid-type1") id; + @my-key("my-value") string? @aliases(["text", "msg"]) message = null; + @my-key("my-value") map @order("DESCENDING") flags; + Counter mainCounter; + /** A list of counters. */ + union{null, @my-key("my-value") array} otherCounters = null; + Nonce nonce; + date my_date; + time_ms my_time; + timestamp_ms my_timestamp; + decimal(12,3) my_number; + @logicalType("time-micros") long my_dummy; +} + +@namespace("common") +enum Flag { + ON, OFF, CANARY +} + +record Counter { + string name; + int count; + /** Because the Flag field is defined earlier in NewMessage, it's already defined and does not need repeating below. */ + common.Flag flag; +} + +fixed Nonce(8); diff --git a/lang/java/idl/src/test/resources/org/apache/avro/util/idl_utils_test_schema.avdl b/lang/java/idl/src/test/resources/org/apache/avro/util/idl_utils_test_schema.avdl deleted file mode 100644 index b500bde004f..00000000000 --- a/lang/java/idl/src/test/resources/org/apache/avro/util/idl_utils_test_schema.avdl +++ /dev/null @@ -1,35 +0,0 @@ -namespace naming; - -schema NewMessage; - -/** A sample record type. */ -@version(2) -@aliases(["naming.OldMessage"]) -record NewMessage { - string @generator("uuid-type1") id; - @my-key("my-value") string? @aliases(["text", "msg"]) message = null; - @my-key("my-value") map @order("DESCENDING") flags; - Counter mainCounter; - /** A list of counters. */ - union{null, @my-key("my-value") array} otherCounters = null; - Nonce nonce; - date my_date; - time_ms my_time; - timestamp_ms my_timestamp; - decimal(12,3) my_number; - @logicalType("time-micros") long my_dummy; -} - -@namespace("common") -enum Flag { - ON, OFF, CANARY -} - -record Counter { - string name; - int count; - /** Because the Flag field is defined earlier in NewMessage, it's already defined and does not need repeating below. */ - common.Flag flag; -} - -fixed Nonce(8);