From 26d61444191597dcb6aa6120cd10ae20e3d5eff5 Mon Sep 17 00:00:00 2001 From: Duda Nogueira Date: Tue, 25 Aug 2026 15:51:29 -0300 Subject: [PATCH] fix: serialise OffsetDateTime as RFC 3339 OffsetDateTime.toString() omits the seconds when the second and the nanosecond are both zero, so a timestamp on an exact minute boundary went on the wire as "2024-03-01T00:00Z". RFC 3339's partial-time requires hour:minute:second, and the server rejects the short form -- breaking writes and filters alike for most timestamps written by hand. DateUtil owned the read side but had no write-side counterpart, which is why the same toString() was copy-pasted into all six marshalling sites. It now has toRFC3339(), which always writes the seconds and keeps the fraction variable-width so sub-second precision is neither invented nor truncated. The array and list variants in Filter and InsertManyRequest were affected too, not just the three scalar sites in the report. Reading is unchanged and stays lenient: OffsetDateTime.parse accepts both forms, so timestamps written by older clients still load. Closes #605 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WmY5dAGWCccWDoqkKNC2JU --- .../io/weaviate/integration/DataITest.java | 6 +- .../io/weaviate/integration/SearchITest.java | 4 +- .../collections/data/InsertManyRequest.java | 7 +- .../v1/api/collections/query/Filter.java | 5 +- .../client6/v1/internal/DateUtil.java | 39 +++++- .../client6/v1/internal/Rfc3339DateTest.java | 127 ++++++++++++++++++ .../client6/v1/internal/json/JSONTest.java | 35 +++++ 7 files changed, 215 insertions(+), 8 deletions(-) create mode 100644 src/test/java/io/weaviate/client6/v1/internal/Rfc3339DateTest.java diff --git a/src/it/java/io/weaviate/integration/DataITest.java b/src/it/java/io/weaviate/integration/DataITest.java index 5136aca8e..d18e0848b 100644 --- a/src/it/java/io/weaviate/integration/DataITest.java +++ b/src/it/java/io/weaviate/integration/DataITest.java @@ -455,7 +455,11 @@ public void testDataTypes() throws IOException { var types = client.collections.use(nsDataTypes); - var now = OffsetDateTime.now(); + // Truncated to the minute on purpose: OffsetDateTime.toString() used to drop + // the seconds when second and nano are both zero, producing a non-RFC3339 + // string the server rejects. A plain OffsetDateTime.now() practically never + // lands on a minute boundary, so it never caught it. + var now = OffsetDateTime.now().withSecond(0).withNano(0); var uuid = UUID.randomUUID(); Map want = Map.ofEntries( diff --git a/src/it/java/io/weaviate/integration/SearchITest.java b/src/it/java/io/weaviate/integration/SearchITest.java index 3c8d7f359..22665127e 100644 --- a/src/it/java/io/weaviate/integration/SearchITest.java +++ b/src/it/java/io/weaviate/integration/SearchITest.java @@ -722,7 +722,9 @@ public void test_filterIsNull() throws IOException { @Test public void test_filterCreateUpdateTime() throws IOException { // Arrange - var now = OffsetDateTime.now().minusHours(1); + // On a minute boundary: the filter operand used to serialize without the + // seconds, and the server failed to parse it as RFC3339. + var now = OffsetDateTime.now().minusHours(1).withSecond(0).withNano(0); var nsCounter = ns("Counter"); var counter = client.collections.create(nsCounter, diff --git a/src/main/java/io/weaviate/client6/v1/api/collections/data/InsertManyRequest.java b/src/main/java/io/weaviate/client6/v1/api/collections/data/InsertManyRequest.java index 1d2deda00..9e7052d8e 100644 --- a/src/main/java/io/weaviate/client6/v1/api/collections/data/InsertManyRequest.java +++ b/src/main/java/io/weaviate/client6/v1/api/collections/data/InsertManyRequest.java @@ -14,6 +14,7 @@ import io.weaviate.client6.v1.api.collections.GeoCoordinates; import io.weaviate.client6.v1.api.collections.PhoneNumber; import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.internal.DateUtil; import io.weaviate.client6.v1.internal.MapUtil; import io.weaviate.client6.v1.internal.grpc.ByteStringUtil; import io.weaviate.client6.v1.internal.grpc.Rpc; @@ -175,7 +176,7 @@ private static com.google.protobuf.Value marshalValue(Object value) { } else if (value instanceof UUID v) { protoValue.setStringValue(v.toString()); } else if (value instanceof OffsetDateTime v) { - protoValue.setStringValue(v.toString()); + protoValue.setStringValue(DateUtil.toRFC3339(v)); } else if (value instanceof Boolean v) { protoValue.setBoolValue(v.booleanValue()); } else if (value instanceof Number v) { @@ -208,7 +209,7 @@ private static com.google.protobuf.Value marshalValue(Object value) { } else if (listValue instanceof UUID lv) { protoListValue.setStringValue(lv.toString()); } else if (listValue instanceof OffsetDateTime lv) { - protoListValue.setStringValue(lv.toString()); + protoListValue.setStringValue(DateUtil.toRFC3339(lv)); } else if (listValue instanceof Boolean lv) { protoListValue.setBoolValue(lv); } else if (listValue instanceof Number lv) { @@ -238,7 +239,7 @@ private static com.google.protobuf.Value marshalValue(Object value) { .map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(lv.toString()).build()).toList(); } else if (value instanceof OffsetDateTime[] v) { values = Arrays.stream(v) - .map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(lv.toString()).build()).toList(); + .map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(DateUtil.toRFC3339(lv)).build()).toList(); } else if (value instanceof Boolean[] v) { values = Arrays.stream(v) .map(lv -> com.google.protobuf.Value.newBuilder().setBoolValue(lv).build()).toList(); diff --git a/src/main/java/io/weaviate/client6/v1/api/collections/query/Filter.java b/src/main/java/io/weaviate/client6/v1/api/collections/query/Filter.java index 6a2dc38a0..cb3818f0b 100644 --- a/src/main/java/io/weaviate/client6/v1/api/collections/query/Filter.java +++ b/src/main/java/io/weaviate/client6/v1/api/collections/query/Filter.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; +import io.weaviate.client6.v1.internal.DateUtil; import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase; import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase.Filters; @@ -908,7 +909,7 @@ private DateOperand(OffsetDateTime value) { @Override public void appendTo(WeaviateProtoBase.Filters.Builder filter) { - filter.setValueText(value.toString()); + filter.setValueText(DateUtil.toRFC3339(value)); } @Override @@ -930,7 +931,7 @@ private DateArrayOperand(OffsetDateTime... values) { } private List formatted() { - return values.stream().map(OffsetDateTime::toString).toList(); + return values.stream().map(DateUtil::toRFC3339).toList(); } @Override diff --git a/src/main/java/io/weaviate/client6/v1/internal/DateUtil.java b/src/main/java/io/weaviate/client6/v1/internal/DateUtil.java index b103e05e7..bc208f47a 100644 --- a/src/main/java/io/weaviate/client6/v1/internal/DateUtil.java +++ b/src/main/java/io/weaviate/client6/v1/internal/DateUtil.java @@ -2,6 +2,9 @@ import java.io.IOException; import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; import com.google.gson.Gson; import com.google.gson.TypeAdapter; @@ -11,6 +14,29 @@ import com.google.gson.stream.JsonWriter; public final class DateUtil { + /** + * RFC 3339 date-time with the seconds always present. + * + *

+ * {@link OffsetDateTime#toString()} and {@link DateTimeFormatter}'s ISO + * constants omit {@code :ss} when the second and the nanosecond are both zero, + * producing {@code 2024-03-01T00:00Z}. RFC 3339's {@code partial-time} requires + * {@code hour ":" minute ":" second}, and Weaviate rejects the shorter form, so + * the seconds are written unconditionally here. The fraction stays optional and + * variable-width so sub-second precision is neither invented nor truncated. + */ + private static final DateTimeFormatter RFC3339 = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .appendLiteral('T') + .appendValue(ChronoField.HOUR_OF_DAY, 2) + .appendLiteral(':') + .appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .appendLiteral(':') + .appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .appendOffsetId() + .toFormatter(); + /** Prevent public initialization. */ private DateUtil() { } @@ -20,6 +46,17 @@ public static OffsetDateTime fromISO8601(String iso8601) { return OffsetDateTime.parse(iso8601); } + /** + * Format the timestamp for the wire as RFC 3339. + * + *

+ * Use this rather than {@link OffsetDateTime#toString()} anywhere a timestamp + * is sent to Weaviate: over REST, in a gRPC batch, or as a filter operand. + */ + public static String toRFC3339(OffsetDateTime dateTime) { + return RFC3339.format(dateTime); + } + public static enum CustomTypeAdapterFactory implements TypeAdapterFactory { INSTANCE; @@ -34,7 +71,7 @@ public TypeAdapter create(Gson gson, TypeToken type) { @Override public void write(JsonWriter out, OffsetDateTime value) throws IOException { - out.value(value.toString()); + out.value(toRFC3339(value)); } @Override diff --git a/src/test/java/io/weaviate/client6/v1/internal/Rfc3339DateTest.java b/src/test/java/io/weaviate/client6/v1/internal/Rfc3339DateTest.java new file mode 100644 index 000000000..90bf8d1a2 --- /dev/null +++ b/src/test/java/io/weaviate/client6/v1/internal/Rfc3339DateTest.java @@ -0,0 +1,127 @@ +package io.weaviate.client6.v1.internal; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.jparams.junit4.JParamsTestRunner; +import com.jparams.junit4.data.DataMethod; +import com.jparams.junit4.description.Name; + +import io.weaviate.client6.v1.api.collections.CollectionHandleDefaults; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.data.InsertManyRequest; +import io.weaviate.client6.v1.api.collections.query.Filter; +import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase; +import io.weaviate.client6.v1.internal.orm.CollectionDescriptor; + +/** + * Timestamps must reach the server as RFC 3339, which requires the seconds. + * + *

+ * {@link OffsetDateTime#toString()} omits them when the second and the + * nanosecond are both zero, so every timestamp on an exact minute boundary used + * to be rejected by the server. The existing date tests all seed from + * {@code OffsetDateTime.now()}, which practically never lands on one -- hence + * the literals here. + */ +@RunWith(JParamsTestRunner.class) +public class Rfc3339DateTest { + /** The value that used to serialize as "2024-03-01T00:00Z". */ + private static final OffsetDateTime ROUND = OffsetDateTime.parse("2024-03-01T00:00:00Z"); + private static final String ROUND_RFC3339 = "2024-03-01T00:00:00Z"; + + public static Object[][] timestamps() { + return new Object[][] { + { "minute boundary", ROUND, ROUND_RFC3339 }, + { "zero nanos only", OffsetDateTime.parse("2024-03-01T00:00:30Z"), "2024-03-01T00:00:30Z" }, + { "millis", OffsetDateTime.parse("2024-03-01T12:34:56.789Z"), "2024-03-01T12:34:56.789Z" }, + { "nanos", OffsetDateTime.parse("2024-03-01T12:34:56.000000001Z"), "2024-03-01T12:34:56.000000001Z" }, + { "non-UTC offset", OffsetDateTime.parse("2024-03-01T00:00:00+02:00"), "2024-03-01T00:00:00+02:00" }, + { "negative offset", OffsetDateTime.parse("2024-03-01T00:00:00-05:30"), "2024-03-01T00:00:00-05:30" }, + }; + } + + @Name("{0}") + @DataMethod(source = Rfc3339DateTest.class, method = "timestamps") + @Test + public void test_format(String __, OffsetDateTime value, String want) { + Assertions.assertThat(DateUtil.toRFC3339(value)).isEqualTo(want); + } + + /** Whatever we write has to be readable again. */ + @Name("{0}") + @DataMethod(source = Rfc3339DateTest.class, method = "timestamps") + @Test + public void test_roundTrip(String __, OffsetDateTime value, String ___) { + Assertions.assertThat(DateUtil.fromISO8601(DateUtil.toRFC3339(value))).isEqualTo(value); + } + + /** The reader stays lenient, so dates written by older clients still parse. */ + @Test + public void test_readsTheOldTruncatedForm() { + Assertions.assertThat(DateUtil.fromISO8601("2024-03-01T00:00Z")).isEqualTo(ROUND); + } + + public static Object[][] comparisons() { + return new Object[][] { + { "eq", (Function) v -> Filter.property("when").eq(v) }, + { "ne", (Function) v -> Filter.property("when").ne(v) }, + { "lt", (Function) v -> Filter.property("when").lt(v) }, + { "lte", (Function) v -> Filter.property("when").lte(v) }, + { "gt", (Function) v -> Filter.property("when").gt(v) }, + { "gte", (Function) v -> Filter.property("when").gte(v) }, + // The metadata filters take OffsetDateTime only -- no String overload to fall + // back on, so these had no workaround at all. + { "createdAt", (Function) v -> Filter.createdAt().gt(v) }, + { "lastUpdatedAt", (Function) v -> Filter.lastUpdatedAt().lt(v) }, + }; + } + + @Name("{0}") + @DataMethod(source = Rfc3339DateTest.class, method = "comparisons") + @Test + public void test_filterOperandKeepsSeconds(String __, Function build) { + Assertions.assertThat(marshal(build.apply(ROUND)).getValueText()).isEqualTo(ROUND_RFC3339); + } + + @Test + public void test_filterArrayOperandKeepsSeconds() { + var filter = Filter.property("when").containsAny(ROUND, OffsetDateTime.parse("2024-03-01T00:00:01Z")); + + Assertions.assertThat(marshal(filter).getValueTextArray().getValuesList()) + .containsExactly(ROUND_RFC3339, "2024-03-01T00:00:01Z"); + } + + @Test + public void test_insertManyKeepsSeconds() { + var properties = Map.of( + "scalar", ROUND, + "list", List.of(ROUND), + "array", new OffsetDateTime[] { ROUND }); + + var fields = InsertManyRequest.buildObject( + WeaviateObject.>of(o -> o.properties(properties)), + CollectionDescriptor.ofMap("Things"), + new CollectionHandleDefaults(Optional.empty(), Optional.empty())) + .getProperties().getNonRefProperties().getFieldsMap(); + + Assertions.assertThat(fields.get("scalar").getStringValue()).as("scalar").isEqualTo(ROUND_RFC3339); + Assertions.assertThat(fields.get("list").getListValue().getValues(0).getStringValue()) + .as("list").isEqualTo(ROUND_RFC3339); + Assertions.assertThat(fields.get("array").getListValue().getValues(0).getStringValue()) + .as("array").isEqualTo(ROUND_RFC3339); + } + + private static WeaviateProtoBase.Filters marshal(Filter filter) { + var builder = WeaviateProtoBase.Filters.newBuilder(); + filter.appendTo(builder); + return builder.build(); + } +} diff --git a/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java b/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java index e7c0d6203..2bbbb8642 100644 --- a/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java +++ b/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java @@ -1,5 +1,6 @@ package io.weaviate.client6.v1.internal.json; +import java.time.OffsetDateTime; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -2269,6 +2270,40 @@ public static Object[][] testCases() { } """ }, + + // DateUtil.CustomTypeAdapterFactory + // + // Weaviate wants RFC 3339, which requires the seconds. OffsetDateTime's own + // toString() drops them when second and nano are both zero, so a timestamp on + // an exact minute boundary used to go out as "2024-03-01T00:00Z" and get + // rejected. + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T00:00:00Z"), + "\"2024-03-01T00:00:00Z\"", + }, + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T00:00:01Z"), + "\"2024-03-01T00:00:01Z\"", + }, + // The fraction is preserved as-is: neither invented nor truncated. + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T12:34:56.789Z"), + "\"2024-03-01T12:34:56.789Z\"", + }, + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T00:00:00.000000001Z"), + "\"2024-03-01T00:00:00.000000001Z\"", + }, + // A non-UTC offset keeps its offset rather than being normalised. + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T00:00:00+02:00"), + "\"2024-03-01T00:00:00+02:00\"", + }, }; }