diff --git a/src/it/java/io/weaviate/integration/DataITest.java b/src/it/java/io/weaviate/integration/DataITest.java index 5136aca8e..d18e0848b 100644 --- a/src/it/java/io/weaviate/integration/DataITest.java +++ b/src/it/java/io/weaviate/integration/DataITest.java @@ -455,7 +455,11 @@ public void testDataTypes() throws IOException { var types = client.collections.use(nsDataTypes); - var now = OffsetDateTime.now(); + // Truncated to the minute on purpose: OffsetDateTime.toString() used to drop + // the seconds when second and nano are both zero, producing a non-RFC3339 + // string the server rejects. A plain OffsetDateTime.now() practically never + // lands on a minute boundary, so it never caught it. + var now = OffsetDateTime.now().withSecond(0).withNano(0); var uuid = UUID.randomUUID(); Map want = Map.ofEntries( diff --git a/src/it/java/io/weaviate/integration/SearchITest.java b/src/it/java/io/weaviate/integration/SearchITest.java index 3c8d7f359..22665127e 100644 --- a/src/it/java/io/weaviate/integration/SearchITest.java +++ b/src/it/java/io/weaviate/integration/SearchITest.java @@ -722,7 +722,9 @@ public void test_filterIsNull() throws IOException { @Test public void test_filterCreateUpdateTime() throws IOException { // Arrange - var now = OffsetDateTime.now().minusHours(1); + // On a minute boundary: the filter operand used to serialize without the + // seconds, and the server failed to parse it as RFC3339. + var now = OffsetDateTime.now().minusHours(1).withSecond(0).withNano(0); var nsCounter = ns("Counter"); var counter = client.collections.create(nsCounter, diff --git a/src/main/java/io/weaviate/client6/v1/api/collections/data/InsertManyRequest.java b/src/main/java/io/weaviate/client6/v1/api/collections/data/InsertManyRequest.java index 1d2deda00..9e7052d8e 100644 --- a/src/main/java/io/weaviate/client6/v1/api/collections/data/InsertManyRequest.java +++ b/src/main/java/io/weaviate/client6/v1/api/collections/data/InsertManyRequest.java @@ -14,6 +14,7 @@ import io.weaviate.client6.v1.api.collections.GeoCoordinates; import io.weaviate.client6.v1.api.collections.PhoneNumber; import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.internal.DateUtil; import io.weaviate.client6.v1.internal.MapUtil; import io.weaviate.client6.v1.internal.grpc.ByteStringUtil; import io.weaviate.client6.v1.internal.grpc.Rpc; @@ -175,7 +176,7 @@ private static com.google.protobuf.Value marshalValue(Object value) { } else if (value instanceof UUID v) { protoValue.setStringValue(v.toString()); } else if (value instanceof OffsetDateTime v) { - protoValue.setStringValue(v.toString()); + protoValue.setStringValue(DateUtil.toRFC3339(v)); } else if (value instanceof Boolean v) { protoValue.setBoolValue(v.booleanValue()); } else if (value instanceof Number v) { @@ -208,7 +209,7 @@ private static com.google.protobuf.Value marshalValue(Object value) { } else if (listValue instanceof UUID lv) { protoListValue.setStringValue(lv.toString()); } else if (listValue instanceof OffsetDateTime lv) { - protoListValue.setStringValue(lv.toString()); + protoListValue.setStringValue(DateUtil.toRFC3339(lv)); } else if (listValue instanceof Boolean lv) { protoListValue.setBoolValue(lv); } else if (listValue instanceof Number lv) { @@ -238,7 +239,7 @@ private static com.google.protobuf.Value marshalValue(Object value) { .map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(lv.toString()).build()).toList(); } else if (value instanceof OffsetDateTime[] v) { values = Arrays.stream(v) - .map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(lv.toString()).build()).toList(); + .map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(DateUtil.toRFC3339(lv)).build()).toList(); } else if (value instanceof Boolean[] v) { values = Arrays.stream(v) .map(lv -> com.google.protobuf.Value.newBuilder().setBoolValue(lv).build()).toList(); diff --git a/src/main/java/io/weaviate/client6/v1/api/collections/query/Filter.java b/src/main/java/io/weaviate/client6/v1/api/collections/query/Filter.java index 6a2dc38a0..cb3818f0b 100644 --- a/src/main/java/io/weaviate/client6/v1/api/collections/query/Filter.java +++ b/src/main/java/io/weaviate/client6/v1/api/collections/query/Filter.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; +import io.weaviate.client6.v1.internal.DateUtil; import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase; import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase.Filters; @@ -908,7 +909,7 @@ private DateOperand(OffsetDateTime value) { @Override public void appendTo(WeaviateProtoBase.Filters.Builder filter) { - filter.setValueText(value.toString()); + filter.setValueText(DateUtil.toRFC3339(value)); } @Override @@ -930,7 +931,7 @@ private DateArrayOperand(OffsetDateTime... values) { } private List formatted() { - return values.stream().map(OffsetDateTime::toString).toList(); + return values.stream().map(DateUtil::toRFC3339).toList(); } @Override diff --git a/src/main/java/io/weaviate/client6/v1/internal/DateUtil.java b/src/main/java/io/weaviate/client6/v1/internal/DateUtil.java index b103e05e7..bc208f47a 100644 --- a/src/main/java/io/weaviate/client6/v1/internal/DateUtil.java +++ b/src/main/java/io/weaviate/client6/v1/internal/DateUtil.java @@ -2,6 +2,9 @@ import java.io.IOException; import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; import com.google.gson.Gson; import com.google.gson.TypeAdapter; @@ -11,6 +14,29 @@ import com.google.gson.stream.JsonWriter; public final class DateUtil { + /** + * RFC 3339 date-time with the seconds always present. + * + *

+ * {@link OffsetDateTime#toString()} and {@link DateTimeFormatter}'s ISO + * constants omit {@code :ss} when the second and the nanosecond are both zero, + * producing {@code 2024-03-01T00:00Z}. RFC 3339's {@code partial-time} requires + * {@code hour ":" minute ":" second}, and Weaviate rejects the shorter form, so + * the seconds are written unconditionally here. The fraction stays optional and + * variable-width so sub-second precision is neither invented nor truncated. + */ + private static final DateTimeFormatter RFC3339 = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .appendLiteral('T') + .appendValue(ChronoField.HOUR_OF_DAY, 2) + .appendLiteral(':') + .appendValue(ChronoField.MINUTE_OF_HOUR, 2) + .appendLiteral(':') + .appendValue(ChronoField.SECOND_OF_MINUTE, 2) + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .appendOffsetId() + .toFormatter(); + /** Prevent public initialization. */ private DateUtil() { } @@ -20,6 +46,17 @@ public static OffsetDateTime fromISO8601(String iso8601) { return OffsetDateTime.parse(iso8601); } + /** + * Format the timestamp for the wire as RFC 3339. + * + *

+ * Use this rather than {@link OffsetDateTime#toString()} anywhere a timestamp + * is sent to Weaviate: over REST, in a gRPC batch, or as a filter operand. + */ + public static String toRFC3339(OffsetDateTime dateTime) { + return RFC3339.format(dateTime); + } + public static enum CustomTypeAdapterFactory implements TypeAdapterFactory { INSTANCE; @@ -34,7 +71,7 @@ public TypeAdapter create(Gson gson, TypeToken type) { @Override public void write(JsonWriter out, OffsetDateTime value) throws IOException { - out.value(value.toString()); + out.value(toRFC3339(value)); } @Override diff --git a/src/test/java/io/weaviate/client6/v1/internal/Rfc3339DateTest.java b/src/test/java/io/weaviate/client6/v1/internal/Rfc3339DateTest.java new file mode 100644 index 000000000..90bf8d1a2 --- /dev/null +++ b/src/test/java/io/weaviate/client6/v1/internal/Rfc3339DateTest.java @@ -0,0 +1,127 @@ +package io.weaviate.client6.v1.internal; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.jparams.junit4.JParamsTestRunner; +import com.jparams.junit4.data.DataMethod; +import com.jparams.junit4.description.Name; + +import io.weaviate.client6.v1.api.collections.CollectionHandleDefaults; +import io.weaviate.client6.v1.api.collections.WeaviateObject; +import io.weaviate.client6.v1.api.collections.data.InsertManyRequest; +import io.weaviate.client6.v1.api.collections.query.Filter; +import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase; +import io.weaviate.client6.v1.internal.orm.CollectionDescriptor; + +/** + * Timestamps must reach the server as RFC 3339, which requires the seconds. + * + *

+ * {@link OffsetDateTime#toString()} omits them when the second and the + * nanosecond are both zero, so every timestamp on an exact minute boundary used + * to be rejected by the server. The existing date tests all seed from + * {@code OffsetDateTime.now()}, which practically never lands on one -- hence + * the literals here. + */ +@RunWith(JParamsTestRunner.class) +public class Rfc3339DateTest { + /** The value that used to serialize as "2024-03-01T00:00Z". */ + private static final OffsetDateTime ROUND = OffsetDateTime.parse("2024-03-01T00:00:00Z"); + private static final String ROUND_RFC3339 = "2024-03-01T00:00:00Z"; + + public static Object[][] timestamps() { + return new Object[][] { + { "minute boundary", ROUND, ROUND_RFC3339 }, + { "zero nanos only", OffsetDateTime.parse("2024-03-01T00:00:30Z"), "2024-03-01T00:00:30Z" }, + { "millis", OffsetDateTime.parse("2024-03-01T12:34:56.789Z"), "2024-03-01T12:34:56.789Z" }, + { "nanos", OffsetDateTime.parse("2024-03-01T12:34:56.000000001Z"), "2024-03-01T12:34:56.000000001Z" }, + { "non-UTC offset", OffsetDateTime.parse("2024-03-01T00:00:00+02:00"), "2024-03-01T00:00:00+02:00" }, + { "negative offset", OffsetDateTime.parse("2024-03-01T00:00:00-05:30"), "2024-03-01T00:00:00-05:30" }, + }; + } + + @Name("{0}") + @DataMethod(source = Rfc3339DateTest.class, method = "timestamps") + @Test + public void test_format(String __, OffsetDateTime value, String want) { + Assertions.assertThat(DateUtil.toRFC3339(value)).isEqualTo(want); + } + + /** Whatever we write has to be readable again. */ + @Name("{0}") + @DataMethod(source = Rfc3339DateTest.class, method = "timestamps") + @Test + public void test_roundTrip(String __, OffsetDateTime value, String ___) { + Assertions.assertThat(DateUtil.fromISO8601(DateUtil.toRFC3339(value))).isEqualTo(value); + } + + /** The reader stays lenient, so dates written by older clients still parse. */ + @Test + public void test_readsTheOldTruncatedForm() { + Assertions.assertThat(DateUtil.fromISO8601("2024-03-01T00:00Z")).isEqualTo(ROUND); + } + + public static Object[][] comparisons() { + return new Object[][] { + { "eq", (Function) v -> Filter.property("when").eq(v) }, + { "ne", (Function) v -> Filter.property("when").ne(v) }, + { "lt", (Function) v -> Filter.property("when").lt(v) }, + { "lte", (Function) v -> Filter.property("when").lte(v) }, + { "gt", (Function) v -> Filter.property("when").gt(v) }, + { "gte", (Function) v -> Filter.property("when").gte(v) }, + // The metadata filters take OffsetDateTime only -- no String overload to fall + // back on, so these had no workaround at all. + { "createdAt", (Function) v -> Filter.createdAt().gt(v) }, + { "lastUpdatedAt", (Function) v -> Filter.lastUpdatedAt().lt(v) }, + }; + } + + @Name("{0}") + @DataMethod(source = Rfc3339DateTest.class, method = "comparisons") + @Test + public void test_filterOperandKeepsSeconds(String __, Function build) { + Assertions.assertThat(marshal(build.apply(ROUND)).getValueText()).isEqualTo(ROUND_RFC3339); + } + + @Test + public void test_filterArrayOperandKeepsSeconds() { + var filter = Filter.property("when").containsAny(ROUND, OffsetDateTime.parse("2024-03-01T00:00:01Z")); + + Assertions.assertThat(marshal(filter).getValueTextArray().getValuesList()) + .containsExactly(ROUND_RFC3339, "2024-03-01T00:00:01Z"); + } + + @Test + public void test_insertManyKeepsSeconds() { + var properties = Map.of( + "scalar", ROUND, + "list", List.of(ROUND), + "array", new OffsetDateTime[] { ROUND }); + + var fields = InsertManyRequest.buildObject( + WeaviateObject.>of(o -> o.properties(properties)), + CollectionDescriptor.ofMap("Things"), + new CollectionHandleDefaults(Optional.empty(), Optional.empty())) + .getProperties().getNonRefProperties().getFieldsMap(); + + Assertions.assertThat(fields.get("scalar").getStringValue()).as("scalar").isEqualTo(ROUND_RFC3339); + Assertions.assertThat(fields.get("list").getListValue().getValues(0).getStringValue()) + .as("list").isEqualTo(ROUND_RFC3339); + Assertions.assertThat(fields.get("array").getListValue().getValues(0).getStringValue()) + .as("array").isEqualTo(ROUND_RFC3339); + } + + private static WeaviateProtoBase.Filters marshal(Filter filter) { + var builder = WeaviateProtoBase.Filters.newBuilder(); + filter.appendTo(builder); + return builder.build(); + } +} diff --git a/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java b/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java index e7c0d6203..2bbbb8642 100644 --- a/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java +++ b/src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java @@ -1,5 +1,6 @@ package io.weaviate.client6.v1.internal.json; +import java.time.OffsetDateTime; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -2269,6 +2270,40 @@ public static Object[][] testCases() { } """ }, + + // DateUtil.CustomTypeAdapterFactory + // + // Weaviate wants RFC 3339, which requires the seconds. OffsetDateTime's own + // toString() drops them when second and nano are both zero, so a timestamp on + // an exact minute boundary used to go out as "2024-03-01T00:00Z" and get + // rejected. + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T00:00:00Z"), + "\"2024-03-01T00:00:00Z\"", + }, + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T00:00:01Z"), + "\"2024-03-01T00:00:01Z\"", + }, + // The fraction is preserved as-is: neither invented nor truncated. + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T12:34:56.789Z"), + "\"2024-03-01T12:34:56.789Z\"", + }, + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T00:00:00.000000001Z"), + "\"2024-03-01T00:00:00.000000001Z\"", + }, + // A non-UTC offset keeps its offset rather than being normalised. + { + OffsetDateTime.class, + OffsetDateTime.parse("2024-03-01T00:00:00+02:00"), + "\"2024-03-01T00:00:00+02:00\"", + }, }; }