diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java index a5c757d8dd361..5d8deb8ac6416 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java @@ -32,6 +32,7 @@ import org.apache.iotdb.commons.trigger.TriggerInformation; import org.apache.iotdb.commons.trigger.service.TriggerExecutableManager; import org.apache.iotdb.commons.udf.UDFInformation; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCQEntry; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq; @@ -354,6 +355,14 @@ private Set createCQs(SyncConfigNodeIServiceClient client) throws TExc "UTC", "root"); + for (TCreateCQReq req : new TCreateCQReq[] {req1, req2}) { + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + } + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.createCQ(req1).getCode()); assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.createCQ(req2).getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java index ea00bbfa646a9..4bdaa2a81bbcc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java @@ -32,6 +32,9 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.Statement; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.concurrent.TimeUnit; import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; @@ -52,6 +55,89 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void testCalendarMonthCQExecutionUsesNaturalMonthWindow() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + connection.setClientInfo("time_zone", "UTC"); + long now = System.currentTimeMillis(); + long firstExecutionTime = now + 30_000; + ZoneId utc = ZoneId.of("UTC"); + long calendarStart = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(firstExecutionTime), utc) + .minusMonths(1) + .toInstant() + .toEpochMilli(); + long thirtyDayStart = firstExecutionTime - TimeUnit.DAYS.toMillis(30); + + statement.execute("create timeseries root.sg.calendar.s1 WITH DATATYPE=INT64"); + statement.execute("create timeseries root.sg.calendar.s1_max WITH DATATYPE=INT64"); + statement.execute("INSERT INTO root.sg.calendar(time, s1) VALUES (0,0)"); + statement.execute( + String.format( + "INSERT INTO root.sg.calendar(time, s1) VALUES (%d, 777), (%d, 100), (%d, 10), (%d, 999)", + calendarStart - 1, calendarStart, firstExecutionTime - 1, firstExecutionTime)); + // On 28/29-day months the 30-day window starts before the natural month. A 30d RANGE + // would include this 888; a calendar RANGE 1mo must not. + if (thirtyDayStart < calendarStart - 1) { + statement.execute( + String.format( + "INSERT INTO root.sg.calendar(time, s1) VALUES (%d, 888)", thirtyDayStart)); + } + + statement.execute( + "CREATE CONTINUOUS QUERY cq_calendar_month\n" + + "RESAMPLE EVERY 1mo\n" + + String.format("BOUNDARY %d\n", firstExecutionTime) + + "RANGE 1mo\n" + + "BEGIN\n" + + " SELECT max_value(s1)\n" + + " INTO root.sg.calendar(s1_max)\n" + + " FROM root.sg.calendar\n" + + " GROUP BY(1mo)\n" + + "END"); + + if (System.currentTimeMillis() > firstExecutionTime) { + // Do not return silently: a vacuous pass would hide a regression in the calendar window. + statement.execute("DROP CQ cq_calendar_month"); + fail("test setup exceeded the scheduled first execution time; increase the margin"); + } + + long targetTime = firstExecutionTime + 10_000; + while (System.currentTimeMillis() - targetTime < 0) { + TimeUnit.SECONDS.sleep(1); + } + + try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.calendar")) { + boolean sawNaturalMonthBucket = false; + while (resultSet.next()) { + long time = resultSet.getLong(TIMESTAMP_STR); + long value = resultSet.getLong("root.sg.calendar.s1_max"); + assertEquals( + "CQ RANGE must stay inside the just-finished natural month", + true, + time >= calendarStart && time < firstExecutionTime); + assertEquals( + "points before/after the natural month must not contribute", true, value != 777); + assertEquals("the BOUNDARY instant is exclusive", true, value != 999); + assertEquals("a 30-day-only point must not become the monthly max", true, value != 888); + if (time == calendarStart && value == 100) { + sawNaturalMonthBucket = true; + } + } + assertEquals( + "the first GROUP BY month bucket should start at BOUNDARY-1mo", + true, + sawNaturalMonthBucket); + } finally { + statement.execute("DROP CQ cq_calendar_month"); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + @Test public void testCQExecution1() { String insertTemplate = diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java index 399694b5e2cee..3e6bf8162e28a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java @@ -36,6 +36,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -407,6 +408,63 @@ public void testCreateCorrectCQ() { } } + @Test + public void testCreateCalendarMonthCQ() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "CREATE CQ calendar_cq_1\n" + + "RESAMPLE EVERY 1mo RANGE 1mo\n" + + "BEGIN\n" + + " SELECT max_value(s1)\n" + + " INTO root.sg.d1(s1_max)\n" + + " FROM root.sg.d1\n" + + " GROUP BY(1mo)\n" + + "END"); + try (ResultSet resultSet = statement.executeQuery("show CQS")) { + boolean found = false; + while (resultSet.next()) { + if ("calendar_cq_1".equals(resultSet.getString(1))) { + found = true; + assertEquals("ACTIVE", resultSet.getString(3)); + } + } + assertTrue(found); + } + statement.execute("DROP CQ calendar_cq_1"); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testRejectIncomparableCalendarAndFixedCQDurations() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try { + statement.execute( + "CREATE CQ calendar_cq_incomparable\n" + + "RESAMPLE EVERY 1mo RANGE 30d\n" + + "BEGIN\n" + + " SELECT max_value(s1)\n" + + " INTO root.sg.d1(s1_max)\n" + + " FROM root.sg.d1\n" + + " GROUP BY(1mo)\n" + + "END"); + fail(); + } catch (Exception e) { + assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": CQ: The start time offset should be greater than or equal to every interval.", + e.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + // =======================================show cq====================================== @Test public void testShowCQ() { diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 10ea4763af9a2..c8abaf5cb43e0 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -702,4 +702,44 @@ private ManagerMessages() {} MESSAGE_ARG_PLEASE_MANUALLY_CHECK_LATER_WHETHER_THE_PROCEDURE_IS_EXECUTED_SUCCESSFULLY_A82B739D = "%s Please manually check later whether the procedure is executed successfully."; + public static final String EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26 = + "CQ EVERY duration must be positive"; + public static final String MESSAGE_CQ_START_OFFSET_MUST_BE_POSITIVE_B837C4F5 = + "CQ start offset must be positive"; + public static final String MESSAGE_CQ_END_OFFSET_MUST_BE_NON_NEGATIVE_64171164 = + "CQ end offset must be non-negative"; + public static final String MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_END_OFFSET_5924C189 = + "CQ start offset must be greater than end offset"; + public static final String + MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_DURATION_89628D43 = + "CQ start offset must be greater than or equal to EVERY duration"; + public static final String EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C = + "CQ timestamp overflows configured precision"; + public static final String + MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92 = + "Invalid CQ duration encoding; version 1 requires all structured fields"; + public static final String MESSAGE_CQ_DURATIONS_MUST_BE_NON_NEGATIVE_BE23CE04 = + "CQ durations must be non-negative"; + public static final String + MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67 = + "CQ legacy duration fields conflict with structured duration fields"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072 = + "CQ calendar duration requires all cluster nodes to support duration encoding version 1"; + public static final String MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A = + "CQ duration encoding marker is required for new requests"; + public static final String MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C = + "CQ does not have occurrence-index metadata"; + public static final String MESSAGE_CQ_OCCURRENCE_CALLBACK_IS_STALE_36C5FBFC = + "CQ occurrence callback is stale"; + public static final String MESSAGE_CQ_OCCURRENCE_INDEX_IS_AHEAD_OF_THE_CALLBACK_8A18ECC9 = + "CQ occurrence index is ahead of the callback"; + public static final String EXCEPTION_INVALID_CQ_OCCURRENCE_INDEX_TRANSITION_AC6BFC4D = + "Invalid CQ occurrence index transition"; + public static final String EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035 = + "Negative CQ snapshot entry count: %d"; + public static final String EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_B2DE4B0F = + "CQ occurrence index does not match the scheduled execution time"; + public static final String EXCEPTION_CQ_RANGE_END_MUST_BE_GREATER_THAN_START_3C91E8B4 = + "CQ RANGE end time must be greater than start time"; + } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 499e922f7a7d2..f6c8173991b51 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -681,4 +681,44 @@ private ManagerMessages() {} MESSAGE_ARG_PLEASE_MANUALLY_CHECK_LATER_WHETHER_THE_PROCEDURE_IS_EXECUTED_SUCCESSFULLY_A82B739D = "%s 请稍后手动检查该 Procedure 是否执行成功。"; + public static final String EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26 = + "CQ EVERY duration 必须为正数"; + public static final String MESSAGE_CQ_START_OFFSET_MUST_BE_POSITIVE_B837C4F5 = + "CQ start offset 必须为正数"; + public static final String MESSAGE_CQ_END_OFFSET_MUST_BE_NON_NEGATIVE_64171164 = + "CQ end offset 必须为非负数"; + public static final String MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_END_OFFSET_5924C189 = + "CQ start offset 必须大于 end offset"; + public static final String + MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_DURATION_89628D43 = + "CQ start offset 必须大于或等于 EVERY duration"; + public static final String EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C = + "CQ timestamp 超出配置的精度范围"; + public static final String + MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92 = + "无效的 CQ duration encoding；版本 1 需要所有结构化字段"; + public static final String MESSAGE_CQ_DURATIONS_MUST_BE_NON_NEGATIVE_BE23CE04 = + "CQ duration 必须为非负数"; + public static final String + MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67 = + "CQ legacy duration fields 与结构化 duration 字段冲突"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072 = + "CQ 日历 duration 要求集群所有节点支持 duration encoding version 1"; + public static final String MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A = + "新建 CQ 请求必须包含 duration encoding marker"; + public static final String MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C = + "CQ 没有 occurrence-index 元数据"; + public static final String MESSAGE_CQ_OCCURRENCE_CALLBACK_IS_STALE_36C5FBFC = + "CQ occurrence callback 已过期"; + public static final String MESSAGE_CQ_OCCURRENCE_INDEX_IS_AHEAD_OF_THE_CALLBACK_8A18ECC9 = + "CQ occurrence index 超前于 callback"; + public static final String EXCEPTION_INVALID_CQ_OCCURRENCE_INDEX_TRANSITION_AC6BFC4D = + "无效的 CQ occurrence index 转换"; + public static final String EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035 = + "CQ snapshot 条目数量不能为负数：%d"; + public static final String EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_B2DE4B0F = + "CQ occurrence index 与计划执行时间不匹配"; + public static final String EXCEPTION_CQ_RANGE_END_MUST_BE_GREATER_THAN_START_3C91E8B4 = + "CQ RANGE 结束时间必须大于开始时间"; + } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java index 62f2cc71e6926..d2894392e26bc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java @@ -59,6 +59,14 @@ protected void serializeImpl(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(nodeId, stream); ReadWriteIOUtils.write(versionInfo.getVersion(), stream); ReadWriteIOUtils.write(versionInfo.getBuildInfo(), stream); + // Optional tail keeps this plan readable by pre-calendar-duration ConfigNodes. + ReadWriteIOUtils.write(versionInfo.isSetSupportedCQDurationEncodingVersions(), stream); + if (versionInfo.isSetSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(versionInfo.getSupportedCQDurationEncodingVersions().size(), stream); + for (short version : versionInfo.getSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(version, stream); + } + } } @Override @@ -67,6 +75,17 @@ protected void deserializeImpl(ByteBuffer buffer) { versionInfo = new TNodeVersionInfo( ReadWriteIOUtils.readString(buffer), ReadWriteIOUtils.readString(buffer)); + if (buffer.hasRemaining()) { + boolean hasCapabilities = ReadWriteIOUtils.readBool(buffer); + if (hasCapabilities) { + int size = ReadWriteIOUtils.readInt(buffer); + java.util.Set capabilities = new java.util.HashSet<>(); + for (int i = 0; i < size; i++) { + capabilities.add(ReadWriteIOUtils.readShort(buffer)); + } + versionInfo.setSupportedCQDurationEncodingVersions(capabilities); + } + } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java index a487ae648e3fa..2fc3dade8cede 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java @@ -20,6 +20,7 @@ package org.apache.iotdb.confignode.consensus.request.write.cq; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.tsfile.external.commons.lang3.Validate; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -39,6 +40,10 @@ public class UpdateCQLastExecTimePlan extends ConfigPhysicalPlan { private String cqToken; + private boolean hasOccurrenceIndex; + private long expectedIndex; + private long targetIndex; + public UpdateCQLastExecTimePlan() { super(UPDATE_CQ_LAST_EXEC_TIME); } @@ -52,6 +57,18 @@ public UpdateCQLastExecTimePlan(String cqId, long executionTime, String cqToken) this.cqToken = cqToken; } + public UpdateCQLastExecTimePlan( + String cqId, long executionTime, String cqToken, long expectedIndex, long targetIndex) { + this(cqId, executionTime, cqToken); + if (expectedIndex < 0 || targetIndex <= expectedIndex) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_INVALID_CQ_OCCURRENCE_INDEX_TRANSITION_AC6BFC4D); + } + this.hasOccurrenceIndex = true; + this.expectedIndex = expectedIndex; + this.targetIndex = targetIndex; + } + public String getCqId() { return cqId; } @@ -64,12 +81,29 @@ public String getCqToken() { return cqToken; } + public boolean hasOccurrenceIndex() { + return hasOccurrenceIndex; + } + + public long getExpectedIndex() { + return expectedIndex; + } + + public long getTargetIndex() { + return targetIndex; + } + @Override protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(cqId, stream); ReadWriteIOUtils.write(executionTime, stream); ReadWriteIOUtils.write(cqToken, stream); + ReadWriteIOUtils.write(hasOccurrenceIndex, stream); + if (hasOccurrenceIndex) { + ReadWriteIOUtils.write(expectedIndex, stream); + ReadWriteIOUtils.write(targetIndex, stream); + } } @Override @@ -77,6 +111,17 @@ protected void deserializeImpl(ByteBuffer buffer) throws IOException { cqId = ReadWriteIOUtils.readString(buffer); executionTime = ReadWriteIOUtils.readLong(buffer); cqToken = ReadWriteIOUtils.readString(buffer); + if (buffer.hasRemaining()) { + hasOccurrenceIndex = ReadWriteIOUtils.readBool(buffer); + if (hasOccurrenceIndex) { + expectedIndex = ReadWriteIOUtils.readLong(buffer); + targetIndex = ReadWriteIOUtils.readLong(buffer); + if (expectedIndex < 0 || targetIndex <= expectedIndex) { + throw new IOException( + ManagerMessages.EXCEPTION_INVALID_CQ_OCCURRENCE_INDEX_TRANSITION_AC6BFC4D); + } + } + } } @Override @@ -92,12 +137,22 @@ public boolean equals(Object o) { } UpdateCQLastExecTimePlan that = (UpdateCQLastExecTimePlan) o; return executionTime == that.executionTime + && hasOccurrenceIndex == that.hasOccurrenceIndex + && expectedIndex == that.expectedIndex + && targetIndex == that.targetIndex && cqId.equals(that.cqId) && cqToken.equals(that.cqToken); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), cqId, executionTime, cqToken); + return Objects.hash( + super.hashCode(), + cqId, + executionTime, + cqToken, + hasOccurrenceIndex, + expectedIndex, + targetIndex); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java new file mode 100644 index 0000000000000..2071f7565a89f --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.confignode.manager.cq; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; +import org.apache.iotdb.confignode.i18n.ManagerMessages; + +import org.apache.tsfile.utils.TimeDuration; + +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; + +/** Pure calendar arithmetic used by CQ scheduling and recovery. */ +public final class CQCalendarUtils { + private CQCalendarUtils() {} + + public static long apply(long base, TimeDuration duration, long multiplier, ZoneId zone) { + long months = Math.multiplyExact((long) duration.monthDuration, multiplier); + long fixed = Math.multiplyExact(duration.nonMonthDuration, multiplier); + return applyVector(base, months, fixed, zone); + } + + public static long applyVector(long base, long months, long fixed, ZoneId zone) { + try { + // Fixed-only vectors must stay on elapsed-tick arithmetic. Routing them through the + // calendar helper would re-resolve local time with atZone and can shift DST-overlap + // instants even when monthPart is zero. + if (months == 0) { + return Math.addExact(base, fixed); + } + // Reuse the GROUP BY TIME helper so CQ cadence, RANGE, and month buckets share DST + // overlap/gap rules (LocalDateTime.plusMonths then atZone). + TimeDuration duration = new TimeDuration(Math.toIntExact(months), fixed); + return DateTimeUtils.calcPositiveIntervalByMonth(base, duration, zone); + } catch (ArithmeticException | DateTimeException e) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C, e); + } + } + + public static long occurrence(long boundary, TimeDuration every, long index, ZoneId zone) { + return apply(boundary, every, index, zone); + } + + public static long localEpochBoundary(ZoneId zone) { + return fromInstant(java.time.LocalDate.of(1970, 1, 1).atStartOfDay(zone).toInstant()); + } + + public static long firstOccurrenceIndex( + long boundary, TimeDuration every, long now, ZoneId zone) { + if (every.monthDuration <= 0 && every.nonMonthDuration <= 0) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26); + } + if (now <= boundary) { + return 0; + } + long high = 1; + while (occurrence(boundary, every, high, zone) < now) { + high = Math.multiplyExact(high, 2); + } + long low = high / 2; + while (low < high) { + long mid = low + (high - low) / 2; + if (occurrence(boundary, every, mid, zone) < now) { + low = mid + 1; + } else { + high = mid; + } + } + return low; + } + + private static long fromInstant(Instant instant) { + String precision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + try { + if ("us".equals(precision)) { + return Math.addExact( + Math.multiplyExact(instant.getEpochSecond(), 1_000_000L), instant.getNano() / 1_000L); + } + if ("ns".equals(precision)) { + return Math.addExact( + Math.multiplyExact(instant.getEpochSecond(), 1_000_000_000L), instant.getNano()); + } + return Math.addExact( + Math.multiplyExact(instant.getEpochSecond(), 1_000L), instant.getNano() / 1_000_000L); + } catch (ArithmeticException e) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C, e); + } + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQDurationUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQDurationUtils.java new file mode 100644 index 0000000000000..b451c92279fe3 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQDurationUtils.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.cq; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; + +import org.apache.tsfile.utils.TimeDuration; + +import java.time.ZoneId; + +/** Shared utilities for converting CQ Thrift types to structured durations and boundaries. */ +public final class CQDurationUtils { + + private CQDurationUtils() {} + + /** + * Converts a Thrift TCQDuration to a structured TimeDuration, falling back to a legacy fixed + * duration if the request predates calendar-aware encoding. + */ + public static TimeDuration toTimeDuration(TCreateCQReq req, TCQDuration d, long legacyFixed) { + if (req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1 && d != null) { + return new TimeDuration(Math.toIntExact(d.getMonthPart()), d.getNonMonthDuration()); + } + return new TimeDuration(0, legacyFixed); + } + + /** + * Resolves the calendar anchor for a CQ. If the request explicitly sets BOUNDARY, that value is + * used as-is; if omitted and the CQ is calendar-aware, the anchor becomes the local epoch + * (1970-01-01 00:00 in the persisted zone). + */ + public static long resolveBoundary(TCreateCQReq req, ZoneId zone, TimeDuration everyDuration) { + if (everyDuration.monthDuration != 0 + && req.isSetBoundaryExplicit() + && !req.isBoundaryExplicit()) { + return CQCalendarUtils.localEpochBoundary(zone); + } + return req.boundaryTime; + } + + /** + * Scales the current system time to the configured timestamp precision, producing a long suitable + * for CQ scheduling arithmetic. + */ + public static long currentTimeInPrecision() { + String precision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + long multiplier; + if ("ns".equals(precision)) { + multiplier = 1_000_000L; + } else if ("us".equals(precision)) { + multiplier = 1_000L; + } else { + multiplier = 1L; + } + return System.currentTimeMillis() * multiplier; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java index 29837bc8aa29f..eba28d39b01f7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java @@ -19,6 +19,8 @@ package org.apache.iotdb.confignode.manager.cq; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; @@ -31,8 +33,10 @@ import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.persistence.cq.CQInfo; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowCQResp; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -43,6 +47,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; @@ -75,6 +80,10 @@ public CQManager(ConfigManager configManager) { } public TSStatus createCQ(TCreateCQReq req) { + TSStatus validation = validateDurationEncoding(req); + if (validation != null) { + return validation; + } lock.readLock().lock(); try { ScheduledExecutorService currentExecutor = executor; @@ -84,6 +93,174 @@ public TSStatus createCQ(TCreateCQReq req) { } } + private TSStatus validateDurationEncoding(TCreateCQReq req) { + if (!req.isSetDurationEncodingVersion()) { + // New CQ creation must use the versioned representation. Legacy requests are still + // supported when loading old procedures/plans/snapshots, but accepting them here would let + // an old DataNode flatten a calendar duration and bypass the mixed-version capability gate. + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage(ManagerMessages.MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A); + } + if (req.getDurationEncodingVersion() != 1 + || !req.isSetEveryDuration() + || !req.isSetStartOffsetDuration() + || !req.isSetEndOffsetDuration() + || !req.isSetBoundaryExplicit()) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages + .MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92); + } + if (req.getEveryDuration().getMonthPart() < 0 + || req.getStartOffsetDuration().getMonthPart() < 0 + || req.getEndOffsetDuration().getMonthPart() < 0 + || req.getEveryDuration().getNonMonthDuration() < 0 + || req.getStartOffsetDuration().getNonMonthDuration() < 0 + || req.getEndOffsetDuration().getNonMonthDuration() < 0) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage(ManagerMessages.MESSAGE_CQ_DURATIONS_MUST_BE_NON_NEGATIVE_BE23CE04); + } + if (req.getEveryDuration().getMonthPart() > Integer.MAX_VALUE + || req.getStartOffsetDuration().getMonthPart() > Integer.MAX_VALUE + || req.getEndOffsetDuration().getMonthPart() > Integer.MAX_VALUE) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage(ManagerMessages.MESSAGE_CQ_DURATIONS_MUST_BE_NON_NEGATIVE_BE23CE04); + } + boolean hasCalendarDuration = + req.getEveryDuration().getMonthPart() != 0 + || req.getStartOffsetDuration().getMonthPart() != 0 + || req.getEndOffsetDuration().getMonthPart() != 0; + // If any structured component contains a calendar month, all legacy fields must carry the + // invalid zero sentinel. Otherwise they must exactly mirror the fixed structured values. + boolean legacyFieldsMatch = + hasCalendarDuration + ? req.everyInterval == 0 && req.startTimeOffset == 0 && req.endTimeOffset == 0 + : req.everyInterval == req.getEveryDuration().getNonMonthDuration() + && req.startTimeOffset == req.getStartOffsetDuration().getNonMonthDuration() + && req.endTimeOffset == req.getEndOffsetDuration().getNonMonthDuration(); + if (!legacyFieldsMatch) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages + .MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67); + } + TSStatus semanticValidation = validateDurationSemantics(req); + if (semanticValidation != null) { + return semanticValidation; + } + if (hasCalendarDuration && !allClusterNodesSupportDurationEncodingV1()) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages.MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072); + } + return null; + } + + private TSStatus validateDurationSemantics(TCreateCQReq req) { + TCQDuration every = req.getEveryDuration(); + TCQDuration start = req.getStartOffsetDuration(); + TCQDuration end = req.getEndOffsetDuration(); + + if (!isPositive(every)) { + return semanticError(ManagerMessages.EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26); + } + if (!isPositive(start)) { + return semanticError(ManagerMessages.MESSAGE_CQ_START_OFFSET_MUST_BE_POSITIVE_B837C4F5); + } + if (end.getMonthPart() < 0 || end.getNonMonthDuration() < 0) { + return semanticError(ManagerMessages.MESSAGE_CQ_END_OFFSET_MUST_BE_NON_NEGATIVE_64171164); + } + if (!dominates(start, end, true)) { + return semanticError( + ManagerMessages.MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_END_OFFSET_5924C189); + } + if (!dominates(start, every, false)) { + return semanticError( + ManagerMessages + .MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_DURATION_89628D43); + } + return null; + } + + private static TSStatus semanticError(String message) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()).setMessage(message); + } + + private static boolean isPositive(TCQDuration duration) { + return duration.getMonthPart() > 0 || duration.getNonMonthDuration() > 0; + } + + private static boolean dominates(TCQDuration left, TCQDuration right, boolean strict) { + boolean result = + left.getMonthPart() >= right.getMonthPart() + && left.getNonMonthDuration() >= right.getNonMonthDuration(); + return result + && (!strict + || left.getMonthPart() != right.getMonthPart() + || left.getNonMonthDuration() != right.getNonMonthDuration()); + } + + /** Returns true when any persisted CQ requires the structured calendar-duration reader. */ + public boolean hasCalendarDurationCQ() { + try { + DataSet response = configManager.getConsensusManager().read(new ShowCQPlan()); + if (!(response instanceof ShowCQResp)) { + // Do not allow a node with unknown metadata to join while the reader barrier is active. + return true; + } + if (((ShowCQResp) response).getCqList() == null) { + // A malformed response is just as unsafe as an unavailable response for this barrier. + return true; + } + return ((ShowCQResp) response) + .getCqList().stream().anyMatch(CQInfo.CQEntry::hasCalendarDuration); + } catch (ConsensusException e) { + // A failed metadata read must fail closed: an old reader must never be admitted blindly. + LOGGER.warn(ManagerMessages.UNEXPECTED_ERROR_HAPPENED_WHILE_FETCHING_CQ_LIST, e); + return true; + } + } + + private boolean allClusterNodesSupportDurationEncodingV1() { + Map versionInfo = + configManager.getNodeManager().getNodeVersionInfo(); + if (versionInfo == null || versionInfo.isEmpty()) { + return false; + } + boolean hasRegisteredNode = false; + // Check every registered node explicitly. A missing heartbeat/version entry must not allow a + // calendar CQ to be created during a rolling upgrade. + List configNodes = + configManager.getNodeManager().getRegisteredConfigNodes(); + List dataNodes = + configManager.getNodeManager().getRegisteredDataNodes(); + if (configNodes == null || dataNodes == null) { + return false; + } + for (TConfigNodeLocation node : configNodes) { + hasRegisteredNode = true; + if (!supportsDurationEncodingV1(versionInfo.get(node.getConfigNodeId()))) { + return false; + } + } + for (TDataNodeConfiguration node : dataNodes) { + hasRegisteredNode = true; + if (!supportsDurationEncodingV1(versionInfo.get(node.getLocation().getDataNodeId()))) { + return false; + } + } + return hasRegisteredNode; + } + + private boolean supportsDurationEncodingV1(TNodeVersionInfo info) { + if (info == null + || !info.isSetSupportedCQDurationEncodingVersions() + || !info.getSupportedCQDurationEncodingVersions().contains((short) 1)) { + return false; + } + return true; + } + public TSStatus dropCQ(TDropCQReq req) { lock.readLock().lock(); try { @@ -208,6 +385,32 @@ public void stopCQScheduler() { } } + /** Reconciles a callback after an ambiguous or stale progress write. */ + public void reconcileCQ(String cqId, String cqToken) { + if (!configManager.getConsensusManager().isLeader()) { + return; + } + try { + ShowCQResp response = + (ShowCQResp) configManager.getConsensusManager().read(new ShowCQPlan(cqId)); + response.getCqList().stream() + .filter(entry -> cqToken.equals(entry.getCqToken()) && entry.getState() == CQState.ACTIVE) + .findFirst() + .ifPresent( + entry -> { + // The failed callback is still registered under this token. Remove that finished + // task before installing the task rebuilt from durable progress. + unmarkCQLocallyScheduled(cqId, cqToken); + CQScheduleTask task = new CQScheduleTask(entry, executor, configManager); + if (markCQLocallyScheduled(cqId, cqToken, task)) { + task.submitSelf(); + } + }); + } catch (ConsensusException | RuntimeException e) { + LOGGER.warn(ManagerMessages.UNEXPECTED_ERROR_HAPPENED_WHILE_FETCHING_CQ_LIST, e); + } + } + public boolean markCQLocallyScheduled(String cqId, String cqToken, CQScheduleTask task) { AtomicBoolean shouldSchedule = new AtomicBoolean(false); LocallyScheduledCQ schedule = new LocallyScheduledCQ(cqToken, task); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java index 6b73ffca95fd5..86c6e26edded6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java @@ -35,9 +35,11 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.tsfile.utils.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.ZoneId; import java.util.Optional; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -83,12 +85,22 @@ public class CQScheduleTask implements Runnable { private final ConfigManager configManager; - private final long retryWaitTimeInMS; + private long retryWaitTimeInMS; private final AtomicBoolean cancelled; private final AtomicReference> scheduledFuture; private long executionTime; + private TimeDuration everyDuration; + private TimeDuration startDuration; + private TimeDuration endDuration; + private long boundaryTime; + private boolean calendarAware; + private boolean scheduleCalendarAware; + private ZoneId scheduleZone; + + /** First occurrence not yet durably acknowledged. -1 denotes a legacy CQ. */ + private long occurrenceIndex = -1; public CQScheduleTask( TCreateCQReq req, @@ -109,6 +121,44 @@ public CQScheduleTask( executor, configManager, firstExecutionTime); + this.everyDuration = + CQDurationUtils.toTimeDuration( + req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval); + this.startDuration = + CQDurationUtils.toTimeDuration( + req, + req.isSetStartOffsetDuration() ? req.getStartOffsetDuration() : null, + req.startTimeOffset); + this.endDuration = + CQDurationUtils.toTimeDuration( + req, + req.isSetEndOffsetDuration() ? req.getEndOffsetDuration() : null, + req.endTimeOffset); + this.calendarAware = + everyDuration.monthDuration != 0 + || startDuration.monthDuration != 0 + || endDuration.monthDuration != 0; + this.scheduleCalendarAware = everyDuration.monthDuration != 0; + this.boundaryTime = req.boundaryTime; + // Fixed-duration CQs do not need calendar arithmetic. Keep the zone opaque in that case so + // legacy requests containing a non-canonical zone string remain compatible. + this.scheduleZone = calendarAware ? ZoneId.of(req.zoneId) : null; + if (calendarAware) { + this.retryWaitTimeInMS = calculateRetryWaitTime(everyDuration); + } + this.boundaryTime = CQDurationUtils.resolveBoundary(req, scheduleZone, everyDuration); + // The procedure has already selected the first occurrence. Recomputing it from wall clock + // time here introduces a race around a calendar boundary and can skip an occurrence. + if (req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1) { + this.occurrenceIndex = + CQCalendarUtils.firstOccurrenceIndex( + boundaryTime, everyDuration, firstExecutionTime, scheduleZone); + long expectedExecution = occurrenceAt(this.occurrenceIndex); + if (expectedExecution != firstExecutionTime) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_B2DE4B0F); + } + } } public CQScheduleTask( @@ -126,6 +176,43 @@ public CQScheduleTask( executor, configManager, entry.getLastExecutionTime() + entry.getEveryInterval()); + this.everyDuration = entry.getEveryDuration(); + this.startDuration = entry.getStartTimeOffsetDuration(); + this.endDuration = entry.getEndTimeOffsetDuration(); + this.calendarAware = + everyDuration.monthDuration != 0 + || startDuration.monthDuration != 0 + || endDuration.monthDuration != 0; + this.scheduleCalendarAware = everyDuration.monthDuration != 0; + this.boundaryTime = entry.getBoundaryTime(); + // Fixed-duration CQs do not need calendar arithmetic. Keep the zone opaque in that case so + // legacy persisted entries containing a non-canonical zone string remain compatible. + this.scheduleZone = calendarAware ? ZoneId.of(entry.getZoneId()) : null; + // Mixed CQs (fixed EVERY + calendar RANGE) persist everyInterval as the zero sentinel. + // Retry delay must follow calendarAware, matching the create constructor; otherwise + // recovery inherits everyInterval / FACTOR == 0 and busy-spins on failure. + if (calendarAware) { + this.retryWaitTimeInMS = calculateRetryWaitTime(everyDuration); + } + if (scheduleCalendarAware) { + if (!entry.isBoundaryExplicit()) { + this.boundaryTime = CQCalendarUtils.localEpochBoundary(scheduleZone); + } + long index = + CQCalendarUtils.firstOccurrenceIndex( + boundaryTime, everyDuration, entry.getLastExecutionTime(), scheduleZone); + if (CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, scheduleZone) + <= entry.getLastExecutionTime()) { + index = Math.addExact(index, 1); + } + this.executionTime = + CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, scheduleZone); + } + if (entry.getNextOccurrenceIndex() >= 0) { + // boundaryTime was already resolved above for calendar EVERY; reuse it unchanged. + this.occurrenceIndex = entry.getNextOccurrenceIndex(); + this.executionTime = occurrenceAt(this.occurrenceIndex); + } } @SuppressWarnings("squid:S107") @@ -157,10 +244,17 @@ public CQScheduleTask( this.cancelled = new AtomicBoolean(false); this.scheduledFuture = new AtomicReference<>(); this.executionTime = executionTime; + this.everyDuration = new TimeDuration(0, everyInterval); + this.startDuration = new TimeDuration(0, startTimeOffset); + this.endDuration = new TimeDuration(0, endTimeOffset); + this.boundaryTime = 0; + // This constructor is used by the legacy fixed-duration path. Calendar-aware constructors + // initialize the zone after determining whether calendar arithmetic is required. + this.scheduleZone = null; } public static long getFirstExecutionTime(long boundaryTime, long everyInterval) { - long now = System.currentTimeMillis() * FACTOR; + long now = CQDurationUtils.currentTimeInPrecision(); return getFirstExecutionTime(boundaryTime, everyInterval, now); } @@ -172,13 +266,50 @@ public static long getFirstExecutionTime(long boundaryTime, long everyInterval, } } + public static long getFirstExecutionTime( + long boundaryTime, TimeDuration everyDuration, long now, ZoneId zoneId) { + long index = CQCalendarUtils.firstOccurrenceIndex(boundaryTime, everyDuration, now, zoneId); + return CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, zoneId); + } + @Override public void run() { + try { + runMayThrow(); + } catch (Throwable t) { + LOGGER.error(ManagerMessages.EXECUTE_CQ_FAILED, cqId, t); + if (needSubmit()) { + submitSelf(retryWaitTimeInMS, TimeUnit.MILLISECONDS); + } + } + } + + private void runMayThrow() { + long currentOccurrenceIndex = occurrenceIndex; if (cancelled.get()) { return; } long startTime = executionTime - startTimeOffset; long endTime = executionTime - endTimeOffset; + if (calendarAware) { + if (currentOccurrenceIndex < 0) { + throw new IllegalStateException( + ManagerMessages.MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C); + } + long expectedExecution = + CQCalendarUtils.occurrence( + boundaryTime, everyDuration, currentOccurrenceIndex, scheduleZone); + if (expectedExecution != executionTime) { + throw new IllegalStateException( + ManagerMessages.EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_B2DE4B0F); + } + startTime = calculateCalendarRangeEndpoint(startDuration, currentOccurrenceIndex); + endTime = calculateCalendarRangeEndpoint(endDuration, currentOccurrenceIndex); + if (endTime <= startTime) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_RANGE_END_MUST_BE_GREATER_THAN_START_3C91E8B4); + } + } Optional targetDataNode = configManager.getNodeManager().getLowestLoadDataNode(); @@ -200,12 +331,22 @@ public void run() { endTime, System.currentTimeMillis() * FACTOR); TExecuteCQ executeCQReq = - new TExecuteCQ(queryBody, startTime, endTime, everyInterval, zoneId, cqId, username); + new TExecuteCQ( + queryBody, + startTime, + endTime, + calendarAware + ? calculateCalendarTimeoutMillis(currentOccurrenceIndex) + : toTimeoutMillis(everyInterval), + zoneId, + cqId, + username); try { AsyncDataNodeInternalServiceClient client = CnToDnInternalServiceAsyncRequestManager.getInstance() .getAsyncClient(targetDataNode.get()); - client.executeCQ(executeCQReq, new AsyncExecuteCQCallback(startTime, endTime)); + client.executeCQ( + executeCQReq, new AsyncExecuteCQCallback(startTime, endTime, currentOccurrenceIndex)); } catch (Exception t) { LOGGER.warn(ManagerMessages.EXECUTE_CQ_FAILED, cqId, t); if (needSubmit()) { @@ -215,6 +356,59 @@ public void run() { } } + long calculateCalendarRangeEndpoint(TimeDuration offset, long currentOccurrenceIndex) { + // Derive every RANGE endpoint from the original boundary. Subtracting a calendar offset from + // an already materialized occurrence is not reversible at month ends and produces gaps or + // overlaps for fixed EVERY intervals combined with calendar RANGE offsets. + return CQCalendarUtils.applyVector( + boundaryTime, + Math.subtractExact( + Math.multiplyExact((long) everyDuration.monthDuration, currentOccurrenceIndex), + offset.monthDuration), + Math.subtractExact( + Math.multiplyExact(everyDuration.nonMonthDuration, currentOccurrenceIndex), + offset.nonMonthDuration), + scheduleZone); + } + + private static long toTimeoutMillis(long deltaTicks) { + if (deltaTicks <= 0) { + return 1; + } + try { + return Math.addExact(deltaTicks, FACTOR - 1) / FACTOR; + } catch (ArithmeticException e) { + return Long.MAX_VALUE; + } + } + + private long calculateCalendarTimeoutMillis(long currentOccurrenceIndex) { + try { + long nextOccurrence = + CQCalendarUtils.occurrence( + boundaryTime, everyDuration, Math.addExact(currentOccurrenceIndex, 1), scheduleZone); + return toTimeoutMillis(Math.subtractExact(nextOccurrence, executionTime)); + } catch (ArithmeticException e) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C, e); + } + } + + private static long calculateRetryWaitTime(TimeDuration duration) { + if (duration.nonMonthDuration <= 0) { + return DEFAULT_RETRY_WAIT_TIME_IN_MS; + } + return Math.min( + DEFAULT_RETRY_WAIT_TIME_IN_MS, Math.max(1L, duration.nonMonthDuration / FACTOR)); + } + + private long occurrenceAt(long index) { + // All current call sites only invoke this once occurrenceIndex has been established + // as non-negative (see the constructors and the persistProgress legacy-path guard), + // so this always resolves through calendar-aware arithmetic. + return CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, scheduleZone); + } + public void submitSelf() { submitSelf( Math.max(0, executionTime / FACTOR - System.currentTimeMillis()), TimeUnit.MILLISECONDS); @@ -253,37 +447,45 @@ private class AsyncExecuteCQCallback implements AsyncMethodCallback { private final long startTime; private final long endTime; + private final long expectedIndex; - public AsyncExecuteCQCallback(long startTime, long endTime) { + public AsyncExecuteCQCallback(long startTime, long endTime, long expectedIndex) { this.startTime = startTime; this.endTime = endTime; + this.expectedIndex = expectedIndex; + } + + private long nextOccurrenceIndex(long callbackTime) { + return calculateNextOccurrenceIndex( + timeoutPolicy, + expectedIndex, + callbackTime, + executionTime, + everyInterval, + occurrenceIndex, + boundaryTime, + everyDuration, + scheduleZone); } - private void updateExecutionTime() { + private void advanceLegacyExecutionTime(long callbackTime) { if (timeoutPolicy == TimeoutPolicy.BLOCKED) { - executionTime = executionTime + everyInterval; - } else if (timeoutPolicy == TimeoutPolicy.DISCARD) { - long now = System.currentTimeMillis() * FACTOR; - executionTime = - executionTime + ((now - executionTime - 1) / everyInterval + 1) * everyInterval; + executionTime = Math.addExact(executionTime, everyInterval); } else { - throw new IllegalArgumentException(ManagerMessages.UNKNOWN_TIMEOUTPOLICY + timeoutPolicy); + if (callbackTime <= executionTime) { + executionTime = Math.addExact(executionTime, everyInterval); + return; + } + executionTime = + Math.addExact( + executionTime, + Math.multiplyExact( + ((callbackTime - executionTime - 1) / everyInterval + 1), everyInterval)); } } - @Override - public void onComplete(TSStatus response) { - if (cancelled.get()) { - return; - } - if (response.code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - - LOGGER.info( - ManagerMessages.ENDEXECUTECQ_TIME_RANGE_IS_CURRENT_TIME_IS, - cqId, - startTime, - endTime, - System.currentTimeMillis() * FACTOR); + private void persistProgress(long targetIndex, long callbackTime) { + if (occurrenceIndex < 0) { TSStatus result; try { result = @@ -294,29 +496,75 @@ public void onComplete(TSStatus response) { result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); result.setMessage(e.getMessage()); } - - // while leadership changed, the update last exec time operation for CQTasks in new leader - // may still update failed because stale CQTask in old leader may update it in advance - if (result.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - LOGGER.warn( - ManagerMessages.FAILED_TO_UPDATE_THE_LAST_EXECUTION_TIME_OF_CQ_BECAUSE, - executionTime, - cqId, - result.getMessage()); - // no such cq, we don't need to submit it again - if (result.getCode() == TSStatusCode.NO_SUCH_CQ.getStatusCode()) { - LOGGER.info(ManagerMessages.STOP_SUBMITTING_CQ_BECAUSE, cqId, result.getMessage()); - return; + if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (needSubmit()) { + advanceLegacyExecutionTime(callbackTime); + submitSelf(); } + } else if (result.getCode() == TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + && needSubmit()) { + executor.schedule( + () -> persistProgress(targetIndex, callbackTime), + retryWaitTimeInMS, + TimeUnit.MILLISECONDS); + } else if (needSubmit()) { + configManager.getCQManager().reconcileCQ(cqId, cqToken); } - + return; + } + long targetLastExecution = occurrenceAt(targetIndex - 1); + TSStatus result; + try { + result = + configManager + .getConsensusManager() + .write( + new UpdateCQLastExecTimePlan( + cqId, targetLastExecution, cqToken, expectedIndex, targetIndex)); + } catch (ConsensusException e) { + result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + result.setMessage(e.getMessage()); + } + if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + occurrenceIndex = targetIndex; + executionTime = occurrenceAt(targetIndex); if (needSubmit()) { - updateExecutionTime(); submitSelf(); - } else { - LOGGER.info( - ManagerMessages.STOP_SUBMITTING_CQ_BECAUSE_CURRENT_NODE_IS_NOT_LEADER_OR, cqId); } + } else if (result.getCode() == TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + && needSubmit()) { + // Retry exactly the same CAS transition; never execute the query again. + executor.schedule( + () -> persistProgress(targetIndex, callbackTime), + retryWaitTimeInMS, + TimeUnit.MILLISECONDS); + } else if (needSubmit()) { + // The write may have committed before its response was lost. Re-read durable progress + // and let CQManager install the single task for the current token/index. + configManager.getCQManager().reconcileCQ(cqId, cqToken); + } + } + + @Override + public void onComplete(TSStatus response) { + if (cancelled.get()) { + return; + } + if (response.code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + + long callbackTime = System.currentTimeMillis() * FACTOR; + + LOGGER.info( + ManagerMessages.ENDEXECUTECQ_TIME_RANGE_IS_CURRENT_TIME_IS, + cqId, + startTime, + endTime, + callbackTime); + long targetIndex = + occurrenceIndex >= 0 + ? nextOccurrenceIndex(callbackTime) + : Math.addExact(expectedIndex, 1); + persistProgress(targetIndex, callbackTime); } else { LOGGER.warn(ManagerMessages.EXECUTE_CQ_FAILED_TSSTATUS_IS, cqId, response); @@ -337,4 +585,39 @@ public void onError(Exception exception) { } } } + + /** Calculates the next occurrence selected after a successful callback. */ + static long calculateNextOccurrenceIndex( + TimeoutPolicy timeoutPolicy, + long expectedIndex, + long callbackTime, + long executionTime, + long everyInterval, + long occurrenceIndex, + long boundaryTime, + TimeDuration everyDuration, + ZoneId scheduleZone) { + long next = Math.addExact(expectedIndex, 1); + if (timeoutPolicy == TimeoutPolicy.BLOCKED) { + return next; + } + if (timeoutPolicy == TimeoutPolicy.DISCARD) { + long lowerBound = + occurrenceIndex >= 0 + ? CQCalendarUtils.firstOccurrenceIndex( + boundaryTime, everyDuration, callbackTime, scheduleZone) + : calculateFixedLowerBound(expectedIndex, callbackTime, executionTime, everyInterval); + return Math.max(next, lowerBound); + } + throw new IllegalArgumentException(ManagerMessages.UNKNOWN_TIMEOUTPOLICY + timeoutPolicy); + } + + private static long calculateFixedLowerBound( + long expectedIndex, long callbackTime, long executionTime, long everyInterval) { + if (callbackTime <= executionTime) { + return expectedIndex; + } + return Math.addExact( + expectedIndex, Math.addExact((callbackTime - executionTime - 1) / everyInterval, 1)); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 64d4cf741e159..35a51b0929c49 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -331,6 +331,11 @@ private TRuntimeConfiguration getRuntimeConfiguration(int dataNodeId) { public DataSet registerDataNode(TDataNodeRegisterReq req) { DataNodeRegisterResp resp = new DataNodeRegisterResp(); resp.setConfigNodeList(getRegisteredConfigNodes()); + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + resp.setStatus(capabilityStatus); + return resp; + } // Create a new DataNodeHeartbeatCache and force update NodeStatus int dataNodeId = nodeInfo.generateNextNodeId(); @@ -371,13 +376,18 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { } public TDataNodeRestartResp updateDataNodeIfNecessary(TDataNodeRestartReq req) { + TDataNodeRestartResp resp = new TDataNodeRestartResp(); + resp.setConfigNodeList(getRegisteredConfigNodes()); + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + resp.setStatus(capabilityStatus); + return resp; + } final String clusterId = configManager .getClusterManager() .getClusterIdWithRetry( CommonDescriptor.getInstance().getConfig().getCnConnectionTimeoutInMS() / 2); - TDataNodeRestartResp resp = new TDataNodeRestartResp(); - resp.setConfigNodeList(getRegisteredConfigNodes()); if (clusterId == null) { resp.setStatus( new TSStatus(TSStatusCode.GET_CLUSTER_ID_ERROR.getStatusCode()) @@ -473,6 +483,10 @@ public DataSet removeDataNode(RemoveDataNodePlan removeDataNodePlan) { } public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + return new TConfigNodeRegisterResp().setStatus(capabilityStatus).setConfigNodeId(-1); + } int nodeId = nodeInfo.generateNextNodeId(); req.getConfigNodeLocation().setConfigNodeId(nodeId); configManager.getProcedureManager().addConfigNode(req); @@ -482,6 +496,10 @@ public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { } public TSStatus updateConfigNodeIfNecessary(int configNodeId, TNodeVersionInfo versionInfo) { + TSStatus capabilityStatus = validateDurationCapability(versionInfo); + if (capabilityStatus != null) { + return capabilityStatus; + } TNodeVersionInfo recordVersionInfo = nodeInfo.getVersionInfo(configNodeId); if (!recordVersionInfo.equals(versionInfo)) { // Update versionInfo when modified during restart @@ -496,6 +514,22 @@ public TSStatus updateConfigNodeIfNecessary(int configNodeId, TNodeVersionInfo v return ClusterNodeStartUtils.ACCEPT_NODE_RESTART; } + private TSStatus validateDurationCapability(TNodeVersionInfo versionInfo) { + if (!supportsDurationEncodingV1(versionInfo) + && configManager.getCQManager().hasCalendarDurationCQ()) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages.MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072); + } + return null; + } + + private static boolean supportsDurationEncodingV1(TNodeVersionInfo versionInfo) { + return versionInfo != null + && versionInfo.isSetSupportedCQDurationEncodingVersions() + && versionInfo.getSupportedCQDurationEncodingVersions().contains((short) 1); + } + public List getRegisteredAINodeInfoList() { List aiNodeInfoList = new ArrayList<>(); for (TAINodeConfiguration aiNodeConfiguration : getRegisteredAINodes()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java index 013e2415f9445..d2a3926efafb1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java @@ -30,22 +30,28 @@ import org.apache.iotdb.confignode.consensus.request.write.cq.UpdateCQLastExecTimePlan; import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; +import org.apache.iotdb.confignode.i18n.ManagerMessages; +import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; +import org.apache.iotdb.confignode.manager.cq.CQDurationUtils; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.concurrent.ThreadSafe; +import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; +import java.time.ZoneId; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -62,6 +68,8 @@ public class CQInfo implements SnapshotProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(CQInfo.class); private static final String SNAPSHOT_FILENAME = "cq_info.snapshot"; + // Optional tail marker. The legacy CQ records stay byte-for-byte compatible with master. + private static final int SNAPSHOT_EXTENSION_MARKER = 0x43515631; private static final String CQ_NOT_EXIST_FORMAT = "CQ %s doesn't exist."; @@ -92,11 +100,40 @@ public TSStatus addCQ(AddCQPlan plan) { res.code = TSStatusCode.CQ_ALREADY_EXIST.getStatusCode(); res.message = String.format("CQ %s has already been created.", cqId); } else { + TCreateCQReq req = plan.getReq(); + boolean versioned = + req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1; + TimeDuration everyDuration = + CQDurationUtils.toTimeDuration( + req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval); + boolean calendarAware = + versioned + && (req.getEveryDuration().getMonthPart() != 0 + || req.getStartOffsetDuration().getMonthPart() != 0 + || req.getEndOffsetDuration().getMonthPart() != 0); + // Fixed-only CQs keep the zone opaque so legacy non-canonical zone strings still load. + ZoneId zone = calendarAware ? ZoneId.of(req.zoneId) : null; + long boundary = CQDurationUtils.resolveBoundary(req, zone, everyDuration); + long firstExecutionTime = plan.getFirstExecutionTime(); + + long lastExecutionTime; + if (everyDuration.monthDuration != 0) { + long index = + CQCalendarUtils.firstOccurrenceIndex( + boundary, everyDuration, firstExecutionTime, zone); + lastExecutionTime = CQCalendarUtils.occurrence(boundary, everyDuration, index - 1, zone); + } else { + // Version 1 may carry zero legacy fields when another component is calendar-aware. Use + // the structured fixed duration to keep the persisted previous occurrence accurate. + lastExecutionTime = firstExecutionTime - everyDuration.nonMonthDuration; + } + long nextOccurrenceIndex = + versioned + ? CQCalendarUtils.firstOccurrenceIndex( + boundary, everyDuration, firstExecutionTime, zone) + : -1; CQEntry cqEntry = - new CQEntry( - plan.getReq(), - plan.getCqToken(), - plan.getFirstExecutionTime() - plan.getReq().everyInterval); + new CQEntry(plan.getReq(), plan.getCqToken(), lastExecutionTime, nextOccurrenceIndex); cqMap.put(cqId, cqEntry); res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); } @@ -213,6 +250,22 @@ public TSStatus updateCQLastExecutionTime(UpdateCQLastExecTimePlan plan) { } else if (!cqToken.equals(cqEntry.cqToken)) { res.code = TSStatusCode.NO_SUCH_CQ.getStatusCode(); res.message = String.format(CQ_TOKEN_NOT_MATCH_FORMAT, cqId); + } else if (plan.hasOccurrenceIndex()) { + if (cqEntry.nextOccurrenceIndex < 0) { + res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); + res.message = ManagerMessages.MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C; + } else if (cqEntry.nextOccurrenceIndex > plan.getExpectedIndex()) { + res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); + res.message = ManagerMessages.MESSAGE_CQ_OCCURRENCE_CALLBACK_IS_STALE_36C5FBFC; + } else if (cqEntry.nextOccurrenceIndex < plan.getExpectedIndex()) { + res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); + res.message = + ManagerMessages.MESSAGE_CQ_OCCURRENCE_INDEX_IS_AHEAD_OF_THE_CALLBACK_8A18ECC9; + } else { + cqEntry.nextOccurrenceIndex = plan.getTargetIndex(); + cqEntry.lastExecutionTime = plan.getExecutionTime(); + res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } } else if (cqEntry.lastExecutionTime >= plan.getExecutionTime()) { res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); res.message = @@ -253,16 +306,50 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept private void serialize(OutputStream stream) throws IOException { ReadWriteIOUtils.write(cqMap.size(), stream); for (CQEntry entry : cqMap.values()) { - entry.serialize(stream); + entry.serializeLegacy(stream); + } + ReadWriteIOUtils.write(SNAPSHOT_EXTENSION_MARKER, stream); + ReadWriteIOUtils.write(cqMap.size(), stream); + for (CQEntry entry : cqMap.values()) { + entry.serializeExtension(stream); } } private void deserialize(InputStream stream) throws IOException { int size = ReadWriteIOUtils.readInt(stream); + if (size < 0) { + throw new IOException( + String.format( + ManagerMessages.EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035, size)); + } for (int i = 0; i < size; i++) { - CQEntry cqEntry = CQEntry.deserialize(stream); + CQEntry cqEntry = CQEntry.deserializeLegacy(stream); cqMap.put(cqEntry.cqId, cqEntry); } + if (stream.available() < Integer.BYTES) { + // Pre-extension snapshots end after the legacy records. + return; + } + int extensionMarker = ReadWriteIOUtils.readInt(stream); + if (extensionMarker != SNAPSHOT_EXTENSION_MARKER) { + return; + } + int extensionSize = ReadWriteIOUtils.readInt(stream); + if (extensionSize < 0) { + throw new IOException( + String.format( + ManagerMessages.EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035, + extensionSize)); + } + for (int i = 0; i < extensionSize; i++) { + String cqId = ReadWriteIOUtils.readString(stream); + CQEntry cqEntry = cqMap.get(cqId); + if (cqEntry == null) { + CQEntry.skipExtension(stream); + } else { + cqEntry.deserializeExtension(stream); + } + } } @Override @@ -275,11 +362,12 @@ public void processLoadSnapshot(File snapshotDir) throws TException, IOException return; } lock.writeLock().lock(); - try (FileInputStream fileInputStream = new FileInputStream(snapshotFile)) { + try (ByteArrayInputStream inputStream = + new ByteArrayInputStream(Files.readAllBytes(snapshotFile.toPath()))) { clear(); - deserialize(fileInputStream); + deserialize(inputStream); } finally { lock.writeLock().unlock(); @@ -321,11 +409,21 @@ public static class CQEntry { private final String zoneId; private final String username; + private TimeDuration everyDuration; + private TimeDuration startTimeOffsetDuration; + private TimeDuration endTimeOffsetDuration; + private boolean boundaryExplicit; private CQState state; private long lastExecutionTime; + private long nextOccurrenceIndex; private CQEntry(TCreateCQReq req, String cqToken, long lastExecutionTime) { + this(req, cqToken, lastExecutionTime, -1); + } + + private CQEntry( + TCreateCQReq req, String cqToken, long lastExecutionTime, long nextOccurrenceIndex) { this( req.cqId, req.everyInterval, @@ -338,8 +436,20 @@ private CQEntry(TCreateCQReq req, String cqToken, long lastExecutionTime) { cqToken, req.zoneId, req.username, + CQDurationUtils.toTimeDuration( + req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval), + CQDurationUtils.toTimeDuration( + req, + req.isSetStartOffsetDuration() ? req.getStartOffsetDuration() : null, + req.startTimeOffset), + CQDurationUtils.toTimeDuration( + req, + req.isSetEndOffsetDuration() ? req.getEndOffsetDuration() : null, + req.endTimeOffset), + req.isSetBoundaryExplicit() && req.isBoundaryExplicit(), CQState.INACTIVE, - lastExecutionTime); + lastExecutionTime, + nextOccurrenceIndex); } private CQEntry(CQEntry other) { @@ -355,8 +465,13 @@ private CQEntry(CQEntry other) { other.cqToken, other.zoneId, other.username, + other.everyDuration, + other.startTimeOffsetDuration, + other.endTimeOffsetDuration, + other.boundaryExplicit, other.state, - other.lastExecutionTime); + other.lastExecutionTime, + other.nextOccurrenceIndex); } @SuppressWarnings("squid:S107") @@ -372,8 +487,13 @@ private CQEntry( String cqToken, String zoneId, String username, + TimeDuration everyDuration, + TimeDuration startTimeOffsetDuration, + TimeDuration endTimeOffsetDuration, + boolean boundaryExplicit, CQState state, - long lastExecutionTime) { + long lastExecutionTime, + long nextOccurrenceIndex) { this.cqId = cqId; this.everyInterval = everyInterval; this.boundaryTime = boundaryTime; @@ -385,11 +505,16 @@ private CQEntry( this.cqToken = cqToken; this.zoneId = zoneId; this.username = username; + this.everyDuration = everyDuration; + this.startTimeOffsetDuration = startTimeOffsetDuration; + this.endTimeOffsetDuration = endTimeOffsetDuration; + this.boundaryExplicit = boundaryExplicit; this.state = state; this.lastExecutionTime = lastExecutionTime; + this.nextOccurrenceIndex = nextOccurrenceIndex; } - private void serialize(OutputStream stream) throws IOException { + private void serializeLegacy(OutputStream stream) throws IOException { ReadWriteIOUtils.write(cqId, stream); ReadWriteIOUtils.write(everyInterval, stream); ReadWriteIOUtils.write(boundaryTime, stream); @@ -405,7 +530,19 @@ private void serialize(OutputStream stream) throws IOException { ReadWriteIOUtils.write(lastExecutionTime, stream); } - private static CQEntry deserialize(InputStream stream) throws IOException { + private void serializeExtension(OutputStream stream) throws IOException { + ReadWriteIOUtils.write(cqId, stream); + ReadWriteIOUtils.write(everyDuration.monthDuration, stream); + ReadWriteIOUtils.write(everyDuration.nonMonthDuration, stream); + ReadWriteIOUtils.write(startTimeOffsetDuration.monthDuration, stream); + ReadWriteIOUtils.write(startTimeOffsetDuration.nonMonthDuration, stream); + ReadWriteIOUtils.write(endTimeOffsetDuration.monthDuration, stream); + ReadWriteIOUtils.write(endTimeOffsetDuration.nonMonthDuration, stream); + ReadWriteIOUtils.write(boundaryExplicit, stream); + ReadWriteIOUtils.write(nextOccurrenceIndex, stream); + } + + private static CQEntry deserializeLegacy(InputStream stream) throws IOException { String cqId = ReadWriteIOUtils.readString(stream); long everyInterval = ReadWriteIOUtils.readLong(stream); long boundaryTime = ReadWriteIOUtils.readLong(stream); @@ -431,8 +568,35 @@ private static CQEntry deserialize(InputStream stream) throws IOException { cqToken, zoneId, username, + new TimeDuration(0, everyInterval), + new TimeDuration(0, startTimeOffset), + new TimeDuration(0, endTimeOffset), + false, state, - lastExecutionTime); + lastExecutionTime, + -1); + } + + private void deserializeExtension(InputStream stream) throws IOException { + everyDuration = + new TimeDuration(ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + startTimeOffsetDuration = + new TimeDuration(ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + endTimeOffsetDuration = + new TimeDuration(ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + boundaryExplicit = ReadWriteIOUtils.readBool(stream); + nextOccurrenceIndex = ReadWriteIOUtils.readLong(stream); + } + + private static void skipExtension(InputStream stream) throws IOException { + ReadWriteIOUtils.readInt(stream); + ReadWriteIOUtils.readLong(stream); + ReadWriteIOUtils.readInt(stream); + ReadWriteIOUtils.readLong(stream); + ReadWriteIOUtils.readInt(stream); + ReadWriteIOUtils.readLong(stream); + ReadWriteIOUtils.readBool(stream); + ReadWriteIOUtils.readLong(stream); } public String getCqId() { @@ -479,6 +643,10 @@ public long getLastExecutionTime() { return lastExecutionTime; } + public long getNextOccurrenceIndex() { + return nextOccurrenceIndex; + } + public String getZoneId() { return zoneId; } @@ -487,6 +655,28 @@ public String getUsername() { return username; } + public TimeDuration getEveryDuration() { + return everyDuration; + } + + public TimeDuration getStartTimeOffsetDuration() { + return startTimeOffsetDuration; + } + + public TimeDuration getEndTimeOffsetDuration() { + return endTimeOffsetDuration; + } + + public boolean isBoundaryExplicit() { + return boundaryExplicit; + } + + public boolean hasCalendarDuration() { + return everyDuration.monthDuration != 0 + || startTimeOffsetDuration.monthDuration != 0 + || endTimeOffsetDuration.monthDuration != 0; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -501,6 +691,7 @@ public boolean equals(Object o) { && startTimeOffset == cqEntry.startTimeOffset && endTimeOffset == cqEntry.endTimeOffset && lastExecutionTime == cqEntry.lastExecutionTime + && nextOccurrenceIndex == cqEntry.nextOccurrenceIndex && Objects.equals(cqId, cqEntry.cqId) && timeoutPolicy == cqEntry.timeoutPolicy && Objects.equals(queryBody, cqEntry.queryBody) @@ -508,6 +699,10 @@ public boolean equals(Object o) { && Objects.equals(cqToken, cqEntry.cqToken) && Objects.equals(zoneId, cqEntry.zoneId) && Objects.equals(username, cqEntry.username) + && Objects.equals(everyDuration, cqEntry.everyDuration) + && Objects.equals(startTimeOffsetDuration, cqEntry.startTimeOffsetDuration) + && Objects.equals(endTimeOffsetDuration, cqEntry.endTimeOffsetDuration) + && boundaryExplicit == cqEntry.boundaryExplicit && state == cqEntry.state; } @@ -525,8 +720,13 @@ public int hashCode() { cqToken, zoneId, username, + everyDuration, + startTimeOffsetDuration, + endTimeOffsetDuration, + boundaryExplicit, state, - lastExecutionTime); + lastExecutionTime, + nextOccurrenceIndex); } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java index d98a05c8a8cdc..a6878b963cbed 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java @@ -68,6 +68,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -722,6 +723,19 @@ private void serializeVersionInfo(OutputStream outputStream) throws IOException ReadWriteIOUtils.write(entry.getValue().getVersion(), outputStream); ReadWriteIOUtils.write(entry.getValue().getBuildInfo(), outputStream); } + // Optional tail; legacy snapshots end after the two strings above. + ReadWriteIOUtils.write(0x43515631, outputStream); + ReadWriteIOUtils.write(nodeVersionInfo.size(), outputStream); + for (Entry entry : nodeVersionInfo.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + Set capabilities = entry.getValue().getSupportedCQDurationEncodingVersions(); + ReadWriteIOUtils.write(capabilities == null ? 0 : capabilities.size(), outputStream); + if (capabilities != null) { + for (short capability : capabilities) { + ReadWriteIOUtils.write(capability, outputStream); + } + } + } } @Override @@ -827,6 +841,31 @@ private void deserializeBuildInfo(InputStream inputStream) throws IOException { nodeVersionInfo.put(nodeId, new TNodeVersionInfo(version, buildInfo)); size--; } + if (inputStream.available() >= Integer.BYTES) { + inputStream.mark(Integer.BYTES); + int marker = ReadWriteIOUtils.readInt(inputStream); + if (marker == 0x43515631) { + int capabilitySize = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < capabilitySize; i++) { + int nodeId = ReadWriteIOUtils.readInt(inputStream); + int sizeOfCapabilities = ReadWriteIOUtils.readInt(inputStream); + TNodeVersionInfo info = nodeVersionInfo.get(nodeId); + if (info != null) { + Set capabilities = new java.util.HashSet<>(); + for (int j = 0; j < sizeOfCapabilities; j++) { + capabilities.add(ReadWriteIOUtils.readShort(inputStream)); + } + info.setSupportedCQDurationEncodingVersions(capabilities); + } else { + for (int j = 0; j < sizeOfCapabilities; j++) { + ReadWriteIOUtils.readShort(inputStream); + } + } + } + } else { + inputStream.reset(); + } + } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java index 7066815ef141e..cf0a5f218a079 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java @@ -28,6 +28,8 @@ import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan; import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; import org.apache.iotdb.confignode.i18n.ProcedureMessages; +import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; +import org.apache.iotdb.confignode.manager.cq.CQDurationUtils; import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; import org.apache.iotdb.confignode.persistence.cq.CQInfo; @@ -41,12 +43,14 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.ZoneId; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -82,8 +86,19 @@ public CreateCQProcedure(TCreateCQReq req, ScheduledExecutorService executor) { this.req = req; this.cqToken = generateCQToken(); this.executor = executor; - this.firstExecutionTime = - CQScheduleTask.getFirstExecutionTime(req.boundaryTime, req.everyInterval); + TimeDuration everyDuration = + CQDurationUtils.toTimeDuration( + req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval); + if (everyDuration.monthDuration != 0) { + ZoneId zone = ZoneId.of(req.zoneId); + long boundary = CQDurationUtils.resolveBoundary(req, zone, everyDuration); + long now = CQDurationUtils.currentTimeInPrecision(); + long index = CQCalendarUtils.firstOccurrenceIndex(boundary, everyDuration, now, zone); + this.firstExecutionTime = CQCalendarUtils.occurrence(boundary, everyDuration, index, zone); + } else { + this.firstExecutionTime = + CQScheduleTask.getFirstExecutionTime(req.boundaryTime, everyDuration.nonMonthDuration); + } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java index a3a5ffb41de4e..ad6f7233bf1b4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java @@ -36,7 +36,9 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; /** add config node procedure */ public class AddConfigNodeProcedure extends AbstractNodeProcedure { @@ -156,6 +158,15 @@ public void serialize(DataOutputStream stream) throws IOException { ThriftConfigNodeSerDeUtils.serializeTConfigNodeLocation(tConfigNodeLocation, stream); ReadWriteIOUtils.write(versionInfo.getVersion(), stream); ReadWriteIOUtils.write(versionInfo.getBuildInfo(), stream); + // Optional tail preserves the joining node's capability across procedure recovery while old + // payloads remain readable because the tail is absent in their serialized form. + ReadWriteIOUtils.write(versionInfo.isSetSupportedCQDurationEncodingVersions(), stream); + if (versionInfo.isSetSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(versionInfo.getSupportedCQDurationEncodingVersions().size(), stream); + for (short version : versionInfo.getSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(version, stream); + } + } } @Override @@ -169,6 +180,14 @@ public void deserialize(ByteBuffer byteBuffer) { versionInfo = new TNodeVersionInfo( ReadWriteIOUtils.readString(byteBuffer), ReadWriteIOUtils.readString(byteBuffer)); + if (byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer)) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + Set capabilities = new HashSet<>(); + for (int i = 0; i < size; i++) { + capabilities.add(ReadWriteIOUtils.readShort(byteBuffer)); + } + versionInfo.setSupportedCQDurationEncodingVersions(capabilities); + } } else { versionInfo = new TNodeVersionInfo("Unknown", "Unknown"); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index cf49a3916bf2a..5bb628a82157e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -78,6 +78,7 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -97,6 +98,11 @@ public class ConfigNode extends ServerCommandLine implements ConfigNodeMBean { private static final int INIT_NON_SEED_CONFIG_NODE_ID = -1; + private static TNodeVersionInfo currentNodeVersionInfo() { + return new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + } + private static final String CONFIGURATION = "IoTDB configuration: {}"; private final String mbeanName = @@ -192,9 +198,7 @@ public void active() { TSStatus status = configManager .getNodeManager() - .updateConfigNodeIfNecessary( - configNodeId, - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + .updateConfigNodeIfNecessary(configNodeId, currentNodeVersionInfo()); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { break; } else { @@ -233,7 +237,7 @@ public void active() { .getNodeManager() .applyConfigNode( CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(SEED_CONFIG_NODE_ID), - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + currentNodeVersionInfo()); setUpMetricService(); // Notice: We always set up Seed-ConfigNode's RPC service lastly to ensure // that the external service is not provided until Seed-ConfigNode is fully initialized @@ -376,7 +380,7 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio configManager.getClusterParameters(), CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(INIT_NON_SEED_CONFIG_NODE_ID)); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo(currentNodeVersionInfo()); TEndPoint seedConfigNode = CONF.getSeedConfigNode(); if (seedConfigNode == null) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 0e8585898031e..16c4c8a76ec3d 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -191,10 +191,14 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Assert; import org.junit.Test; +import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -1866,6 +1870,42 @@ public void UpdateCQLastExecTimePlanTest() throws IOException { Assert.assertEquals(updateCQLastExecTimePlan0, updateCQLastExecTimePlan1); } + @Test + public void UpdateCQLastExecTimePlanWithOccurrenceIndexTest() throws IOException { + UpdateCQLastExecTimePlan plan0 = + new UpdateCQLastExecTimePlan("calendarCq", 1000L, "calendarToken", 5L, 6L); + UpdateCQLastExecTimePlan plan1 = + (UpdateCQLastExecTimePlan) ConfigPhysicalPlan.Factory.create(plan0.serializeToByteBuffer()); + + Assert.assertEquals(plan0, plan1); + Assert.assertTrue(plan1.hasOccurrenceIndex()); + Assert.assertEquals(5L, plan1.getExpectedIndex()); + Assert.assertEquals(6L, plan1.getTargetIndex()); + } + + @Test + public void UpdateCQLastExecTimePlanLegacyFormatWithoutIndexTest() throws IOException { + PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + // Byte layout written by pre-calendar ConfigNodes: type, cqId, executionTime, cqToken, with + // no occurrence-index tail. Ratis log replay must still accept it. + outputStream.writeShort(ConfigPhysicalPlanType.UPDATE_CQ_LAST_EXEC_TIME.getPlanType()); + ReadWriteIOUtils.write("legacyCq", outputStream); + ReadWriteIOUtils.write(2000L, outputStream); + ReadWriteIOUtils.write("legacyToken", outputStream); + + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + UpdateCQLastExecTimePlan plan = + (UpdateCQLastExecTimePlan) ConfigPhysicalPlan.Factory.create(buffer); + + Assert.assertEquals("legacyCq", plan.getCqId()); + Assert.assertEquals(2000L, plan.getExecutionTime()); + Assert.assertEquals("legacyToken", plan.getCqToken()); + Assert.assertFalse(plan.hasOccurrenceIndex()); + } + @Test public void RemoveDataNodePlanTest() throws IOException { List locations = new ArrayList<>(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java new file mode 100644 index 0000000000000..a4d183850f3f8 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.confignode; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataOutputStream; +import java.nio.ByteBuffer; +import java.util.Collections; + +public class UpdateVersionInfoPlanTest { + + @Test + public void capabilitiesRoundTrip() throws Exception { + TNodeVersionInfo versionInfo = + new TNodeVersionInfo("2.0.0", "build") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + UpdateVersionInfoPlan plan = new UpdateVersionInfoPlan(versionInfo, 3); + UpdateVersionInfoPlan restored = + (UpdateVersionInfoPlan) ConfigPhysicalPlan.Factory.create(plan.serializeToByteBuffer()); + Assert.assertEquals(plan, restored); + Assert.assertTrue( + restored.getVersionInfo().getSupportedCQDurationEncodingVersions().contains((short) 1)); + } + + @Test + public void legacyPayloadWithoutCapabilityTailStillDeserializes() throws Exception { + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write( + org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType.UpdateVersionInfo + .getPlanType(), + outputStream); + ReadWriteIOUtils.write(7, outputStream); + ReadWriteIOUtils.write("1.3.0", outputStream); + ReadWriteIOUtils.write("legacy", outputStream); + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + UpdateVersionInfoPlan restored = + (UpdateVersionInfoPlan) ConfigPhysicalPlan.Factory.create(buffer); + Assert.assertEquals(7, restored.getNodeId()); + Assert.assertEquals("1.3.0", restored.getVersionInfo().getVersion()); + Assert.assertFalse(restored.getVersionInfo().isSetSupportedCQDurationEncodingVersions()); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java index a0bc5a523ba70..71b990c75bfdd 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java @@ -24,6 +24,9 @@ import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.rpc.TSStatusCode; @@ -34,6 +37,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class CQManagerTest { @@ -82,6 +86,215 @@ public void newTokenShouldCancelPreviousLocallyScheduledTask() { } } + @Test + public void mixedCalendarAndFixedDurationsReachCapabilityValidation() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + NodeManager nodeManager = Mockito.mock(NodeManager.class); + Mockito.when(configManager.getNodeManager()).thenReturn(nodeManager); + Mockito.when(nodeManager.getNodeVersionInfo()).thenReturn(java.util.Collections.emptyMap()); + CQManager cqManager = new CQManager(configManager); + + TCreateCQReq req = + new TCreateCQReq( + "mixedDurationCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq mixedDurationCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 86_400_000)); + req.setStartOffsetDuration(new TCQDuration(1, 86_400_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + try { + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void markerlessCreateCQIsRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = + new TCreateCQReq( + "legacyCq", + 1_000, + 0, + 1_000, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq legacyCq", + "UTC", + "root"); + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void invalidStructuredDurationIsRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = + new TCreateCQReq( + "invalidDurationCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq invalidDurationCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 0)); + req.setStartOffsetDuration(new TCQDuration(0, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void unknownDurationEncodingVersionIsRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = versionedReq("unknownVersionCq", 0, 1_000, 0, 1_000); + req.setDurationEncodingVersion((short) 2); + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void partialStructuredDurationFieldsAreRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = + new TCreateCQReq( + "partialFieldsCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq partialFieldsCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setBoundaryExplicit(true); + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void calendarLegacySentinelConflictIsRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = versionedReq("sentinelConflictCq", 1, 0, 1, 0); + req.everyInterval = 2_592_000_000L; + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void fixedLegacyFieldsMustMirrorStructuredDurations() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = versionedReq("fixedConflictCq", 0, 1_000, 0, 1_000); + req.everyInterval = 2_000; + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + private static TCreateCQReq versionedReq( + String cqId, long everyMonths, long everyFixed, long startMonths, long startFixed) { + boolean calendar = everyMonths != 0 || startMonths != 0; + TCreateCQReq req = + new TCreateCQReq( + cqId, + calendar ? 0 : everyFixed, + 0, + calendar ? 0 : startFixed, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq " + cqId, + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(everyMonths, everyFixed)); + req.setStartOffsetDuration(new TCQDuration(startMonths, startFixed)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + return req; + } + @SuppressWarnings("unchecked") private CQScheduleTask newScheduledTask( ConfigManager configManager, ScheduledFuture scheduledFuture, String cqToken) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java index 44f8536ee0fdc..1097c8d6bfb84 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java @@ -18,11 +18,35 @@ */ package org.apache.iotdb.confignode.cq; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.confignode.consensus.request.read.cq.ShowCQPlan; +import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; +import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.persistence.cq.CQInfo; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.utils.TimeDuration; import org.junit.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class CQScheduleTaskTest { @@ -41,4 +65,213 @@ public void testGetFirstExecutionTime2() { long everyInterval = 30L; assertEquals(110L, CQScheduleTask.getFirstExecutionTime(boundaryTime, everyInterval, now)); } + + @Test + public void testFixedDurationCqDoesNotRequireCanonicalZoneId() { + new CQScheduleTask( + "testCq", + 1000, + 0, + 1000, + TimeoutPolicy.BLOCKED, + "select s1 into root.backup.d1.s1 from root.sg.d1", + "token", + "Asia", + "root", + null, + null, + 1000); + } + + @Test + public void testFixedDurationVersionedCqUsesLegacySchedulerPath() { + TCreateCQReq req = + new TCreateCQReq( + "fixedVersionedCq", + 1000, + 0, + 1000, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq fixedVersionedCq", + "Asia", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + // A versioned fixed-duration CQ must not enter the calendar path (which requires a ZoneId). + new CQScheduleTask(req, 1000, "token", null, null); + } + + @Test + public void testCalendarConstructorKeepsProcedureSelectedFirstOccurrence() throws Exception { + TCreateCQReq req = + new TCreateCQReq( + "calendarCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq calendarCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(false); + long firstOccurrence = + ZonedDateTime.of(2030, 2, 1, 0, 0, 0, 0, ZoneId.of("UTC")).toInstant().toEpochMilli(); + + CQScheduleTask task = new CQScheduleTask(req, firstOccurrence, "token", null, null); + Field executionTime = CQScheduleTask.class.getDeclaredField("executionTime"); + executionTime.setAccessible(true); + assertEquals(firstOccurrence, executionTime.getLong(task)); + } + + @Test(expected = IllegalArgumentException.class) + public void testCalendarConstructorRejectsMismatchedFirstOccurrence() { + TCreateCQReq req = + new TCreateCQReq( + "calendarCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq calendarCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(false); + new CQScheduleTask(req, 1L, "token", null, null); + } + + @Test + public void testCalendarOccurrencesRecomputeFromOriginalBoundary() { + ZoneId zone = ZoneId.of("UTC"); + long boundary = ZonedDateTime.of(2024, 1, 31, 0, 0, 0, 0, zone).toInstant().toEpochMilli(); + TimeDuration month = new TimeDuration(1, 0); + assertEquals( + ZonedDateTime.of(2024, 2, 29, 0, 0, 0, 0, zone).toInstant().toEpochMilli(), + CQCalendarUtils.occurrence(boundary, month, 1, zone)); + assertEquals( + ZonedDateTime.of(2024, 3, 31, 0, 0, 0, 0, zone).toInstant().toEpochMilli(), + CQCalendarUtils.occurrence(boundary, month, 2, zone)); + } + + @Test + public void testDiscardLowerBoundNeverMovesBeforeCurrentOccurrence() { + ZoneId zone = ZoneId.of("UTC"); + long boundary = ZonedDateTime.of(2024, 1, 1, 0, 0, 0, 0, zone).toInstant().toEpochMilli(); + TimeDuration month = new TimeDuration(1, 0); + long current = CQCalendarUtils.occurrence(boundary, month, 2, zone); + long lowerBound = CQCalendarUtils.firstOccurrenceIndex(boundary, month, current, zone); + assertEquals(2, lowerBound); + assertEquals(3, Math.max(2 + 1, lowerBound)); + } + + @Test + public void recoveredMixedCalendarRangeKeepsPositiveRetryWait() throws Exception { + TCreateCQReq req = + new TCreateCQReq( + "mixedRetryCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq mixedRetryCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1_000)); + req.setStartOffsetDuration(new TCQDuration(1, 1_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + CQInfo cqInfo = new CQInfo(); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "mixedRetryToken", 10_000)).getCode()); + CQInfo.CQEntry entry = cqInfo.showCQ(new ShowCQPlan("mixedRetryCq")).getCqList().get(0); + + CQScheduleTask recovered = new CQScheduleTask(entry, null, null); + Field retryWaitTimeInMS = CQScheduleTask.class.getDeclaredField("retryWaitTimeInMS"); + retryWaitTimeInMS.setAccessible(true); + // Structured EVERY is 1000ms; the persisted legacy everyInterval is the zero sentinel. + assertEquals(1_000L, retryWaitTimeInMS.getLong(recovered)); + } + + @Test + public void staleLastExecUpdateOnLegacyCqReconciles() throws Exception { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + ConsensusManager consensusManager = Mockito.mock(ConsensusManager.class); + CQManager cqManager = Mockito.mock(CQManager.class); + Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); + Mockito.when(consensusManager.isLeader()).thenReturn(true); + Mockito.when(consensusManager.write(Mockito.any())) + .thenReturn(new TSStatus(TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode())); + + ScheduledExecutorService executor = Mockito.mock(ScheduledExecutorService.class); + Mockito.when(executor.isShutdown()).thenReturn(false); + ScheduledFuture future = Mockito.mock(ScheduledFuture.class); + Mockito.when( + executor.schedule( + Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.any(TimeUnit.class))) + .thenReturn((ScheduledFuture) future); + + CQScheduleTask task = + new CQScheduleTask( + "legacyCq", + 1000, + 0, + 1000, + TimeoutPolicy.BLOCKED, + "select 1", + "token", + "Asia", + "root", + executor, + configManager, + 10_000); + + Field occurrenceIndex = CQScheduleTask.class.getDeclaredField("occurrenceIndex"); + occurrenceIndex.setAccessible(true); + assertEquals(-1L, occurrenceIndex.getLong(task)); + + Class callbackClass = null; + for (Class nested : CQScheduleTask.class.getDeclaredClasses()) { + if (nested.getSimpleName().equals("AsyncExecuteCQCallback")) { + callbackClass = nested; + break; + } + } + assertNotNull(callbackClass); + Constructor constructor = + callbackClass.getDeclaredConstructor( + CQScheduleTask.class, long.class, long.class, long.class); + constructor.setAccessible(true); + Object callback = constructor.newInstance(task, 0L, 1000L, 0L); + Method onComplete = callbackClass.getMethod("onComplete", TSStatus.class); + onComplete.setAccessible(true); + onComplete.invoke(callback, new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + Mockito.verify(consensusManager, Mockito.times(1)).write(Mockito.any()); + Mockito.verify(cqManager).reconcileCQ("legacyCq", "token"); + Mockito.verify(executor, Mockito.never()) + .schedule(Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.any(TimeUnit.class)); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java new file mode 100644 index 0000000000000..122f0e4fd81c9 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.confignode.manager.cq; + +import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; + +import org.apache.tsfile.utils.TimeDuration; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class CQCalendarUtilsTest { + + private static final ZoneId UTC = ZoneId.of("UTC"); + + @Test + public void testMonthEndSequenceIsAnchoredToOriginalBoundary() { + long boundary = epochTimestamp(2024, 1, 31, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + + assertEquals( + epochTimestamp(2024, 2, 29, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, every, 1, UTC)); + assertEquals( + epochTimestamp(2024, 3, 31, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, every, 2, UTC)); + assertEquals( + epochTimestamp(2024, 4, 30, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, every, 3, UTC)); + } + + @Test + public void testLeapDayYearSequenceReachesNextLeapYear() { + long boundary = epochTimestamp(2020, 2, 29, 0, 0, UTC); + TimeDuration everyYear = new TimeDuration(12, 0); + + assertEquals( + epochTimestamp(2021, 2, 28, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, everyYear, 1, UTC)); + assertEquals( + epochTimestamp(2024, 2, 29, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, everyYear, 4, UTC)); + } + + @Test + public void testOmittedBoundaryUsesLocalEpochInPersistedZone() { + ZoneId shanghai = ZoneId.of("Asia/Shanghai"); + assertEquals( + epochTimestamp(1970, 1, 1, 0, 0, shanghai), CQCalendarUtils.localEpochBoundary(shanghai)); + } + + @Test + public void testRangeEndpointsAreDerivedFromBoundaryVector() { + long boundary = epochTimestamp(2024, 1, 31, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long start = CQCalendarUtils.applyVector(boundary, 1, 0, UTC); + long end = CQCalendarUtils.applyVector(boundary, 2, 0, UTC); + + assertEquals(epochTimestamp(2024, 2, 29, 0, 0, UTC), start); + assertEquals(epochTimestamp(2024, 3, 31, 0, 0, UTC), end); + assertEquals(end, CQCalendarUtils.occurrence(boundary, every, 2, UTC)); + } + + @Test + public void testFixedCadenceRangeUsesCurrentCalendarMonth() { + long day = TimestampPrecisionUtils.currPrecision.convert(1, TimeUnit.DAYS); + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + long execution = epochTimestamp(2024, 4, 1, 0, 0, UTC); + TimeDuration start = new TimeDuration(1, day); + TimeDuration end = new TimeDuration(1, 0); + CQScheduleTask task = + calendarTask(boundary, execution, new TimeDuration(0, day), start, end, UTC); + + // EVERY 1d RANGE 1mo1d, 1mo passes the component-wise duration validation. Its two + // offsets must use March's calendar boundary, rather than January's 31-day length. + assertEquals( + epochTimestamp(2024, 2, 29, 0, 0, UTC), task.calculateCalendarRangeEndpoint(start, 91)); + assertEquals( + epochTimestamp(2024, 3, 1, 0, 0, UTC), task.calculateCalendarRangeEndpoint(end, 91)); + + // The calculation must remain anchored to the original boundary even when the fixed cadence + // has not reached a month boundary. Subtracting RANGE from the occurrence would yield Jan 29. + long marchFirst = epochTimestamp(2024, 3, 1, 0, 0, UTC); + task = calendarTask(boundary, marchFirst, new TimeDuration(0, day), start, end, UTC); + assertEquals( + epochTimestamp(2024, 1, 29, 0, 0, UTC), task.calculateCalendarRangeEndpoint(start, 60)); + } + + @Test + public void testFixedCadenceRangePreservesMonthClampingAndDst() { + long day = TimestampPrecisionUtils.currPrecision.convert(1, TimeUnit.DAYS); + TimeDuration every = new TimeDuration(0, day); + TimeDuration start = new TimeDuration(1, day); + TimeDuration end = new TimeDuration(0, 0); + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + long execution = epochTimestamp(2024, 3, 31, 0, 0, UTC); + CQScheduleTask task = calendarTask(boundary, execution, every, start, end, UTC); + assertEquals( + epochTimestamp(2024, 2, 28, 0, 0, UTC), task.calculateCalendarRangeEndpoint(start, 90)); + assertEquals(execution, task.calculateCalendarRangeEndpoint(end, 90)); + + ZoneId newYork = ZoneId.of("America/New_York"); + boundary = epochTimestamp(2024, 1, 1, 11, 0, newYork); + execution = epochTimestamp(2024, 4, 10, 12, 0, newYork); + task = calendarTask(boundary, execution, every, start, end, newYork); + // Subtracting one calendar month reaches March 10 at noon; subtracting another 24 hours + // crosses the spring DST transition and reaches March 9 at 11:00. + assertEquals( + epochTimestamp(2024, 3, 9, 11, 0, newYork), + task.calculateCalendarRangeEndpoint(start, 100)); + } + + @Test + public void testCalendarCadenceRangeRetainsOriginalMonthEndAnchor() { + long boundary = epochTimestamp(2024, 1, 31, 0, 0, UTC); + long execution = epochTimestamp(2024, 4, 30, 0, 0, UTC); + TimeDuration month = new TimeDuration(1, 0); + TimeDuration zero = new TimeDuration(0, 0); + CQScheduleTask task = calendarTask(boundary, execution, month, month, zero, UTC); + + assertEquals( + epochTimestamp(2024, 3, 31, 0, 0, UTC), task.calculateCalendarRangeEndpoint(month, 3)); + assertEquals(execution, task.calculateCalendarRangeEndpoint(zero, 3)); + } + + @Test + public void testFirstOccurrenceRangeIsTheJustFinishedNaturalMonth() { + long boundary = epochTimestamp(2024, 3, 1, 0, 0, UTC); + TimeDuration month = new TimeDuration(1, 0); + TimeDuration zero = new TimeDuration(0, 0); + CQScheduleTask task = calendarTask(boundary, boundary, month, month, zero, UTC); + + // Occurrence 0 is the user-visible first fire. RANGE 1mo must look backward from the original + // boundary, including the negative month vector, rather than subtracting from a later clamped + // occurrence. + assertEquals( + epochTimestamp(2024, 2, 1, 0, 0, UTC), task.calculateCalendarRangeEndpoint(month, 0)); + assertEquals(boundary, task.calculateCalendarRangeEndpoint(zero, 0)); + assertEquals( + epochTimestamp(2024, 2, 1, 0, 0, UTC), CQCalendarUtils.applyVector(boundary, -1, 0, UTC)); + } + + @Test + public void testMonthEndFirstOccurrenceRangeClampsFebruary() { + long boundary = epochTimestamp(2024, 3, 31, 0, 0, UTC); + TimeDuration month = new TimeDuration(1, 0); + TimeDuration zero = new TimeDuration(0, 0); + CQScheduleTask task = calendarTask(boundary, boundary, month, month, zero, UTC); + + assertEquals( + epochTimestamp(2024, 2, 29, 0, 0, UTC), task.calculateCalendarRangeEndpoint(month, 0)); + assertEquals(boundary, task.calculateCalendarRangeEndpoint(zero, 0)); + } + + @Test + public void testCalendarTimeoutUsesActualAdjacentOccurrenceDistance() throws Exception { + long boundary = epochTimestamp(2024, 1, 31, 0, 0, UTC); + TimeDuration month = new TimeDuration(1, 0); + TimeDuration zero = new TimeDuration(0, 0); + java.lang.reflect.Method timeout = + CQScheduleTask.class.getDeclaredMethod("calculateCalendarTimeoutMillis", long.class); + timeout.setAccessible(true); + + long januaryToFebruary = epochTimestamp(2024, 2, 29, 0, 0, UTC) - boundary; + long februaryToMarch = + epochTimestamp(2024, 3, 31, 0, 0, UTC) - epochTimestamp(2024, 2, 29, 0, 0, UTC); + CQScheduleTask first = calendarTask(boundary, boundary, month, month, zero, UTC); + CQScheduleTask second = + calendarTask(boundary, epochTimestamp(2024, 2, 29, 0, 0, UTC), month, month, zero, UTC); + // n=0 spans 29 days (Jan 31 -> Feb 29); n=1 spans 31 days (Feb 29 -> Mar 31). Neither is 30d. + assertEquals(januaryToFebruary, timeout.invoke(first, 0L)); + assertEquals(februaryToMarch, timeout.invoke(second, 1L)); + } + + @Test + public void testCheckedCalendarArithmeticRejectsOverflow() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + try { + CQCalendarUtils.apply(boundary, new TimeDuration(1, 0), Integer.MAX_VALUE + 1L, UTC); + org.junit.Assert.fail("expected month multiplication to overflow"); + } catch (IllegalArgumentException e) { + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C, + e.getMessage()); + } + } + + private static CQScheduleTask calendarTask( + long boundary, + long execution, + TimeDuration every, + TimeDuration start, + TimeDuration end, + ZoneId zone) { + TCreateCQReq req = + new TCreateCQReq( + "rangeCq", + 0, + boundary, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select s1 into root.backup.d1.s1 from root.sg.d1", + "create cq rangeCq", + zone.getId(), + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(every.monthDuration, every.nonMonthDuration)); + req.setStartOffsetDuration(new TCQDuration(start.monthDuration, start.nonMonthDuration)); + req.setEndOffsetDuration(new TCQDuration(end.monthDuration, end.nonMonthDuration)); + req.setBoundaryExplicit(true); + return new CQScheduleTask(req, execution, "token", null, null); + } + + @Test + public void testCalendarOccurrenceMatchesGroupByTimeHelper() { + ZoneId newYork = ZoneId.of("America/New_York"); + long boundary = epochTimestamp(2024, 1, 3, 1, 30, newYork); + TimeDuration tenMonths = new TimeDuration(10, 0); + long cqOccurrence = CQCalendarUtils.occurrence(boundary, tenMonths, 1, newYork); + long groupBy = + org.apache.iotdb.commons.queryengine.utils.DateTimeUtils.calcPositiveIntervalByMonth( + boundary, tenMonths, newYork); + assertEquals(groupBy, cqOccurrence); + // DST overlap: GROUP BY / atZone selects the earlier offset (-04:00 = 05:30Z). + assertEquals(epochTimestamp(2024, 11, 3, 1, 30, newYork), cqOccurrence); + } + + @Test + public void testDstUsesZoneRulesForCalendarAndElapsedParts() { + ZoneId newYork = ZoneId.of("America/New_York"); + long monthBoundary = epochTimestamp(2024, 2, 10, 2, 30, newYork); + long monthOccurrence = + CQCalendarUtils.occurrence(monthBoundary, new TimeDuration(1, 0), 1, newYork); + assertEquals(epochTimestamp(2024, 3, 10, 3, 30, newYork), monthOccurrence); + + long elapsedBoundary = epochTimestamp(2024, 3, 9, 12, 0, newYork); + long oneDay = TimestampPrecisionUtils.currPrecision.convert(1, TimeUnit.DAYS); + long elapsedOccurrence = + CQCalendarUtils.occurrence(elapsedBoundary, new TimeDuration(0, oneDay), 1, newYork); + assertEquals(epochTimestamp(2024, 3, 10, 13, 0, newYork), elapsedOccurrence); + } + + @Test + public void testBlockedKeepsTheNextOccurrenceWhileDiscardSkipsMissedOccurrences() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long callbackTime = epochTimestamp(2024, 1, 4, 12, 0, UTC); + long executionTime = CQCalendarUtils.occurrence(boundary, every, 1, UTC); + + assertEquals( + 2, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.BLOCKED, 1, callbackTime, executionTime, 0, 1, boundary, every, UTC)); + assertEquals( + 2, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.DISCARD, 1, callbackTime, executionTime, 0, 1, boundary, every, UTC)); + } + + @Test + public void testDiscardNeverMovesBeforeTheCurrentOccurrence() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long currentOccurrence = CQCalendarUtils.occurrence(boundary, every, 2, UTC); + + assertEquals( + 3, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.DISCARD, + 2, + currentOccurrence, + currentOccurrence, + 0, + 2, + boundary, + every, + UTC)); + } + + @Test + public void testDiscardSkipsMultipleMissedCalendarOccurrences() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long executionTime = CQCalendarUtils.occurrence(boundary, every, 1, UTC); + long callbackTime = epochTimestamp(2024, 5, 15, 12, 0, UTC); + + // Missed March/April/May. The next durable index is June (n=5), not February+1. + assertEquals( + 5, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.DISCARD, 1, callbackTime, executionTime, 0, 1, boundary, every, UTC)); + assertEquals( + 2, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.BLOCKED, 1, callbackTime, executionTime, 0, 1, boundary, every, UTC)); + } + + @Test + public void testDiscardClockRollbackKeepsTheNextOccurrence() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long executionTime = CQCalendarUtils.occurrence(boundary, every, 2, UTC); + long callbackTime = epochTimestamp(2023, 12, 1, 0, 0, UTC); + + assertEquals( + 3, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.DISCARD, 2, callbackTime, executionTime, 0, 2, boundary, every, UTC)); + } + + @Test + public void testExplicitBoundaryZeroStaysUnixEpochWhileOmittedBoundaryUsesLocalEpoch() + throws Exception { + ZoneId shanghai = ZoneId.of("Asia/Shanghai"); + long unixEpoch = epochTimestamp(1970, 1, 1, 0, 0, UTC); + long shanghaiLocalEpoch = epochTimestamp(1970, 1, 1, 0, 0, shanghai); + // The two anchors differ by exactly the zone offset; a flipped explicit-flag check would + // silently shift every occurrence of a BOUNDARY 0 CQ by 8 hours in Asia/Shanghai. + assertEquals( + TimestampPrecisionUtils.currPrecision.convert(8, TimeUnit.HOURS), + unixEpoch - shanghaiLocalEpoch); + + // Explicit BOUNDARY 0: the anchor is the Unix epoch instant, observed in the CQ zone. + CQScheduleTask explicitTask = + new CQScheduleTask(shanghaiMonthlyReq(true), unixEpoch, "token", null, null); + assertEquals(0L, boundaryTimeOf(explicitTask)); + assertEquals(unixEpoch, executionTimeOf(explicitTask)); + + // Omitted BOUNDARY: the anchor is local 1970-01-01 00:00 in the persisted CQ zone. + CQScheduleTask omittedTask = + new CQScheduleTask(shanghaiMonthlyReq(false), shanghaiLocalEpoch, "token", null, null); + assertEquals(shanghaiLocalEpoch, boundaryTimeOf(omittedTask)); + assertEquals(shanghaiLocalEpoch, executionTimeOf(omittedTask)); + assertEquals(CQCalendarUtils.localEpochBoundary(shanghai), boundaryTimeOf(omittedTask)); + + // Cross-wiring the two anchors must be rejected by the occurrence/execution-time check. + assertThrows( + IllegalArgumentException.class, + () -> new CQScheduleTask(shanghaiMonthlyReq(true), shanghaiLocalEpoch, "t", null, null)); + assertThrows( + IllegalArgumentException.class, + () -> new CQScheduleTask(shanghaiMonthlyReq(false), unixEpoch, "t", null, null)); + } + + private static TCreateCQReq shanghaiMonthlyReq(boolean boundaryExplicit) { + TCreateCQReq req = + new TCreateCQReq( + "boundaryCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq boundaryCq", + "Asia/Shanghai", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(boundaryExplicit); + return req; + } + + private static long boundaryTimeOf(CQScheduleTask task) throws Exception { + Field field = CQScheduleTask.class.getDeclaredField("boundaryTime"); + field.setAccessible(true); + return field.getLong(task); + } + + private static long executionTimeOf(CQScheduleTask task) throws Exception { + Field field = CQScheduleTask.class.getDeclaredField("executionTime"); + field.setAccessible(true); + return field.getLong(task); + } + + private static long epochTimestamp( + int year, int month, int day, int hour, int minute, ZoneId zone) { + Instant instant = ZonedDateTime.of(year, month, day, hour, minute, 0, 0, zone).toInstant(); + return Math.addExact( + TimestampPrecisionUtils.currPrecision.convert(instant.getEpochSecond(), TimeUnit.SECONDS), + TimestampPrecisionUtils.currPrecision.convert(instant.getNano(), TimeUnit.NANOSECONDS)); + } + + private static long epochTimestamp(int year, int month, int day, int hour, ZoneId zone) { + return epochTimestamp(year, month, day, hour, 0, zone); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java index f39ecd46d5c6a..60434905b4180 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java @@ -20,16 +20,25 @@ package org.apache.iotdb.confignode.manager.node; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; +import org.apache.iotdb.confignode.consensus.response.datanode.DataNodeRegisterResp; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; +import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.consensus.IConsensus; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -57,6 +66,8 @@ public class NodeManagerTest { private IConsensus consensus; private ConsensusManager consensusManager; + private NodeInfo nodeInfo; + private CQManager cqManager; private NodeManager nodeManager; private int originalCnConnectionTimeout; private int originalTransferLeaderTimeout; @@ -73,10 +84,12 @@ public void setUp() { consensus = Mockito.mock(IConsensus.class); consensusManager = Mockito.mock(ConsensusManager.class); IManager configManager = Mockito.mock(IManager.class); - NodeInfo nodeInfo = Mockito.mock(NodeInfo.class); + nodeInfo = Mockito.mock(NodeInfo.class); LoadManager loadManager = Mockito.mock(LoadManager.class); + cqManager = Mockito.mock(CQManager.class); Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); Mockito.when(configManager.getLoadManager()).thenReturn(loadManager); Mockito.when(consensusManager.getConsensusImpl()).thenReturn(consensus); Mockito.when(consensusManager.getConsensusGroupId()) @@ -126,6 +139,64 @@ public void transferLeaderShouldTryAnotherCandidateAfterTransientFailure() throw peerCaptor.getAllValues().stream().map(Peer::getNodeId).collect(Collectors.toList())); } + @Test + public void configNodeWithoutCalendarDurationCapabilityIsRejectedWhileCalendarCQExists() { + TNodeVersionInfo unsupportedVersion = new TNodeVersionInfo("old", "old"); + TConfigNodeRegisterReq registerReq = + new TConfigNodeRegisterReq().setConfigNodeLocation(firstCandidate); + registerReq.setVersionInfo(unsupportedVersion); + + // The CQ manager is mocked in setUp; make the persisted metadata barrier active for this test. + Mockito.when(cqManager.hasCalendarDurationCQ()).thenReturn(true); + + TConfigNodeRegisterResp registerResp = nodeManager.registerConfigNode(registerReq); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), registerResp.getStatus().getCode()); + Assert.assertEquals(-1, registerResp.getConfigNodeId()); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + nodeManager + .updateConfigNodeIfNecessary(firstCandidate.getConfigNodeId(), unsupportedVersion) + .getCode()); + Mockito.verify(nodeInfo, Mockito.never()).generateNextNodeId(); + } + + @Test + public void dataNodeWithoutCalendarDurationCapabilityIsRejectedWhileCalendarCQExists() { + TNodeVersionInfo unsupportedVersion = new TNodeVersionInfo("old", "old"); + TDataNodeLocation location = + new TDataNodeLocation( + 11, + new TEndPoint("127.0.0.1", 6667), + new TEndPoint("127.0.0.1", 10730), + new TEndPoint("127.0.0.1", 10740), + new TEndPoint("127.0.0.1", 10750), + new TEndPoint("127.0.0.1", 10760)); + TDataNodeConfiguration configuration = new TDataNodeConfiguration().setLocation(location); + TDataNodeRegisterReq registerReq = + new TDataNodeRegisterReq() + .setClusterName("cluster") + .setDataNodeConfiguration(configuration) + .setVersionInfo(unsupportedVersion); + + Mockito.when(cqManager.hasCalendarDurationCQ()).thenReturn(true); + + DataNodeRegisterResp registerResp = + (DataNodeRegisterResp) nodeManager.registerDataNode(registerReq); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), registerResp.getStatus().getCode()); + Mockito.verify(nodeInfo, Mockito.never()).generateNextNodeId(); + + TDataNodeRestartReq restartReq = + new TDataNodeRestartReq() + .setClusterName("cluster") + .setDataNodeConfiguration(configuration) + .setVersionInfo(unsupportedVersion); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + nodeManager.updateDataNodeIfNecessary(restartReq).getStatus().getCode()); + } + @Test public void transferLeaderShouldRedirectToActualLeaderAfterFailedResponse() throws Exception { Mockito.when(consensusManager.getLeaderLocation()) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java index 64bbd69c5b6a7..ccca5bc035035 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.confignode.consensus.request.write.cq.UpdateCQLastExecTimePlan; import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; import org.apache.iotdb.confignode.persistence.cq.CQInfo; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.rpc.TSStatusCode; @@ -36,6 +37,10 @@ import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.time.ZoneId; +import java.time.ZonedDateTime; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -159,4 +164,275 @@ public void testShowCQCanFilterByCQId() throws Exception { Assert.assertEquals(1, showCQResp.getCqList().size()); Assert.assertEquals("testCq4", showCQResp.getCqList().get(0).getCqId()); } + + @Test + public void testOccurrenceIndexCasFencesOutOfOrderCallbacks() { + TCreateCQReq req = + new TCreateCQReq( + "indexedCq", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select 1", + "create cq indexedCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "indexedToken", 1000)).getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("indexedCq", 1000, "indexedToken", 1, 2)) + .getCode()); + Assert.assertEquals( + TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("indexedCq", 2000, "indexedToken", 1, 2)) + .getCode()); + } + + @Test + public void testOccurrenceIndexCasRejectsAheadCallbackAndTokenMismatch() { + TCreateCQReq req = + new TCreateCQReq( + "fencedCq", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select 1", + "create cq fencedCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "fencedToken", 1000)).getCode()); + + Assert.assertEquals( + TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("fencedCq", 2000, "fencedToken", 2, 3)) + .getCode()); + Assert.assertEquals( + TSStatusCode.NO_SUCH_CQ.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("fencedCq", 1000, "otherToken", 1, 2)) + .getCode()); + } + + @Test + public void testCommittedProgressRetryIsDetectedAsStaleForReconciliation() throws TException { + TCreateCQReq req = + new TCreateCQReq( + "lostAckCq", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select 1", + "create cq lostAckCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "lostAckToken", 1000)).getCode()); + UpdateCQLastExecTimePlan advance = + new UpdateCQLastExecTimePlan("lostAckCq", 1000, "lostAckToken", 1, 2); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.updateCQLastExecutionTime(advance).getCode()); + // A retry after the first response was lost must be recognized as stale and reconciled. + Assert.assertEquals( + TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(), + cqInfo.updateCQLastExecutionTime(advance).getCode()); + } + + @Test + public void testMixedCalendarDurationUsesStructuredFixedEveryForLastExecution() { + TCreateCQReq req = + new TCreateCQReq( + "mixedFixedEveryCq", + 0, + 0, + 0, + 0, + (byte) 0, + "select 1", + "create cq mixedFixedEveryCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1_000)); + req.setStartOffsetDuration(new TCQDuration(1, 1_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "mixedFixedEveryToken", 10_000)).getCode()); + + CQInfo.CQEntry entry = cqInfo.showCQ(new ShowCQPlan("mixedFixedEveryCq")).getCqList().get(0); + Assert.assertEquals(9_000, entry.getLastExecutionTime()); + Assert.assertEquals(10, entry.getNextOccurrenceIndex()); + } + + @Test + public void testFixedVersionedDurationKeepsLegacyZoneOpaque() { + TCreateCQReq req = + new TCreateCQReq( + "fixedVersionedCq", + 1_000, + 0, + 1_000, + 0, + (byte) 0, + "select 1", + "create cq fixedVersionedCq", + "Asia", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1_000)); + req.setStartOffsetDuration(new TCQDuration(0, 1_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(false); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "fixedVersionedToken", 10_000)).getCode()); + } + + @Test + public void testCalendarOccurrenceProgressSurvivesSnapshotRecovery() throws Exception { + File calendarSnapshotDir = new File(BASE_OUTPUT_PATH, "snapshot-calendar"); + if (calendarSnapshotDir.exists()) { + FileUtils.deleteDirectory(calendarSnapshotDir); + } + calendarSnapshotDir.mkdirs(); + + ZoneId zone = ZoneId.of("UTC"); + long boundary = ZonedDateTime.of(2024, 1, 31, 0, 0, 0, 0, zone).toInstant().toEpochMilli(); + long firstOccurrence = + ZonedDateTime.of(2024, 2, 29, 0, 0, 0, 0, zone).toInstant().toEpochMilli(); + TCreateCQReq req = + new TCreateCQReq( + "snapshotCalendarCq", + 0, + boundary, + 0, + 0, + (byte) 0, + "select 1", + "create cq snapshotCalendarCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "snapshotCalendarToken", firstOccurrence)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan( + "snapshotCalendarCq", firstOccurrence, "snapshotCalendarToken", 1, 2)) + .getCode()); + + cqInfo.processTakeSnapshot(calendarSnapshotDir); + CQInfo restored = new CQInfo(); + restored.processLoadSnapshot(calendarSnapshotDir); + + ShowCQResp response = restored.showCQ(new ShowCQPlan("snapshotCalendarCq")); + Assert.assertEquals(1, response.getCqList().size()); + Assert.assertEquals(2, response.getCqList().get(0).getNextOccurrenceIndex()); + Assert.assertEquals(firstOccurrence, response.getCqList().get(0).getLastExecutionTime()); + FileUtils.deleteDirectory(calendarSnapshotDir); + } + + @Test + public void testPreExtensionSnapshotLoadsAsLegacyCq() throws Exception { + File legacyDir = new File(BASE_OUTPUT_PATH, "snapshot-pre-extension"); + if (legacyDir.exists()) { + FileUtils.deleteDirectory(legacyDir); + } + legacyDir.mkdirs(); + try { + TCreateCQReq req = + new TCreateCQReq( + "legacySnapshotCq", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select 1", + "create cq legacySnapshotCq", + "Asia", + "root"); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "legacySnapshotToken", 10_000)).getCode()); + Assert.assertTrue(cqInfo.processTakeSnapshot(legacyDir)); + + File snapshotFile = new File(legacyDir, "cq_info.snapshot"); + byte[] bytes = Files.readAllBytes(snapshotFile.toPath()); + byte[] marker = new byte[] {0x43, 0x51, 0x56, 0x31}; + int markerAt = indexOf(bytes, marker); + Assert.assertTrue(markerAt >= 0); + try (RandomAccessFile raf = new RandomAccessFile(snapshotFile, "rw")) { + raf.setLength(markerAt); + } + + CQInfo restored = new CQInfo(); + restored.processLoadSnapshot(legacyDir); + ShowCQResp response = restored.showCQ(new ShowCQPlan("legacySnapshotCq")); + Assert.assertEquals(1, response.getCqList().size()); + Assert.assertEquals("legacySnapshotCq", response.getCqList().get(0).getCqId()); + Assert.assertEquals(-1, response.getCqList().get(0).getNextOccurrenceIndex()); + } finally { + FileUtils.deleteDirectory(legacyDir); + } + } + + private static int indexOf(byte[] haystack, byte[] needle) { + outer: + for (int i = 0; i <= haystack.length - needle.length; i++) { + for (int j = 0; j < needle.length; j++) { + if (haystack[i + j] != needle[j]) { + continue outer; + } + } + return i; + } + return -1; + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java index e3ced5b069428..3b2a74634b5bd 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java @@ -26,8 +26,10 @@ import org.apache.iotdb.common.rpc.thrift.TNodeResource; import org.apache.iotdb.commons.exception.StartupException; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; @@ -38,6 +40,7 @@ import java.io.File; import java.io.IOException; +import java.util.Collections; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -73,6 +76,35 @@ public void testSnapshot() throws TException, IOException { Assert.assertEquals(nodeInfo, nodeInfo1); } + @Test + public void testSnapshotPreservesDurationEncodingCapabilities() throws TException, IOException { + File capabilitySnapshotDir = new File(BASE_OUTPUT_PATH, "snapshot-node-capabilities"); + if (capabilitySnapshotDir.exists()) { + FileUtils.deleteDirectory(capabilitySnapshotDir); + } + capabilitySnapshotDir.mkdirs(); + try { + registerConfigNodes(); + registerDataNodes(); + TNodeVersionInfo versionInfo = + new TNodeVersionInfo("2.0.0", "build") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + nodeInfo.updateVersionInfo(new UpdateVersionInfoPlan(versionInfo, 10000)); + Assert.assertTrue(nodeInfo.processTakeSnapshot(capabilitySnapshotDir)); + + NodeInfo restored = new NodeInfo(); + restored.processLoadSnapshot(capabilitySnapshotDir); + Assert.assertEquals(nodeInfo, restored); + Assert.assertTrue( + restored + .getVersionInfo(10000) + .getSupportedCQDurationEncodingVersions() + .contains((short) 1)); + } finally { + FileUtils.deleteDirectory(capabilitySnapshotDir); + } + } + private void registerConfigNodes() { for (int i = 0; i < 3; i++) { ApplyConfigNodePlan applyConfigNodePlan = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java index 3e7fd2052ad53..9e35935d59554 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.service.ConfigNode; @@ -66,6 +67,34 @@ public void tokenShouldBeUniqueForSameCQId() { } } + @Test + public void mixedCalendarRangeWithFixedEveryUsesStructuredEveryDuration() { + TCreateCQReq req = + new TCreateCQReq( + "mixedFixedEveryCq", + 0, + 0, + 0, + 0, + (byte) 0, + "select s1 into root.backup.d1(s1) from root.sg.d1", + "create cq mixedFixedEveryCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1_000)); + req.setStartOffsetDuration(new TCQDuration(1, 1_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + try { + new CreateCQProcedure(req, executor); + } finally { + executor.shutdown(); + } + } + @Test public void serializeDeserializeTest() { @@ -111,4 +140,53 @@ public void serializeDeserializeTest() { executor.shutdown(); } } + + @Test + public void calendarCqWithOmittedBoundarySurvivesSerializeDeserialize() { + PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + TCreateCQReq req = + new TCreateCQReq( + "calendarCq", + 0, + 0, + 0, + 0, + (byte) 0, + "select s1 into root.backup.d1(s1) from root.sg.d1", + "create cq calendarCq", + "Asia/Shanghai", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(false); + + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + CreateCQProcedure procedure1 = new CreateCQProcedure(req, executor); + + CQManager cqManager = Mockito.mock(CQManager.class); + Mockito.when(cqManager.getExecutor()).thenReturn(executor); + ConfigManager configManager = Mockito.mock(ConfigManager.class); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); + ConfigNode configNode = new ConfigNode(); + configNode.setConfigManager(configManager); + + try { + procedure1.serialize(outputStream); + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + + CreateCQProcedure procedure2 = + (CreateCQProcedure) ProcedureFactory.getInstance().create(buffer); + assertEquals(procedure1, procedure2); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + executor.shutdown(); + } + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java index 682b478134664..835e21dca9c00 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java @@ -31,6 +31,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; public class AddConfigNodeProcedureTest { @@ -40,7 +41,8 @@ public void serDeTest() throws IOException { new AddConfigNodeProcedure( new TConfigNodeLocation( 0, new TEndPoint("127.0.0.1", 10710), new TEndPoint("0.0.0.0", 10720)), - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { @@ -50,4 +52,21 @@ public void serDeTest() throws IOException { Assert.assertEquals(procedure0, ProcedureFactory.getInstance().create(buffer)); } } + + @Test + public void deserializeLegacyPayloadWithoutCapabilities() throws IOException { + AddConfigNodeProcedure procedure0 = + new AddConfigNodeProcedure( + new TConfigNodeLocation( + 0, new TEndPoint("127.0.0.1", 10710), new TEndPoint("0.0.0.0", 10720)), + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + procedure0.serialize(outputStream); + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size() - 1); + Assert.assertEquals(procedure0, ProcedureFactory.getInstance().create(buffer)); + } + } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 2c25ba7460643..f3ea7245a5ee8 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -1229,6 +1229,16 @@ public final class DataNodeQueryMessages { "CQ: The start time offset should be greater than 0."; public static final String CQ_THE_END_TIME_OFFSET_SHOULD_BE_GREATER = "CQ: The end time offset should be greater than or equal to 0."; + public static final String EXCEPTION_CQ_DURATION_CANNOT_BE_EMPTY_C7269AB2 = + "CQ duration cannot be empty"; + public static final String EXCEPTION_INVALID_CQ_DURATION_ARG_F4917D5C = + "Invalid CQ duration: %s"; + public static final String EXCEPTION_CQ_DURATION_COMPONENT_OVERFLOWS_ARG_ED5B0962 = + "CQ duration component overflows: %s"; + public static final String EXCEPTION_CQ_DURATION_MONTH_COMPONENT_OVERFLOWS_ARG_EBF2A2B6 = + "CQ duration month component overflows: %s"; + public static final String EXCEPTION_CQ_EVERY_INTERVAL_MUST_BE_POSITIVE_26259019 = + "CQ: The every interval should be greater than 0."; public static final String CQ_THE_QUERY_BODY_MISSES_AN_INTO_CLAUSE = "CQ: The query body misses an INTO clause."; public static final String CQ_SPECIFYING_TIME_FILTERS_IN_THE_QUERY_BODY = @@ -1934,6 +1944,8 @@ public final class DataNodeQueryMessages { "Sorting by device is only supported in ALIGN BY DEVICE queries."; public static final String CQ_EVERY_INTERVAL_D_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL = "CQ: Every interval [%d] should not be lower than the `continuous_query_minimum_every_interval` [%d] configured."; + public static final String CQ_EVERY_INTERVAL_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL = + "CQ: Every interval [%s] should not be lower than the `continuous_query_minimum_every_interval` [%d] configured."; public static final String CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_END_TIME_OFFSET = "CQ: The start time offset should be greater than end time offset."; public static final String CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_INTERVAL = @@ -3877,4 +3889,6 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "Only InMemoryDeviceEntryDataSet supports get inline device entries"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3 = + "CQ calendar duration cannot be created until all cluster nodes support duration encoding version 1"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index c6a55ebad3e06..efea296f6d57b 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -1211,6 +1211,16 @@ public final class DataNodeQueryMessages { "连续查询：起始时间偏移量应大于 0。"; public static final String CQ_THE_END_TIME_OFFSET_SHOULD_BE_GREATER = "连续查询：结束时间偏移量应大于或等于 0。"; + public static final String EXCEPTION_CQ_DURATION_CANNOT_BE_EMPTY_C7269AB2 = + "CQ duration 不能为空"; + public static final String EXCEPTION_INVALID_CQ_DURATION_ARG_F4917D5C = + "无效的 CQ duration：%s"; + public static final String EXCEPTION_CQ_DURATION_COMPONENT_OVERFLOWS_ARG_ED5B0962 = + "CQ duration 组件溢出：%s"; + public static final String EXCEPTION_CQ_DURATION_MONTH_COMPONENT_OVERFLOWS_ARG_EBF2A2B6 = + "CQ duration 月份组件溢出：%s"; + public static final String EXCEPTION_CQ_EVERY_INTERVAL_MUST_BE_POSITIVE_26259019 = + "CQ：every interval 应大于 0。"; public static final String CQ_THE_QUERY_BODY_MISSES_AN_INTO_CLAUSE = "连续查询：查询体缺少 INTO 子句。"; public static final String CQ_SPECIFYING_TIME_FILTERS_IN_THE_QUERY_BODY = @@ -2065,6 +2075,9 @@ public final class DataNodeQueryMessages { public static final String CQ_EVERY_INTERVAL_D_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL = "CQ：every interval [%d] 不应小于配置项 `continuous_query_minimum_every_interval` [%d]。"; + public static final String CQ_EVERY_INTERVAL_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL = + + "CQ：every interval [%s] 不应小于配置项 `continuous_query_minimum_every_interval` [%d]。"; public static final String CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_END_TIME_OFFSET = "CQ：开始时间偏移量应大于结束时间偏移量。"; @@ -4635,4 +4648,6 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "只有 InMemoryDeviceEntryDataSet 支持获取内存中的设备条目"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3 = + "集群所有节点支持 duration encoding version 1 后才能创建 CQ 日历 duration"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index d8692a3573b3a..1244879ed425c 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -63,7 +63,7 @@ private StorageEngineMessages() {} public static final String FILE_NO_SUCH_TIME_SERIES = "文件中不包含该时间序列 {}。"; // ======================== Resource Control - Disk ======================== - + public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY = "所有目录已满，切换系统为只读模式。"; public static final String FAILED_TO_PROCESS_FOLDER = "处理目录失败 '"; public static final String FAIL_TO_GET_CANONICAL_PATH = "获取数据目录 {} 的规范路径失败"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 3ddd2531200dc..f7d921b29df7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -112,6 +112,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCountDatabaseResp; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; @@ -156,6 +157,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetUDFTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetUdfTableReq; import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TPipeConfigTransferReq; import org.apache.iotdb.confignode.rpc.thrift.TPipeConfigTransferResp; import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq; @@ -4157,24 +4159,77 @@ public SettableFuture createContinuousQuery( createContinuousQueryStatement.semanticCheck(); final String queryBody = createContinuousQueryStatement.getQueryBody(); + // CQ body group-by bounds are populated only when the CQ executes. Month-aware filters + // cannot initialize an empty [0, 0) range during this validation pass, so provide a minimal + // non-empty placeholder; the execution RPC replaces these bounds with the real window. + if (createContinuousQueryStatement.getQueryBodyStatement().getGroupByTimeComponent() != null + && createContinuousQueryStatement + .getQueryBodyStatement() + .getGroupByTimeComponent() + .getInterval() + .containsMonth()) { + createContinuousQueryStatement + .getQueryBodyStatement() + .getGroupByTimeComponent() + .setStartTime(0); + createContinuousQueryStatement + .getQueryBodyStatement() + .getGroupByTimeComponent() + .setEndTime(1); + } // TODO Do not modify Statement in Analyzer Analyzer.analyze(createContinuousQueryStatement.getQueryBodyStatement(), context); final SettableFuture future = SettableFuture.create(); try (final ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + final boolean hasCalendarDuration = + createContinuousQueryStatement.getEveryDuration().monthDuration != 0 + || createContinuousQueryStatement.getStartTimeOffsetDuration().monthDuration != 0 + || createContinuousQueryStatement.getEndTimeOffsetDuration().monthDuration != 0; + if (hasCalendarDuration && !allClusterNodesSupportCQDurationEncoding(client.showCluster())) { + future.setException( + new SemanticException( + DataNodeQueryMessages + .MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3)); + return future; + } + // The legacy fields are still required on the wire. If any structured component contains a + // calendar month, none of the legacy fields has a valid representation and all three use the + // zero sentinel. This prevents an old reader from interpreting a mixed request as a partially + // valid fixed-duration CQ. + final long legacyEvery = + hasCalendarDuration ? 0 : createContinuousQueryStatement.getEveryInterval(); + final long legacyStart = + hasCalendarDuration ? 0 : createContinuousQueryStatement.getStartTimeOffset(); + final long legacyEnd = + hasCalendarDuration ? 0 : createContinuousQueryStatement.getEndTimeOffset(); final TCreateCQReq tCreateCQReq = new TCreateCQReq( createContinuousQueryStatement.getCqId(), - createContinuousQueryStatement.getEveryInterval(), + legacyEvery, createContinuousQueryStatement.getBoundaryTime(), - createContinuousQueryStatement.getStartTimeOffset(), - createContinuousQueryStatement.getEndTimeOffset(), + legacyStart, + legacyEnd, createContinuousQueryStatement.getTimeoutPolicy().getType(), queryBody, context.getSql(), context.getZoneId().getId(), context.getSession() == null ? null : context.getSession().getUserName()); + tCreateCQReq.setDurationEncodingVersion((short) 1); + tCreateCQReq.setEveryDuration( + new TCQDuration( + createContinuousQueryStatement.getEveryDuration().monthDuration, + createContinuousQueryStatement.getEveryDuration().nonMonthDuration)); + tCreateCQReq.setStartOffsetDuration( + new TCQDuration( + createContinuousQueryStatement.getStartTimeOffsetDuration().monthDuration, + createContinuousQueryStatement.getStartTimeOffsetDuration().nonMonthDuration)); + tCreateCQReq.setEndOffsetDuration( + new TCQDuration( + createContinuousQueryStatement.getEndTimeOffsetDuration().monthDuration, + createContinuousQueryStatement.getEndTimeOffsetDuration().nonMonthDuration)); + tCreateCQReq.setBoundaryExplicit(createContinuousQueryStatement.isBoundaryExplicit()); final TSStatus executionStatus = client.createCQ(tCreateCQReq); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != executionStatus.getCode()) { future.setException(new IoTDBException(executionStatus)); @@ -4187,6 +4242,35 @@ public SettableFuture createContinuousQuery( return future; } + private boolean allClusterNodesSupportCQDurationEncoding(TShowClusterResp response) { + if (response == null + || response.getConfigNodeList() == null + || response.getDataNodeList() == null + || response.getNodeVersionInfo() == null) { + return false; + } + if (response.getConfigNodeList().isEmpty() && response.getDataNodeList().isEmpty()) { + return false; + } + for (TConfigNodeLocation node : response.getConfigNodeList()) { + if (!supportsCQDurationEncoding(response.getNodeVersionInfo().get(node.getConfigNodeId()))) { + return false; + } + } + for (TDataNodeLocation node : response.getDataNodeList()) { + if (!supportsCQDurationEncoding(response.getNodeVersionInfo().get(node.getDataNodeId()))) { + return false; + } + } + return true; + } + + private boolean supportsCQDurationEncoding(TNodeVersionInfo info) { + return info != null + && info.isSetSupportedCQDurationEncodingVersions() + && info.getSupportedCQDurationEncodingVersions().contains((short) 1); + } + @Override public SettableFuture dropContinuousQuery(final String cqId) { final SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 1a9b8297ec44c..455dfbe687f68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -1240,10 +1240,10 @@ public Statement visitCreateContinuousQuery(IoTDBSqlParser.CreateContinuousQuery .CQ_AT_LEAST_ONE_OF_THE_PARAMETERS_EVERY_INTERVAL_AND_GROUP_BY_INTERVAL_NEEDS_TO_BE); } - long interval = - queryStatement.getGroupByTimeComponent().getInterval().getTotalDuration(currPrecision); - statement.setEveryInterval(interval); - statement.setStartTimeOffset(interval); + org.apache.tsfile.utils.TimeDuration interval = + queryStatement.getGroupByTimeComponent().getInterval(); + statement.setEveryDuration(interval); + statement.setStartTimeOffsetDuration(interval); } if (ctx.timeoutPolicyClause() != null) { @@ -1256,8 +1256,8 @@ public Statement visitCreateContinuousQuery(IoTDBSqlParser.CreateContinuousQuery private void parseResampleClause( IoTDBSqlParser.ResampleClauseContext ctx, CreateContinuousQueryStatement statement) { if (ctx.EVERY() != null) { - statement.setEveryInterval( - DataNodeDateTimeUtils.convertDurationStrToLong(ctx.everyInterval.getText())); + statement.setEveryDuration( + DataNodeDateTimeUtils.constructTimeDurationForCQ(ctx.everyInterval.getText())); } else { QueryStatement queryStatement = statement.getQueryBodyStatement(); if (!queryStatement.isGroupByTime()) { @@ -1265,24 +1265,24 @@ private void parseResampleClause( DataNodeQueryMessages .CQ_AT_LEAST_ONE_OF_THE_PARAMETERS_EVERY_INTERVAL_AND_GROUP_BY_INTERVAL_NEEDS_TO_BE); } - statement.setEveryInterval( - queryStatement.getGroupByTimeComponent().getInterval().getTotalDuration(currPrecision)); + statement.setEveryDuration(queryStatement.getGroupByTimeComponent().getInterval()); } if (ctx.BOUNDARY() != null) { + statement.setBoundaryExplicit(true); statement.setBoundaryTime( parseTimeValue(ctx.boundaryTime, CommonDateTimeUtils.currentTime())); } if (ctx.RANGE() != null) { - statement.setStartTimeOffset( - DataNodeDateTimeUtils.convertDurationStrToLong(ctx.startTimeOffset.getText())); + statement.setStartTimeOffsetDuration( + DataNodeDateTimeUtils.constructTimeDurationForCQ(ctx.startTimeOffset.getText())); if (ctx.endTimeOffset != null) { - statement.setEndTimeOffset( - DataNodeDateTimeUtils.convertDurationStrToLong(ctx.endTimeOffset.getText())); + statement.setEndTimeOffsetDuration( + DataNodeDateTimeUtils.constructTimeDurationForCQ(ctx.endTimeOffset.getText())); } } else { - statement.setStartTimeOffset(statement.getEveryInterval()); + statement.setStartTimeOffsetDuration(statement.getEveryDuration()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java index ef7506c8d20b2..ae4cd3f176872 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.cq.TimeoutPolicy; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils; @@ -33,6 +34,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.component.GroupByTimeComponent; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.tsfile.utils.TimeDuration; + import java.util.Collections; import java.util.List; @@ -52,6 +55,13 @@ public class CreateContinuousQueryStatement extends Statement implements IConfig // The end time of each query execution, default value is 0. private long endTimeOffset = 0; + // Structured representations retained for calendar-aware CQ scheduling. The legacy long + // fields above are kept as wire/backward-compatible projections for fixed-only durations. + private TimeDuration everyDuration = new TimeDuration(0, 0); + private TimeDuration startTimeOffsetDuration = new TimeDuration(0, 0); + private TimeDuration endTimeOffsetDuration = new TimeDuration(0, 0); + private boolean boundaryExplicit; + // Specify how we deal with the cq task whose previous time interval execution is not finished // while the next execution time has reached, default value is BLOCKED. private TimeoutPolicy timeoutPolicy = TimeoutPolicy.BLOCKED; @@ -78,6 +88,19 @@ public long getEveryInterval() { public void setEveryInterval(long everyInterval) { this.everyInterval = everyInterval; + this.everyDuration = new TimeDuration(0, everyInterval); + } + + public TimeDuration getEveryDuration() { + return everyDuration; + } + + public void setEveryDuration(TimeDuration everyDuration) { + this.everyDuration = everyDuration; + this.everyInterval = + everyDuration.monthDuration == 0 + ? everyDuration.getTotalDuration(TimestampPrecisionUtils.currPrecision) + : 0; } public long getBoundaryTime() { @@ -88,12 +111,33 @@ public void setBoundaryTime(long boundaryTime) { this.boundaryTime = boundaryTime; } + public boolean isBoundaryExplicit() { + return boundaryExplicit; + } + + public void setBoundaryExplicit(boolean boundaryExplicit) { + this.boundaryExplicit = boundaryExplicit; + } + public long getStartTimeOffset() { return startTimeOffset; } public void setStartTimeOffset(long startTimeOffset) { this.startTimeOffset = startTimeOffset; + this.startTimeOffsetDuration = new TimeDuration(0, startTimeOffset); + } + + public TimeDuration getStartTimeOffsetDuration() { + return startTimeOffsetDuration; + } + + public void setStartTimeOffsetDuration(TimeDuration duration) { + this.startTimeOffsetDuration = duration; + this.startTimeOffset = + duration.monthDuration == 0 + ? duration.getTotalDuration(TimestampPrecisionUtils.currPrecision) + : 0; } public long getEndTimeOffset() { @@ -102,6 +146,19 @@ public long getEndTimeOffset() { public void setEndTimeOffset(long endTimeOffset) { this.endTimeOffset = endTimeOffset; + this.endTimeOffsetDuration = new TimeDuration(0, endTimeOffset); + } + + public TimeDuration getEndTimeOffsetDuration() { + return endTimeOffsetDuration; + } + + public void setEndTimeOffsetDuration(TimeDuration duration) { + this.endTimeOffsetDuration = duration; + this.endTimeOffset = + duration.monthDuration == 0 + ? duration.getTotalDuration(TimestampPrecisionUtils.currPrecision) + : 0; } public TimeoutPolicy getTimeoutPolicy() { @@ -135,11 +192,28 @@ public String constructFormattedSQL() { StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("CREATE CQ ").append(cqId).append('\n'); sqlBuilder.append("RESAMPLE\n"); - sqlBuilder.append('\t').append("EVERY ").append(everyInterval).append("ms\n"); + sqlBuilder + .append('\t') + .append("EVERY ") + .append( + everyDuration.monthDuration == 0 ? everyInterval + "ms" : formatDuration(everyDuration)) + .append('\n'); sqlBuilder.append('\t').append("BOUNDARY ").append(boundaryTime).append("\n"); - sqlBuilder.append('\t').append("RANGE ").append(startTimeOffset).append("ms"); - if (endTimeOffset != 0) { - sqlBuilder.append(", ").append(endTimeOffset).append("ms\n"); + sqlBuilder + .append('\t') + .append("RANGE ") + .append( + startTimeOffsetDuration.monthDuration == 0 + ? startTimeOffset + "ms" + : formatDuration(startTimeOffsetDuration)); + if (endTimeOffset != 0 || endTimeOffsetDuration.monthDuration != 0) { + sqlBuilder + .append(", ") + .append( + endTimeOffsetDuration.monthDuration == 0 + ? endTimeOffset + "ms" + : formatDuration(endTimeOffsetDuration)) + .append('\n'); } else { sqlBuilder.append("\n"); } @@ -154,6 +228,24 @@ public String constructFormattedSQL() { return sqlBuilder.toString(); } + private static String formatDuration(TimeDuration duration) { + StringBuilder result = new StringBuilder(); + int months = duration.monthDuration; + if (months >= 12) { + result.append(months / 12).append('y'); + months %= 12; + } + if (months != 0) { + result.append(months).append("mo"); + } + if (duration.nonMonthDuration != 0) { + result.append(duration.nonMonthDuration).append(TimestampPrecisionUtils.TIMESTAMP_PRECISION); + } + return result.length() == 0 + ? "0" + TimestampPrecisionUtils.TIMESTAMP_PRECISION + : result.toString(); + } + @Override public QueryType getQueryType() { return QueryType.OTHER; @@ -170,26 +262,50 @@ public R accept(StatementVisitor visitor, C context) { } public void semanticCheck() { - if (everyInterval - < IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval()) { + // Positivity first: the lower-bound arithmetic below assumes a non-degenerate EVERY. + if (!isPositive(everyDuration)) { throw new SemanticException( - String.format( - DataNodeQueryMessages - .CQ_EVERY_INTERVAL_D_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL, - everyInterval, - IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval())); + DataNodeQueryMessages.EXCEPTION_CQ_EVERY_INTERVAL_MUST_BE_POSITIVE_26259019); } - if (startTimeOffset <= 0) { + long minimumEvery = + IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval(); + long minimumElapsed = + everyDuration.monthDuration == 0 + ? everyDuration.nonMonthDuration + : Math.subtractExact( + Math.addExact( + Math.multiplyExact( + (long) everyDuration.monthDuration, + TimestampPrecisionUtils.currPrecision.convert( + 28L * 86_400_000L, java.util.concurrent.TimeUnit.MILLISECONDS)), + everyDuration.nonMonthDuration), + TimestampPrecisionUtils.currPrecision.convert( + 36L * 3_600_000L, java.util.concurrent.TimeUnit.MILLISECONDS)); + if (minimumElapsed < minimumEvery) { + throw new SemanticException( + everyDuration.monthDuration == 0 + ? String.format( + DataNodeQueryMessages + .CQ_EVERY_INTERVAL_D_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL, + everyInterval, + minimumEvery) + : String.format( + DataNodeQueryMessages + .CQ_EVERY_INTERVAL_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL, + formatDuration(everyDuration), + minimumEvery)); + } + if (!isPositive(startTimeOffsetDuration)) { throw new SemanticException(DataNodeQueryMessages.CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER); } - if (endTimeOffset < 0) { + if (endTimeOffsetDuration.monthDuration < 0 || endTimeOffsetDuration.nonMonthDuration < 0) { throw new SemanticException(DataNodeQueryMessages.CQ_THE_END_TIME_OFFSET_SHOULD_BE_GREATER); } - if (startTimeOffset <= endTimeOffset) { + if (!dominates(startTimeOffsetDuration, endTimeOffsetDuration, true)) { throw new SemanticException( DataNodeQueryMessages.CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_END_TIME_OFFSET); } - if (everyInterval > startTimeOffset) { + if (!dominates(startTimeOffsetDuration, everyDuration, false)) { throw new SemanticException( DataNodeQueryMessages .CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_INTERVAL); @@ -211,4 +327,20 @@ public void semanticCheck() { DataNodeQueryMessages.CQ_SPECIFYING_TIME_FILTERS_IN_THE_QUERY_BODY); } } + + private static boolean dominates(TimeDuration left, TimeDuration right, boolean strict) { + boolean result = + left.monthDuration >= right.monthDuration + && left.nonMonthDuration >= right.nonMonthDuration; + if (!result) { + return false; + } + return !strict + || left.monthDuration != right.monthDuration + || left.nonMonthDuration != right.nonMonthDuration; + } + + private static boolean isPositive(TimeDuration duration) { + return duration.monthDuration > 0 || duration.nonMonthDuration > 0; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 5ce282db7a701..1e1659c74bd1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -561,7 +561,9 @@ private void sendRegisterRequestToConfigNode(boolean isPreCheck) req.setPreCheck(isPreCheck); req.setDataNodeConfiguration(generateDataNodeConfiguration()); req.setClusterName(config.getClusterName()); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo( + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); TDataNodeRegisterResp dataNodeRegisterResp = null; while (retry > 0) { try (ConfigNodeClient configNodeClient = @@ -737,7 +739,9 @@ private void sendRestartRequestToConfigNode() throws StartupException { req.setClusterName( config.getClusterName() == null ? DEFAULT_CLUSTER_NAME : config.getClusterName()); req.setDataNodeConfiguration(generateDataNodeConfiguration()); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo( + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); req.setClusterId(config.getClusterId()); TDataNodeRestartResp dataNodeRestartResp = null; while (retry > 0) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java index 617572d0f8643..2d76c6fa651a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java @@ -38,8 +38,70 @@ import java.time.ZoneId; import java.util.Calendar; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class DataNodeDateTimeUtils { + private static final Pattern CQ_DURATION_COMPONENT = + // Match multi-character units before their one-character prefixes (for example, ms before + // m), otherwise 1ms would be tokenized as 1m followed by an invalid trailing s. + Pattern.compile("(\\d+)(y|mo|w|d|h|ms|us|ns|m|s)", Pattern.CASE_INSENSITIVE); + + /** + * Parses the CQ duration grammar while retaining calendar months. Full aliases are deliberately + * not accepted here; CQ uses the same mo/y abbreviations as Tree SQL. + */ + public static TimeDuration constructTimeDurationForCQ(String duration) { + if (duration == null || duration.isEmpty()) { + throw new IllegalArgumentException( + DataNodeQueryMessages.EXCEPTION_CQ_DURATION_CANNOT_BE_EMPTY_C7269AB2); + } + Matcher matcher = CQ_DURATION_COMPONENT.matcher(duration); + long months = 0; + long fixed = 0; + int end = 0; + String precision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + while (matcher.find()) { + if (matcher.start() != end) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_INVALID_CQ_DURATION_ARG_F4917D5C, duration)); + } + long value; + try { + value = Long.parseLong(matcher.group(1)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_CQ_DURATION_COMPONENT_OVERFLOWS_ARG_ED5B0962, + duration), + e); + } + String unit = matcher.group(2).toLowerCase(Locale.ROOT); + if (unit.equals("y")) { + months = Math.addExact(months, Math.multiplyExact(value, 12)); + } else if (unit.equals("mo")) { + months = Math.addExact(months, value); + } else { + fixed = Math.addExact(fixed, convertDurationStrToLong(-1, value, unit, precision)); + } + end = matcher.end(); + } + if (end != duration.length()) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_INVALID_CQ_DURATION_ARG_F4917D5C, duration)); + } + if (months > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_CQ_DURATION_MONTH_COMPONENT_OVERFLOWS_ARG_EBF2A2B6, + duration)); + } + return new TimeDuration((int) months, fixed); + } + public static Long parseDateTimeExpressionToLong(String dateExpression, ZoneId zoneId) { ASTVisitor astVisitor = new ASTVisitor(); astVisitor.setZoneId(zoneId); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java new file mode 100644 index 0000000000000..c8065a975ca97 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.executor; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class ClusterConfigTaskExecutorCQTest { + + @Test + public void mixedVersionClusterIsRejectedAtDataNodeIngress() throws Exception { + Method method = + ClusterConfigTaskExecutor.class.getDeclaredMethod( + "allClusterNodesSupportCQDurationEncoding", TShowClusterResp.class); + method.setAccessible(true); + ClusterConfigTaskExecutor executor = ClusterConfigTaskExecutor.getInstance(); + + Assert.assertFalse((Boolean) method.invoke(executor, new TShowClusterResp())); + Assert.assertFalse((Boolean) method.invoke(executor, (Object) null)); + + TConfigNodeLocation configNode = + new TConfigNodeLocation( + 1, new TEndPoint("127.0.0.1", 10710), new TEndPoint("127.0.0.1", 10720)); + TDataNodeLocation dataNode = + new TDataNodeLocation( + 11, + new TEndPoint("127.0.0.1", 6667), + new TEndPoint("127.0.0.1", 10730), + new TEndPoint("127.0.0.1", 10740), + new TEndPoint("127.0.0.1", 10750), + new TEndPoint("127.0.0.1", 10760)); + TNodeVersionInfo supported = + new TNodeVersionInfo("2.0.0", "new") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + TNodeVersionInfo unsupported = new TNodeVersionInfo("1.3.0", "old"); + + TShowClusterResp mixed = new TShowClusterResp(); + mixed.setConfigNodeList(Collections.singletonList(configNode)); + mixed.setDataNodeList(Collections.singletonList(dataNode)); + Map versions = new HashMap<>(); + versions.put(configNode.getConfigNodeId(), supported); + versions.put(dataNode.getDataNodeId(), unsupported); + mixed.setNodeVersionInfo(versions); + Assert.assertFalse((Boolean) method.invoke(executor, mixed)); + + versions.put(dataNode.getDataNodeId(), supported); + Assert.assertTrue((Boolean) method.invoke(executor, mixed)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java new file mode 100644 index 0000000000000..747ec62977693 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.metadata; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.time.ZoneId; + +public class CreateContinuousQueryStatementTest { + + private long originalMinimumEvery; + + @Before + public void setUp() { + originalMinimumEvery = + IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval(); + } + + @After + public void tearDown() { + IoTDBDescriptor.getInstance() + .getConfig() + .setContinuousQueryMinimumEveryInterval(originalMinimumEvery); + } + + @Test + public void calendarEveryAndRangeAreAccepted() { + CreateContinuousQueryStatement statement = parse(calendarSql("1mo", "1mo")); + statement.semanticCheck(); + Assert.assertEquals(1, statement.getEveryDuration().monthDuration); + Assert.assertEquals(0, statement.getEveryDuration().nonMonthDuration); + Assert.assertEquals(1, statement.getStartTimeOffsetDuration().monthDuration); + } + + @Test + public void inheritedGroupByMonthKeepsCalendarEvery() { + CreateContinuousQueryStatement statement = + parse( + "CREATE CQ cq_inherited BEGIN " + + "SELECT max_value(s1) INTO root.sg.d1(s1_max) FROM root.sg.d1 GROUP BY(1mo) END"); + statement.semanticCheck(); + Assert.assertEquals(1, statement.getEveryDuration().monthDuration); + Assert.assertEquals(1, statement.getStartTimeOffsetDuration().monthDuration); + } + + @Test + public void compoundCalendarDurationDominatesAndIsAccepted() { + parse(calendarSql("1mo", "1mo3d")).semanticCheck(); + } + + @Test + public void incomparableCalendarAndFixedDurationsAreRejected() { + try { + parse(calendarSql("1mo", "30d")).semanticCheck(); + Assert.fail("expected incomparable RANGE vs EVERY to be rejected"); + } catch (SemanticException e) { + Assert.assertTrue( + e.getMessage() + .contains("The start time offset should be greater than or equal to every interval")); + } + } + + @Test + public void monthAndYearAliasesAreRejectedByTheParser() { + assertParseFails(calendarSql("1month", "1month")); + assertParseFails(calendarSql("1year", "1year")); + assertParseFails(calendarSql("2months", "2months")); + assertParseFails(calendarSql("2years", "2years")); + } + + @Test + public void inheritedGroupByYearKeepsCalendarEvery() { + CreateContinuousQueryStatement statement = + parse( + "CREATE CQ cq_inherited_year BEGIN " + + "SELECT max_value(s1) INTO root.sg.d1(s1_max) FROM root.sg.d1 GROUP BY(1y) END"); + statement.semanticCheck(); + Assert.assertEquals(12, statement.getEveryDuration().monthDuration); + Assert.assertEquals(12, statement.getStartTimeOffsetDuration().monthDuration); + } + + @Test + public void calendarMinimumEveryUsesElapsedLowerBound() { + // 1mo lower bound is M * 28d - 36h. Reject when the configured minimum exceeds that bound. + IoTDBDescriptor.getInstance() + .getConfig() + .setContinuousQueryMinimumEveryInterval(2_289_600_001L); + try { + parse(calendarSql("1mo", "1mo")).semanticCheck(); + Assert.fail("expected 1mo to fail the conservative minimum-EVERY bound"); + } catch (SemanticException e) { + Assert.assertTrue(e.getMessage().contains("1mo")); + Assert.assertFalse(e.getMessage().contains("[0]")); + } + } + + private static CreateContinuousQueryStatement parse(String sql) { + return (CreateContinuousQueryStatement) + StatementGenerator.createStatement(sql, ZoneId.of("UTC")); + } + + private static void assertParseFails(String sql) { + try { + StatementGenerator.createStatement(sql, ZoneId.of("UTC")); + Assert.fail("expected parser to reject " + sql); + } catch (Exception ignored) { + // Tree SQL duration literals only accept y/mo abbreviations. + } + } + + private static String calendarSql(String every, String range) { + return "CREATE CQ cq_calendar RESAMPLE EVERY " + + every + + " RANGE " + + range + + " BEGIN SELECT max_value(s1) INTO root.sg.d1(s1_max) FROM root.sg.d1 GROUP BY(1mo) END"; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java index 0e48ffb883a9b..321dd0699b981 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java @@ -394,6 +394,31 @@ public void testConstructTimeDuration() { Assert.assertEquals(10000000000L, timeDuration.nonMonthDuration); } + @Test + public void testConstructTimeDurationForCQSupportsMinuteAndSecond() { + TimeDuration timeDuration = DataNodeDateTimeUtils.constructTimeDurationForCQ("5m30s"); + Assert.assertEquals(0, timeDuration.monthDuration); + Assert.assertEquals(330_000L, timeDuration.nonMonthDuration); + + timeDuration = DataNodeDateTimeUtils.constructTimeDurationForCQ("1ms"); + Assert.assertEquals(0, timeDuration.monthDuration); + Assert.assertEquals(1L, timeDuration.nonMonthDuration); + + timeDuration = DataNodeDateTimeUtils.constructTimeDurationForCQ("1y2mo"); + Assert.assertEquals(14, timeDuration.monthDuration); + Assert.assertEquals(0, timeDuration.nonMonthDuration); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> DataNodeDateTimeUtils.constructTimeDurationForCQ("1month")); + Assert.assertThrows( + IllegalArgumentException.class, + () -> DataNodeDateTimeUtils.constructTimeDurationForCQ("1year")); + Assert.assertThrows( + IllegalArgumentException.class, + () -> DataNodeDateTimeUtils.constructTimeDurationForCQ("1mo 1d")); + } + @Test public void testConstructTimeDurationOverflow() { Assert.assertThrows( diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 512e2e05f05fc..8ce6213a44004 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -653,6 +653,8 @@ struct TGetClusterIdResp { struct TNodeVersionInfo { 1: required string version; 2: required string buildInfo; + // Capabilities are optional so older nodes can still register. Missing means unsupported. + 3: optional set supportedCQDurationEncodingVersions; } struct TNodeActivateInfo { @@ -1132,6 +1134,11 @@ struct TGetCommitProgressResp { // ==================================================== // CQ // ==================================================== +struct TCQDuration { + 1: required i64 monthPart + 2: required i64 nonMonthDuration +} + struct TCreateCQReq { 1: required string cqId, 2: required i64 everyInterval @@ -1143,6 +1150,12 @@ struct TCreateCQReq { 8: required string sql 9: required string zoneId 10: required string username + // Versioned calendar-aware duration representation. Legacy fields above remain unchanged. + 11: optional i16 durationEncodingVersion + 12: optional TCQDuration everyDuration + 13: optional TCQDuration startOffsetDuration + 14: optional TCQDuration endOffsetDuration + 15: optional bool boundaryExplicit } struct TDropCQReq {