From d4ffaf89eb8ffc3d02aa6b082aa82392e8b95a4c Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Sun, 20 Sep 2026 11:33:49 +0800 Subject: [PATCH 01/10] feat: support calendar month and year CQ intervals Treat CQ EVERY/RANGE mo/y as calendar months and years instead of flattening them to 30d/365d. Reuse GROUP BY TIME calendar arithmetic, fail closed on mixed-version clusters, and keep RANGE anchored to the original boundary. Fixes #18428 --- .../it/IoTDBConfigNodeSnapshotIT.java | 9 + .../org/apache/iotdb/db/it/cq/IoTDBCQIT.java | 58 +++ .../confignode/i18n/ManagerMessages.java | 40 ++ .../confignode/i18n/ManagerMessages.java | 40 ++ .../confignode/UpdateVersionInfoPlan.java | 19 + .../write/cq/UpdateCQLastExecTimePlan.java | 57 ++- .../manager/cq/CQCalendarUtils.java | 117 ++++++ .../confignode/manager/cq/CQManager.java | 200 ++++++++++ .../confignode/manager/cq/CQScheduleTask.java | 358 ++++++++++++++++-- .../confignode/manager/node/NodeManager.java | 38 +- .../confignode/persistence/cq/CQInfo.java | 254 ++++++++++++- .../confignode/persistence/node/NodeInfo.java | 39 ++ .../procedure/impl/cq/CreateCQProcedure.java | 37 +- .../impl/node/AddConfigNodeProcedure.java | 19 + .../iotdb/confignode/service/ConfigNode.java | 14 +- .../confignode/UpdateVersionInfoPlanTest.java | 69 ++++ .../iotdb/confignode/cq/CQManagerTest.java | 104 +++++ .../confignode/cq/CQScheduleTaskTest.java | 124 ++++++ .../manager/cq/CQCalendarUtilsTest.java | 253 +++++++++++++ .../manager/node/NodeManagerTest.java | 73 +++- .../confignode/persistence/CQInfoTest.java | 179 +++++++++ .../confignode/persistence/NodeInfoTest.java | 32 ++ .../procedure/impl/CreateCQProcedureTest.java | 29 ++ .../impl/node/AddConfigNodeProcedureTest.java | 21 +- .../iotdb/db/i18n/DataNodeQueryMessages.java | 14 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 15 + .../iotdb/db/i18n/StorageEngineMessages.java | 2 +- .../executor/ClusterConfigTaskExecutor.java | 90 ++++- .../queryengine/plan/parser/ASTVisitor.java | 26 +- .../CreateContinuousQueryStatement.java | 161 +++++++- .../org/apache/iotdb/db/service/DataNode.java | 8 +- .../iotdb/db/utils/DataNodeDateTimeUtils.java | 62 +++ .../CreateContinuousQueryStatementTest.java | 131 +++++++ .../iotdb/db/utils/DateTimeUtilsTest.java | 22 ++ .../src/main/thrift/confignode.thrift | 13 + 35 files changed, 2626 insertions(+), 101 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java index a5c757d8dd361..5d8deb8ac6416 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/IoTDBConfigNodeSnapshotIT.java @@ -32,6 +32,7 @@ import org.apache.iotdb.commons.trigger.TriggerInformation; import org.apache.iotdb.commons.trigger.service.TriggerExecutableManager; import org.apache.iotdb.commons.udf.UDFInformation; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCQEntry; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq; @@ -354,6 +355,14 @@ private Set createCQs(SyncConfigNodeIServiceClient client) throws TExc "UTC", "root"); + for (TCreateCQReq req : new TCreateCQReq[] {req1, req2}) { + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + } + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.createCQ(req1).getCode()); assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.createCQ(req2).getCode()); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java index 399694b5e2cee..3e6bf8162e28a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQIT.java @@ -36,6 +36,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -407,6 +408,63 @@ public void testCreateCorrectCQ() { } } + @Test + public void testCreateCalendarMonthCQ() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "CREATE CQ calendar_cq_1\n" + + "RESAMPLE EVERY 1mo RANGE 1mo\n" + + "BEGIN\n" + + " SELECT max_value(s1)\n" + + " INTO root.sg.d1(s1_max)\n" + + " FROM root.sg.d1\n" + + " GROUP BY(1mo)\n" + + "END"); + try (ResultSet resultSet = statement.executeQuery("show CQS")) { + boolean found = false; + while (resultSet.next()) { + if ("calendar_cq_1".equals(resultSet.getString(1))) { + found = true; + assertEquals("ACTIVE", resultSet.getString(3)); + } + } + assertTrue(found); + } + statement.execute("DROP CQ calendar_cq_1"); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testRejectIncomparableCalendarAndFixedCQDurations() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + try { + statement.execute( + "CREATE CQ calendar_cq_incomparable\n" + + "RESAMPLE EVERY 1mo RANGE 30d\n" + + "BEGIN\n" + + " SELECT max_value(s1)\n" + + " INTO root.sg.d1(s1_max)\n" + + " FROM root.sg.d1\n" + + " GROUP BY(1mo)\n" + + "END"); + fail(); + } catch (Exception e) { + assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": CQ: The start time offset should be greater than or equal to every interval.", + e.getMessage()); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + // =======================================show cq====================================== @Test public void testShowCQ() { diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 10ea4763af9a2..885dc8339f75c 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -702,4 +702,44 @@ private ManagerMessages() {} MESSAGE_ARG_PLEASE_MANUALLY_CHECK_LATER_WHETHER_THE_PROCEDURE_IS_EXECUTED_SUCCESSFULLY_A82B739D = "%s Please manually check later whether the procedure is executed successfully."; + public static final String EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26 = + "CQ EVERY duration must be positive"; + public static final String MESSAGE_CQ_START_OFFSET_MUST_BE_POSITIVE_B837C4F5 = + "CQ start offset must be positive"; + public static final String MESSAGE_CQ_END_OFFSET_MUST_BE_NON_NEGATIVE_64171164 = + "CQ end offset must be non-negative"; + public static final String MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_END_OFFSET_5924C189 = + "CQ start offset must be greater than end offset"; + public static final String + MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_DURATION_89628D43 = + "CQ start offset must be greater than or equal to EVERY duration"; + public static final String EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C = + "CQ timestamp overflows configured precision"; + public static final String + MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92 = + "Invalid CQ duration encoding; version 1 requires all structured fields"; + public static final String MESSAGE_CQ_DURATIONS_MUST_BE_NON_NEGATIVE_BE23CE04 = + "CQ durations must be non-negative"; + public static final String + MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67 = + "CQ legacy duration fields conflict with structured duration fields"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072 = + "CQ calendar duration requires all cluster nodes to support duration encoding version 1"; + public static final String MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A = + "CQ duration encoding marker is required for new requests"; + public static final String MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C = + "CQ does not have occurrence-index metadata"; + public static final String MESSAGE_CQ_OCCURRENCE_CALLBACK_IS_STALE_36C5FBFC = + "CQ occurrence callback is stale"; + public static final String MESSAGE_CQ_OCCURRENCE_INDEX_IS_AHEAD_OF_THE_CALLBACK_8A18ECC9 = + "CQ occurrence index is ahead of the callback"; + public static final String EXCEPTION_INVALID_CQ_OCCURRENCE_INDEX_TRANSITION_AC6BFC4D = + "Invalid CQ occurrence index transition"; + public static final String EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035 = + "Negative CQ snapshot entry count: %d"; + public static final String EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_7E4B91A2 = + "CQ occurrence index does not match the scheduled execution time"; + public static final String EXCEPTION_CQ_RANGE_END_MUST_BE_GREATER_THAN_START_3C91E8B4 = + "CQ RANGE end time must be greater than start time"; + } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 499e922f7a7d2..b56336a27fc73 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -681,4 +681,44 @@ private ManagerMessages() {} MESSAGE_ARG_PLEASE_MANUALLY_CHECK_LATER_WHETHER_THE_PROCEDURE_IS_EXECUTED_SUCCESSFULLY_A82B739D = "%s 请稍后手动检查该 Procedure 是否执行成功。"; + public static final String EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26 = + "CQ EVERY duration 必须为正数"; + public static final String MESSAGE_CQ_START_OFFSET_MUST_BE_POSITIVE_B837C4F5 = + "CQ start offset 必须为正数"; + public static final String MESSAGE_CQ_END_OFFSET_MUST_BE_NON_NEGATIVE_64171164 = + "CQ end offset 必须为非负数"; + public static final String MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_END_OFFSET_5924C189 = + "CQ start offset 必须大于 end offset"; + public static final String + MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_DURATION_89628D43 = + "CQ start offset 必须大于或等于 EVERY duration"; + public static final String EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C = + "CQ timestamp 超出配置的精度范围"; + public static final String + MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92 = + "无效的 CQ duration encoding；版本 1 需要所有结构化字段"; + public static final String MESSAGE_CQ_DURATIONS_MUST_BE_NON_NEGATIVE_BE23CE04 = + "CQ duration 必须为非负数"; + public static final String + MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67 = + "CQ legacy duration fields 与结构化 duration 字段冲突"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072 = + "CQ 日历 duration 要求集群所有节点支持 duration encoding version 1"; + public static final String MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A = + "新建 CQ 请求必须包含 duration encoding marker"; + public static final String MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C = + "CQ 没有 occurrence-index 元数据"; + public static final String MESSAGE_CQ_OCCURRENCE_CALLBACK_IS_STALE_36C5FBFC = + "CQ occurrence callback 已过期"; + public static final String MESSAGE_CQ_OCCURRENCE_INDEX_IS_AHEAD_OF_THE_CALLBACK_8A18ECC9 = + "CQ occurrence index 超前于 callback"; + public static final String EXCEPTION_INVALID_CQ_OCCURRENCE_INDEX_TRANSITION_AC6BFC4D = + "无效的 CQ occurrence index 转换"; + public static final String EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035 = + "CQ snapshot 条目数量不能为负数：%d"; + public static final String EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_7E4B91A2 = + "CQ occurrence index 与计划执行时间不匹配"; + public static final String EXCEPTION_CQ_RANGE_END_MUST_BE_GREATER_THAN_START_3C91E8B4 = + "CQ RANGE 结束时间必须大于开始时间"; + } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java index 62f2cc71e6926..d2894392e26bc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java @@ -59,6 +59,14 @@ protected void serializeImpl(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(nodeId, stream); ReadWriteIOUtils.write(versionInfo.getVersion(), stream); ReadWriteIOUtils.write(versionInfo.getBuildInfo(), stream); + // Optional tail keeps this plan readable by pre-calendar-duration ConfigNodes. + ReadWriteIOUtils.write(versionInfo.isSetSupportedCQDurationEncodingVersions(), stream); + if (versionInfo.isSetSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(versionInfo.getSupportedCQDurationEncodingVersions().size(), stream); + for (short version : versionInfo.getSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(version, stream); + } + } } @Override @@ -67,6 +75,17 @@ protected void deserializeImpl(ByteBuffer buffer) { versionInfo = new TNodeVersionInfo( ReadWriteIOUtils.readString(buffer), ReadWriteIOUtils.readString(buffer)); + if (buffer.hasRemaining()) { + boolean hasCapabilities = ReadWriteIOUtils.readBool(buffer); + if (hasCapabilities) { + int size = ReadWriteIOUtils.readInt(buffer); + java.util.Set capabilities = new java.util.HashSet<>(); + for (int i = 0; i < size; i++) { + capabilities.add(ReadWriteIOUtils.readShort(buffer)); + } + versionInfo.setSupportedCQDurationEncodingVersions(capabilities); + } + } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java index a487ae648e3fa..2fc3dade8cede 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java @@ -20,6 +20,7 @@ package org.apache.iotdb.confignode.consensus.request.write.cq; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.tsfile.external.commons.lang3.Validate; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -39,6 +40,10 @@ public class UpdateCQLastExecTimePlan extends ConfigPhysicalPlan { private String cqToken; + private boolean hasOccurrenceIndex; + private long expectedIndex; + private long targetIndex; + public UpdateCQLastExecTimePlan() { super(UPDATE_CQ_LAST_EXEC_TIME); } @@ -52,6 +57,18 @@ public UpdateCQLastExecTimePlan(String cqId, long executionTime, String cqToken) this.cqToken = cqToken; } + public UpdateCQLastExecTimePlan( + String cqId, long executionTime, String cqToken, long expectedIndex, long targetIndex) { + this(cqId, executionTime, cqToken); + if (expectedIndex < 0 || targetIndex <= expectedIndex) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_INVALID_CQ_OCCURRENCE_INDEX_TRANSITION_AC6BFC4D); + } + this.hasOccurrenceIndex = true; + this.expectedIndex = expectedIndex; + this.targetIndex = targetIndex; + } + public String getCqId() { return cqId; } @@ -64,12 +81,29 @@ public String getCqToken() { return cqToken; } + public boolean hasOccurrenceIndex() { + return hasOccurrenceIndex; + } + + public long getExpectedIndex() { + return expectedIndex; + } + + public long getTargetIndex() { + return targetIndex; + } + @Override protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(cqId, stream); ReadWriteIOUtils.write(executionTime, stream); ReadWriteIOUtils.write(cqToken, stream); + ReadWriteIOUtils.write(hasOccurrenceIndex, stream); + if (hasOccurrenceIndex) { + ReadWriteIOUtils.write(expectedIndex, stream); + ReadWriteIOUtils.write(targetIndex, stream); + } } @Override @@ -77,6 +111,17 @@ protected void deserializeImpl(ByteBuffer buffer) throws IOException { cqId = ReadWriteIOUtils.readString(buffer); executionTime = ReadWriteIOUtils.readLong(buffer); cqToken = ReadWriteIOUtils.readString(buffer); + if (buffer.hasRemaining()) { + hasOccurrenceIndex = ReadWriteIOUtils.readBool(buffer); + if (hasOccurrenceIndex) { + expectedIndex = ReadWriteIOUtils.readLong(buffer); + targetIndex = ReadWriteIOUtils.readLong(buffer); + if (expectedIndex < 0 || targetIndex <= expectedIndex) { + throw new IOException( + ManagerMessages.EXCEPTION_INVALID_CQ_OCCURRENCE_INDEX_TRANSITION_AC6BFC4D); + } + } + } } @Override @@ -92,12 +137,22 @@ public boolean equals(Object o) { } UpdateCQLastExecTimePlan that = (UpdateCQLastExecTimePlan) o; return executionTime == that.executionTime + && hasOccurrenceIndex == that.hasOccurrenceIndex + && expectedIndex == that.expectedIndex + && targetIndex == that.targetIndex && cqId.equals(that.cqId) && cqToken.equals(that.cqToken); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), cqId, executionTime, cqToken); + return Objects.hash( + super.hashCode(), + cqId, + executionTime, + cqToken, + hasOccurrenceIndex, + expectedIndex, + targetIndex); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java new file mode 100644 index 0000000000000..820fec0514534 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.confignode.manager.cq; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.queryengine.utils.DateTimeUtils; +import org.apache.iotdb.confignode.i18n.ManagerMessages; + +import org.apache.tsfile.utils.TimeDuration; + +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; + +/** Pure calendar arithmetic used by CQ scheduling and recovery. */ +public final class CQCalendarUtils { + private CQCalendarUtils() {} + + public static long apply(long base, TimeDuration duration, long multiplier, ZoneId zone) { + long months = Math.multiplyExact((long) duration.monthDuration, multiplier); + long fixed = Math.multiplyExact(duration.nonMonthDuration, multiplier); + return applyVector(base, months, fixed, zone); + } + + public static long applyVector(long base, long months, long fixed, ZoneId zone) { + try { + // Fixed-only vectors must stay on elapsed-tick arithmetic. Routing them through the + // calendar helper would re-resolve local time with atZone and can shift DST-overlap + // instants even when monthPart is zero. + if (months == 0) { + return Math.addExact(base, fixed); + } + // Reuse the GROUP BY TIME helper so CQ cadence, RANGE, and month buckets share DST + // overlap/gap rules (LocalDateTime.plusMonths then atZone). + TimeDuration duration = new TimeDuration(Math.toIntExact(months), fixed); + return DateTimeUtils.calcPositiveIntervalByMonth(base, duration, zone); + } catch (ArithmeticException | DateTimeException e) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C, e); + } + } + + public static long occurrence(long boundary, TimeDuration every, long index, ZoneId zone) { + return apply(boundary, every, index, zone); + } + + public static long localEpochBoundary(ZoneId zone) { + return fromInstant(java.time.LocalDate.of(1970, 1, 1).atStartOfDay(zone).toInstant()); + } + + public static long firstOccurrenceIndex( + long boundary, TimeDuration every, long now, ZoneId zone) { + if (every.monthDuration <= 0 && every.nonMonthDuration <= 0) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26); + } + if (now <= boundary) { + return 0; + } + long high = 1; + while (occurrence(boundary, every, high, zone) < now) { + high = Math.multiplyExact(high, 2); + } + long low = high / 2; + while (low < high) { + long mid = low + (high - low) / 2; + if (occurrence(boundary, every, mid, zone) < now) { + low = mid + 1; + } else { + high = mid; + } + } + return low; + } + + public static long nextAfter(long boundary, TimeDuration every, long previous, ZoneId zone) { + long index = firstOccurrenceIndex(boundary, every, previous, zone); + if (occurrence(boundary, every, index, zone) <= previous) { + index = Math.addExact(index, 1); + } + return occurrence(boundary, every, index, zone); + } + + private static long fromInstant(Instant instant) { + String precision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + try { + if ("us".equals(precision)) { + return Math.addExact( + Math.multiplyExact(instant.getEpochSecond(), 1_000_000L), instant.getNano() / 1_000L); + } + if ("ns".equals(precision)) { + return Math.addExact( + Math.multiplyExact(instant.getEpochSecond(), 1_000_000_000L), instant.getNano()); + } + return Math.addExact( + Math.multiplyExact(instant.getEpochSecond(), 1_000L), instant.getNano() / 1_000_000L); + } catch (ArithmeticException e) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C, e); + } + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java index 29837bc8aa29f..0159cc3af3be3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java @@ -31,8 +31,10 @@ import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.persistence.cq.CQInfo; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowCQResp; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -75,6 +77,10 @@ public CQManager(ConfigManager configManager) { } public TSStatus createCQ(TCreateCQReq req) { + TSStatus validation = validateDurationEncoding(req); + if (validation != null) { + return validation; + } lock.readLock().lock(); try { ScheduledExecutorService currentExecutor = executor; @@ -84,6 +90,174 @@ public TSStatus createCQ(TCreateCQReq req) { } } + private TSStatus validateDurationEncoding(TCreateCQReq req) { + if (!req.isSetDurationEncodingVersion()) { + // New CQ creation must use the versioned representation. Legacy requests are still + // supported when loading old procedures/plans/snapshots, but accepting them here would let + // an old DataNode flatten a calendar duration and bypass the mixed-version capability gate. + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage(ManagerMessages.MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A); + } + if (req.getDurationEncodingVersion() != 1 + || !req.isSetEveryDuration() + || !req.isSetStartOffsetDuration() + || !req.isSetEndOffsetDuration() + || !req.isSetBoundaryExplicit()) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages + .MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92); + } + if (req.getEveryDuration().getMonthPart() < 0 + || req.getStartOffsetDuration().getMonthPart() < 0 + || req.getEndOffsetDuration().getMonthPart() < 0 + || req.getEveryDuration().getNonMonthDuration() < 0 + || req.getStartOffsetDuration().getNonMonthDuration() < 0 + || req.getEndOffsetDuration().getNonMonthDuration() < 0) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage(ManagerMessages.MESSAGE_CQ_DURATIONS_MUST_BE_NON_NEGATIVE_BE23CE04); + } + if (req.getEveryDuration().getMonthPart() > Integer.MAX_VALUE + || req.getStartOffsetDuration().getMonthPart() > Integer.MAX_VALUE + || req.getEndOffsetDuration().getMonthPart() > Integer.MAX_VALUE) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage(ManagerMessages.MESSAGE_CQ_DURATIONS_MUST_BE_NON_NEGATIVE_BE23CE04); + } + boolean hasCalendarDuration = + req.getEveryDuration().getMonthPart() != 0 + || req.getStartOffsetDuration().getMonthPart() != 0 + || req.getEndOffsetDuration().getMonthPart() != 0; + // If any structured component contains a calendar month, all legacy fields must carry the + // invalid zero sentinel. Otherwise they must exactly mirror the fixed structured values. + boolean legacyFieldsMatch = + hasCalendarDuration + ? req.everyInterval == 0 && req.startTimeOffset == 0 && req.endTimeOffset == 0 + : req.everyInterval == req.getEveryDuration().getNonMonthDuration() + && req.startTimeOffset == req.getStartOffsetDuration().getNonMonthDuration() + && req.endTimeOffset == req.getEndOffsetDuration().getNonMonthDuration(); + if (!legacyFieldsMatch) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages + .MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67); + } + TSStatus semanticValidation = validateDurationSemantics(req); + if (semanticValidation != null) { + return semanticValidation; + } + if (hasCalendarDuration && !allClusterNodesSupportDurationEncodingV1()) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages.MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072); + } + return null; + } + + private TSStatus validateDurationSemantics(TCreateCQReq req) { + TCQDuration every = req.getEveryDuration(); + TCQDuration start = req.getStartOffsetDuration(); + TCQDuration end = req.getEndOffsetDuration(); + + if (!isPositive(every)) { + return semanticError(ManagerMessages.EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26); + } + if (!isPositive(start)) { + return semanticError(ManagerMessages.MESSAGE_CQ_START_OFFSET_MUST_BE_POSITIVE_B837C4F5); + } + if (end.getMonthPart() < 0 || end.getNonMonthDuration() < 0) { + return semanticError(ManagerMessages.MESSAGE_CQ_END_OFFSET_MUST_BE_NON_NEGATIVE_64171164); + } + if (!dominates(start, end, true)) { + return semanticError( + ManagerMessages.MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_END_OFFSET_5924C189); + } + if (!dominates(start, every, false)) { + return semanticError( + ManagerMessages + .MESSAGE_CQ_START_OFFSET_MUST_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_DURATION_89628D43); + } + return null; + } + + private static TSStatus semanticError(String message) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()).setMessage(message); + } + + private static boolean isPositive(TCQDuration duration) { + return duration.getMonthPart() > 0 || duration.getNonMonthDuration() > 0; + } + + private static boolean dominates(TCQDuration left, TCQDuration right, boolean strict) { + boolean result = + left.getMonthPart() >= right.getMonthPart() + && left.getNonMonthDuration() >= right.getNonMonthDuration(); + return result + && (!strict + || left.getMonthPart() != right.getMonthPart() + || left.getNonMonthDuration() != right.getNonMonthDuration()); + } + + /** Returns true when any persisted CQ requires the structured calendar-duration reader. */ + public boolean hasCalendarDurationCQ() { + try { + DataSet response = configManager.getConsensusManager().read(new ShowCQPlan()); + if (!(response instanceof ShowCQResp)) { + // Do not allow a node with unknown metadata to join while the reader barrier is active. + return true; + } + if (((ShowCQResp) response).getCqList() == null) { + // A malformed response is just as unsafe as an unavailable response for this barrier. + return true; + } + return ((ShowCQResp) response) + .getCqList().stream().anyMatch(CQInfo.CQEntry::hasCalendarDuration); + } catch (ConsensusException e) { + // A failed metadata read must fail closed: an old reader must never be admitted blindly. + LOGGER.warn(ManagerMessages.UNEXPECTED_ERROR_HAPPENED_WHILE_FETCHING_CQ_LIST, e); + return true; + } + } + + private boolean allClusterNodesSupportDurationEncodingV1() { + java.util.Map versionInfo = + configManager.getNodeManager().getNodeVersionInfo(); + if (versionInfo == null || versionInfo.isEmpty()) { + return false; + } + boolean hasRegisteredNode = false; + // Check every registered node explicitly. A missing heartbeat/version entry must not allow a + // calendar CQ to be created during a rolling upgrade. + List configNodes = + configManager.getNodeManager().getRegisteredConfigNodes(); + List dataNodes = + configManager.getNodeManager().getRegisteredDataNodes(); + if (configNodes == null || dataNodes == null) { + return false; + } + for (org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation node : configNodes) { + hasRegisteredNode = true; + if (!supportsDurationEncodingV1(versionInfo.get(node.getConfigNodeId()))) { + return false; + } + } + for (org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration node : dataNodes) { + hasRegisteredNode = true; + if (!supportsDurationEncodingV1(versionInfo.get(node.getLocation().getDataNodeId()))) { + return false; + } + } + return hasRegisteredNode; + } + + private boolean supportsDurationEncodingV1(TNodeVersionInfo info) { + if (info == null + || !info.isSetSupportedCQDurationEncodingVersions() + || !info.getSupportedCQDurationEncodingVersions().contains((short) 1)) { + return false; + } + return true; + } + public TSStatus dropCQ(TDropCQReq req) { lock.readLock().lock(); try { @@ -208,6 +382,32 @@ public void stopCQScheduler() { } } + /** Reconciles a callback after an ambiguous or stale progress write. */ + public void reconcileCQ(String cqId, String cqToken) { + if (!configManager.getConsensusManager().isLeader()) { + return; + } + try { + ShowCQResp response = + (ShowCQResp) configManager.getConsensusManager().read(new ShowCQPlan(cqId)); + response.getCqList().stream() + .filter(entry -> cqToken.equals(entry.getCqToken()) && entry.getState() == CQState.ACTIVE) + .findFirst() + .ifPresent( + entry -> { + // The failed callback is still registered under this token. Remove that finished + // task before installing the task rebuilt from durable progress. + unmarkCQLocallyScheduled(cqId, cqToken); + CQScheduleTask task = new CQScheduleTask(entry, executor, configManager); + if (markCQLocallyScheduled(cqId, cqToken, task)) { + task.submitSelf(); + } + }); + } catch (ConsensusException | RuntimeException e) { + LOGGER.warn(ManagerMessages.UNEXPECTED_ERROR_HAPPENED_WHILE_FETCHING_CQ_LIST, e); + } + } + public boolean markCQLocallyScheduled(String cqId, String cqToken, CQScheduleTask task) { AtomicBoolean shouldSchedule = new AtomicBoolean(false); LocallyScheduledCQ schedule = new LocallyScheduledCQ(cqToken, task); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java index 6b73ffca95fd5..8d3b9b8454f00 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java @@ -35,9 +35,11 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.tsfile.utils.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.ZoneId; import java.util.Optional; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -83,12 +85,22 @@ public class CQScheduleTask implements Runnable { private final ConfigManager configManager; - private final long retryWaitTimeInMS; + private long retryWaitTimeInMS; private final AtomicBoolean cancelled; private final AtomicReference> scheduledFuture; private long executionTime; + private TimeDuration everyDuration; + private TimeDuration startDuration; + private TimeDuration endDuration; + private long boundaryTime; + private boolean calendarAware; + private boolean scheduleCalendarAware; + private ZoneId scheduleZone; + + /** First occurrence not yet durably acknowledged. -1 denotes a legacy CQ. */ + private long occurrenceIndex = -1; public CQScheduleTask( TCreateCQReq req, @@ -109,6 +121,51 @@ public CQScheduleTask( executor, configManager, firstExecutionTime); + this.everyDuration = + req.isSetEveryDuration() + ? new TimeDuration( + Math.toIntExact(req.getEveryDuration().getMonthPart()), + req.getEveryDuration().getNonMonthDuration()) + : new TimeDuration(0, req.everyInterval); + this.startDuration = + req.isSetStartOffsetDuration() + ? new TimeDuration( + Math.toIntExact(req.getStartOffsetDuration().getMonthPart()), + req.getStartOffsetDuration().getNonMonthDuration()) + : new TimeDuration(0, req.startTimeOffset); + this.endDuration = + req.isSetEndOffsetDuration() + ? new TimeDuration( + Math.toIntExact(req.getEndOffsetDuration().getMonthPart()), + req.getEndOffsetDuration().getNonMonthDuration()) + : new TimeDuration(0, req.endTimeOffset); + this.calendarAware = + everyDuration.monthDuration != 0 + || startDuration.monthDuration != 0 + || endDuration.monthDuration != 0; + this.scheduleCalendarAware = everyDuration.monthDuration != 0; + this.boundaryTime = req.boundaryTime; + // Fixed-duration CQs do not need calendar arithmetic. Keep the zone opaque in that case so + // legacy requests containing a non-canonical zone string remain compatible. + this.scheduleZone = calendarAware ? ZoneId.of(req.zoneId) : null; + if (calendarAware) { + this.retryWaitTimeInMS = calculateRetryWaitTime(everyDuration); + } + if (scheduleCalendarAware && req.isSetBoundaryExplicit() && !req.isBoundaryExplicit()) { + this.boundaryTime = CQCalendarUtils.localEpochBoundary(scheduleZone); + } + // The procedure has already selected the first occurrence. Recomputing it from wall clock + // time here introduces a race around a calendar boundary and can skip an occurrence. + if (req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1) { + this.occurrenceIndex = + CQCalendarUtils.firstOccurrenceIndex( + boundaryTime, everyDuration, firstExecutionTime, scheduleZone); + long expectedExecution = occurrenceAt(this.occurrenceIndex); + if (expectedExecution != firstExecutionTime) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_7E4B91A2); + } + } } public CQScheduleTask( @@ -126,6 +183,40 @@ public CQScheduleTask( executor, configManager, entry.getLastExecutionTime() + entry.getEveryInterval()); + this.everyDuration = entry.getEveryDuration(); + this.startDuration = entry.getStartTimeOffsetDuration(); + this.endDuration = entry.getEndTimeOffsetDuration(); + this.calendarAware = + everyDuration.monthDuration != 0 + || startDuration.monthDuration != 0 + || endDuration.monthDuration != 0; + this.scheduleCalendarAware = everyDuration.monthDuration != 0; + this.boundaryTime = entry.getBoundaryTime(); + // Fixed-duration CQs do not need calendar arithmetic. Keep the zone opaque in that case so + // legacy persisted entries containing a non-canonical zone string remain compatible. + this.scheduleZone = calendarAware ? ZoneId.of(entry.getZoneId()) : null; + if (scheduleCalendarAware) { + this.retryWaitTimeInMS = calculateRetryWaitTime(everyDuration); + if (!entry.isBoundaryExplicit()) { + this.boundaryTime = CQCalendarUtils.localEpochBoundary(scheduleZone); + } + long index = + CQCalendarUtils.firstOccurrenceIndex( + boundaryTime, everyDuration, entry.getLastExecutionTime(), scheduleZone); + if (CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, scheduleZone) + <= entry.getLastExecutionTime()) { + index = Math.addExact(index, 1); + } + this.executionTime = + CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, scheduleZone); + } + if (entry.getNextOccurrenceIndex() >= 0) { + this.occurrenceIndex = entry.getNextOccurrenceIndex(); + if (!entry.isBoundaryExplicit() && scheduleCalendarAware) { + this.boundaryTime = CQCalendarUtils.localEpochBoundary(scheduleZone); + } + this.executionTime = occurrenceAt(this.occurrenceIndex); + } } @SuppressWarnings("squid:S107") @@ -157,6 +248,13 @@ public CQScheduleTask( this.cancelled = new AtomicBoolean(false); this.scheduledFuture = new AtomicReference<>(); this.executionTime = executionTime; + this.everyDuration = new TimeDuration(0, everyInterval); + this.startDuration = new TimeDuration(0, startTimeOffset); + this.endDuration = new TimeDuration(0, endTimeOffset); + this.boundaryTime = 0; + // This constructor is used by the legacy fixed-duration path. Calendar-aware constructors + // initialize the zone after determining whether calendar arithmetic is required. + this.scheduleZone = null; } public static long getFirstExecutionTime(long boundaryTime, long everyInterval) { @@ -172,13 +270,39 @@ public static long getFirstExecutionTime(long boundaryTime, long everyInterval, } } + public static long getFirstExecutionTime( + long boundaryTime, TimeDuration everyDuration, long now, ZoneId zoneId) { + long index = CQCalendarUtils.firstOccurrenceIndex(boundaryTime, everyDuration, now, zoneId); + return CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, zoneId); + } + @Override public void run() { + long currentOccurrenceIndex = occurrenceIndex; if (cancelled.get()) { return; } long startTime = executionTime - startTimeOffset; long endTime = executionTime - endTimeOffset; + if (calendarAware) { + if (currentOccurrenceIndex < 0) { + throw new IllegalStateException( + ManagerMessages.MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C); + } + long expectedExecution = + CQCalendarUtils.occurrence( + boundaryTime, everyDuration, currentOccurrenceIndex, scheduleZone); + if (expectedExecution != executionTime) { + throw new IllegalStateException( + ManagerMessages.EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_7E4B91A2); + } + startTime = calculateCalendarRangeEndpoint(startDuration, currentOccurrenceIndex); + endTime = calculateCalendarRangeEndpoint(endDuration, currentOccurrenceIndex); + if (endTime <= startTime) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_RANGE_END_MUST_BE_GREATER_THAN_START_3C91E8B4); + } + } Optional targetDataNode = configManager.getNodeManager().getLowestLoadDataNode(); @@ -200,12 +324,22 @@ public void run() { endTime, System.currentTimeMillis() * FACTOR); TExecuteCQ executeCQReq = - new TExecuteCQ(queryBody, startTime, endTime, everyInterval, zoneId, cqId, username); + new TExecuteCQ( + queryBody, + startTime, + endTime, + calendarAware + ? calculateCalendarTimeoutMillis(currentOccurrenceIndex) + : toTimeoutMillis(everyInterval), + zoneId, + cqId, + username); try { AsyncDataNodeInternalServiceClient client = CnToDnInternalServiceAsyncRequestManager.getInstance() .getAsyncClient(targetDataNode.get()); - client.executeCQ(executeCQReq, new AsyncExecuteCQCallback(startTime, endTime)); + client.executeCQ( + executeCQReq, new AsyncExecuteCQCallback(startTime, endTime, currentOccurrenceIndex)); } catch (Exception t) { LOGGER.warn(ManagerMessages.EXECUTE_CQ_FAILED, cqId, t); if (needSubmit()) { @@ -215,6 +349,59 @@ public void run() { } } + long calculateCalendarRangeEndpoint(TimeDuration offset, long currentOccurrenceIndex) { + // Derive every RANGE endpoint from the original boundary. Subtracting a calendar offset from + // an already materialized occurrence is not reversible at month ends and produces gaps or + // overlaps for fixed EVERY intervals combined with calendar RANGE offsets. + return CQCalendarUtils.applyVector( + boundaryTime, + Math.subtractExact( + Math.multiplyExact((long) everyDuration.monthDuration, currentOccurrenceIndex), + offset.monthDuration), + Math.subtractExact( + Math.multiplyExact(everyDuration.nonMonthDuration, currentOccurrenceIndex), + offset.nonMonthDuration), + scheduleZone); + } + + private static long toTimeoutMillis(long deltaTicks) { + if (deltaTicks <= 0) { + return 1; + } + try { + return Math.addExact(deltaTicks, FACTOR - 1) / FACTOR; + } catch (ArithmeticException e) { + return Long.MAX_VALUE; + } + } + + private long calculateCalendarTimeoutMillis(long currentOccurrenceIndex) { + try { + long nextOccurrence = + CQCalendarUtils.occurrence( + boundaryTime, everyDuration, Math.addExact(currentOccurrenceIndex, 1), scheduleZone); + return toTimeoutMillis(Math.subtractExact(nextOccurrence, executionTime)); + } catch (ArithmeticException e) { + throw new IllegalArgumentException( + ManagerMessages.EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C, e); + } + } + + private static long calculateRetryWaitTime(TimeDuration duration) { + if (duration.nonMonthDuration <= 0) { + return DEFAULT_RETRY_WAIT_TIME_IN_MS; + } + return Math.min( + DEFAULT_RETRY_WAIT_TIME_IN_MS, Math.max(1L, duration.nonMonthDuration / FACTOR)); + } + + private long occurrenceAt(long index) { + if (occurrenceIndex < 0) { + return executionTime + (index - occurrenceIndex) * everyInterval; + } + return CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, scheduleZone); + } + public void submitSelf() { submitSelf( Math.max(0, executionTime / FACTOR - System.currentTimeMillis()), TimeUnit.MILLISECONDS); @@ -253,37 +440,45 @@ private class AsyncExecuteCQCallback implements AsyncMethodCallback { private final long startTime; private final long endTime; + private final long expectedIndex; - public AsyncExecuteCQCallback(long startTime, long endTime) { + public AsyncExecuteCQCallback(long startTime, long endTime, long expectedIndex) { this.startTime = startTime; this.endTime = endTime; + this.expectedIndex = expectedIndex; + } + + private long nextOccurrenceIndex(long callbackTime) { + return calculateNextOccurrenceIndex( + timeoutPolicy, + expectedIndex, + callbackTime, + executionTime, + everyInterval, + occurrenceIndex, + boundaryTime, + everyDuration, + scheduleZone); } - private void updateExecutionTime() { + private void advanceLegacyExecutionTime(long callbackTime) { if (timeoutPolicy == TimeoutPolicy.BLOCKED) { - executionTime = executionTime + everyInterval; - } else if (timeoutPolicy == TimeoutPolicy.DISCARD) { - long now = System.currentTimeMillis() * FACTOR; - executionTime = - executionTime + ((now - executionTime - 1) / everyInterval + 1) * everyInterval; + executionTime = Math.addExact(executionTime, everyInterval); } else { - throw new IllegalArgumentException(ManagerMessages.UNKNOWN_TIMEOUTPOLICY + timeoutPolicy); + if (callbackTime <= executionTime) { + executionTime = Math.addExact(executionTime, everyInterval); + return; + } + executionTime = + Math.addExact( + executionTime, + Math.multiplyExact( + ((callbackTime - executionTime - 1) / everyInterval + 1), everyInterval)); } } - @Override - public void onComplete(TSStatus response) { - if (cancelled.get()) { - return; - } - if (response.code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - - LOGGER.info( - ManagerMessages.ENDEXECUTECQ_TIME_RANGE_IS_CURRENT_TIME_IS, - cqId, - startTime, - endTime, - System.currentTimeMillis() * FACTOR); + private void persistProgress(long targetIndex, long callbackTime) { + if (occurrenceIndex < 0) { TSStatus result; try { result = @@ -294,29 +489,73 @@ public void onComplete(TSStatus response) { result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); result.setMessage(e.getMessage()); } - - // while leadership changed, the update last exec time operation for CQTasks in new leader - // may still update failed because stale CQTask in old leader may update it in advance - if (result.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - LOGGER.warn( - ManagerMessages.FAILED_TO_UPDATE_THE_LAST_EXECUTION_TIME_OF_CQ_BECAUSE, - executionTime, - cqId, - result.getMessage()); - // no such cq, we don't need to submit it again - if (result.getCode() == TSStatusCode.NO_SUCH_CQ.getStatusCode()) { - LOGGER.info(ManagerMessages.STOP_SUBMITTING_CQ_BECAUSE, cqId, result.getMessage()); - return; + if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (needSubmit()) { + advanceLegacyExecutionTime(callbackTime); + submitSelf(); } + } else if (result.getCode() == TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + && needSubmit()) { + executor.schedule( + () -> persistProgress(targetIndex, callbackTime), + retryWaitTimeInMS, + TimeUnit.MILLISECONDS); } - + return; + } + long targetLastExecution = occurrenceAt(targetIndex - 1); + TSStatus result; + try { + result = + configManager + .getConsensusManager() + .write( + new UpdateCQLastExecTimePlan( + cqId, targetLastExecution, cqToken, expectedIndex, targetIndex)); + } catch (ConsensusException e) { + result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + result.setMessage(e.getMessage()); + } + if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + occurrenceIndex = targetIndex; + executionTime = occurrenceAt(targetIndex); if (needSubmit()) { - updateExecutionTime(); submitSelf(); - } else { - LOGGER.info( - ManagerMessages.STOP_SUBMITTING_CQ_BECAUSE_CURRENT_NODE_IS_NOT_LEADER_OR, cqId); } + } else if (result.getCode() == TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + && needSubmit()) { + // Retry exactly the same CAS transition; never execute the query again. + executor.schedule( + () -> persistProgress(targetIndex, callbackTime), + retryWaitTimeInMS, + TimeUnit.MILLISECONDS); + } else if (needSubmit()) { + // The write may have committed before its response was lost. Re-read durable progress + // and let CQManager install the single task for the current token/index. + configManager.getCQManager().reconcileCQ(cqId, cqToken); + } + } + + @Override + public void onComplete(TSStatus response) { + if (cancelled.get()) { + return; + } + if (response.code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + + long callbackTime = System.currentTimeMillis() * FACTOR; + + LOGGER.info( + ManagerMessages.ENDEXECUTECQ_TIME_RANGE_IS_CURRENT_TIME_IS, + cqId, + startTime, + endTime, + callbackTime); + long targetIndex = + occurrenceIndex >= 0 + ? nextOccurrenceIndex(callbackTime) + : Math.addExact(expectedIndex, 1); + persistProgress(targetIndex, callbackTime); } else { LOGGER.warn(ManagerMessages.EXECUTE_CQ_FAILED_TSSTATUS_IS, cqId, response); @@ -337,4 +576,39 @@ public void onError(Exception exception) { } } } + + /** Calculates the next occurrence selected after a successful callback. */ + static long calculateNextOccurrenceIndex( + TimeoutPolicy timeoutPolicy, + long expectedIndex, + long callbackTime, + long executionTime, + long everyInterval, + long occurrenceIndex, + long boundaryTime, + TimeDuration everyDuration, + ZoneId scheduleZone) { + long next = Math.addExact(expectedIndex, 1); + if (timeoutPolicy == TimeoutPolicy.BLOCKED) { + return next; + } + if (timeoutPolicy == TimeoutPolicy.DISCARD) { + long lowerBound = + occurrenceIndex >= 0 + ? CQCalendarUtils.firstOccurrenceIndex( + boundaryTime, everyDuration, callbackTime, scheduleZone) + : calculateFixedLowerBound(expectedIndex, callbackTime, executionTime, everyInterval); + return Math.max(next, lowerBound); + } + throw new IllegalArgumentException(ManagerMessages.UNKNOWN_TIMEOUTPOLICY + timeoutPolicy); + } + + private static long calculateFixedLowerBound( + long expectedIndex, long callbackTime, long executionTime, long everyInterval) { + if (callbackTime <= executionTime) { + return expectedIndex; + } + return Math.addExact( + expectedIndex, Math.addExact((callbackTime - executionTime - 1) / everyInterval, 1)); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 64d4cf741e159..35a51b0929c49 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -331,6 +331,11 @@ private TRuntimeConfiguration getRuntimeConfiguration(int dataNodeId) { public DataSet registerDataNode(TDataNodeRegisterReq req) { DataNodeRegisterResp resp = new DataNodeRegisterResp(); resp.setConfigNodeList(getRegisteredConfigNodes()); + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + resp.setStatus(capabilityStatus); + return resp; + } // Create a new DataNodeHeartbeatCache and force update NodeStatus int dataNodeId = nodeInfo.generateNextNodeId(); @@ -371,13 +376,18 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { } public TDataNodeRestartResp updateDataNodeIfNecessary(TDataNodeRestartReq req) { + TDataNodeRestartResp resp = new TDataNodeRestartResp(); + resp.setConfigNodeList(getRegisteredConfigNodes()); + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + resp.setStatus(capabilityStatus); + return resp; + } final String clusterId = configManager .getClusterManager() .getClusterIdWithRetry( CommonDescriptor.getInstance().getConfig().getCnConnectionTimeoutInMS() / 2); - TDataNodeRestartResp resp = new TDataNodeRestartResp(); - resp.setConfigNodeList(getRegisteredConfigNodes()); if (clusterId == null) { resp.setStatus( new TSStatus(TSStatusCode.GET_CLUSTER_ID_ERROR.getStatusCode()) @@ -473,6 +483,10 @@ public DataSet removeDataNode(RemoveDataNodePlan removeDataNodePlan) { } public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + return new TConfigNodeRegisterResp().setStatus(capabilityStatus).setConfigNodeId(-1); + } int nodeId = nodeInfo.generateNextNodeId(); req.getConfigNodeLocation().setConfigNodeId(nodeId); configManager.getProcedureManager().addConfigNode(req); @@ -482,6 +496,10 @@ public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { } public TSStatus updateConfigNodeIfNecessary(int configNodeId, TNodeVersionInfo versionInfo) { + TSStatus capabilityStatus = validateDurationCapability(versionInfo); + if (capabilityStatus != null) { + return capabilityStatus; + } TNodeVersionInfo recordVersionInfo = nodeInfo.getVersionInfo(configNodeId); if (!recordVersionInfo.equals(versionInfo)) { // Update versionInfo when modified during restart @@ -496,6 +514,22 @@ public TSStatus updateConfigNodeIfNecessary(int configNodeId, TNodeVersionInfo v return ClusterNodeStartUtils.ACCEPT_NODE_RESTART; } + private TSStatus validateDurationCapability(TNodeVersionInfo versionInfo) { + if (!supportsDurationEncodingV1(versionInfo) + && configManager.getCQManager().hasCalendarDurationCQ()) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages.MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072); + } + return null; + } + + private static boolean supportsDurationEncodingV1(TNodeVersionInfo versionInfo) { + return versionInfo != null + && versionInfo.isSetSupportedCQDurationEncodingVersions() + && versionInfo.getSupportedCQDurationEncodingVersions().contains((short) 1); + } + public List getRegisteredAINodeInfoList() { List aiNodeInfoList = new ArrayList<>(); for (TAINodeConfiguration aiNodeConfiguration : getRegisteredAINodes()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java index 013e2415f9445..5af0d9794e034 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java @@ -30,16 +30,21 @@ import org.apache.iotdb.confignode.consensus.request.write.cq.UpdateCQLastExecTimePlan; import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; +import org.apache.iotdb.confignode.i18n.ManagerMessages; +import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.concurrent.ThreadSafe; +import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -62,6 +67,8 @@ public class CQInfo implements SnapshotProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(CQInfo.class); private static final String SNAPSHOT_FILENAME = "cq_info.snapshot"; + // Optional tail marker. The legacy CQ records stay byte-for-byte compatible with master. + private static final int SNAPSHOT_EXTENSION_MARKER = 0x43515631; private static final String CQ_NOT_EXIST_FORMAT = "CQ %s doesn't exist."; @@ -92,11 +99,54 @@ public TSStatus addCQ(AddCQPlan plan) { res.code = TSStatusCode.CQ_ALREADY_EXIST.getStatusCode(); res.message = String.format("CQ %s has already been created.", cqId); } else { + TCreateCQReq req = plan.getReq(); + TimeDuration everyDuration = + req.isSetDurationEncodingVersion() + && req.getDurationEncodingVersion() == 1 + && req.isSetEveryDuration() + ? new TimeDuration( + Math.toIntExact(req.getEveryDuration().getMonthPart()), + req.getEveryDuration().getNonMonthDuration()) + : new TimeDuration(0, req.everyInterval); + long lastExecutionTime; + if (everyDuration.monthDuration != 0) { + java.time.ZoneId zone = java.time.ZoneId.of(plan.getReq().zoneId); + long boundary = + req.isSetBoundaryExplicit() && !req.isBoundaryExplicit() + ? CQCalendarUtils.localEpochBoundary(zone) + : req.boundaryTime; + long index = + CQCalendarUtils.firstOccurrenceIndex( + boundary, everyDuration, plan.getFirstExecutionTime(), zone); + lastExecutionTime = CQCalendarUtils.occurrence(boundary, everyDuration, index - 1, zone); + } else { + // Version 1 may carry zero legacy fields when another component is calendar-aware. Use + // the structured fixed duration to keep the persisted previous occurrence accurate. + lastExecutionTime = plan.getFirstExecutionTime() - everyDuration.nonMonthDuration; + } + long nextOccurrenceIndex = -1; + if (req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1) { + TimeDuration duration = + new TimeDuration( + Math.toIntExact(req.getEveryDuration().getMonthPart()), + req.getEveryDuration().getNonMonthDuration()); + boolean calendarAware = + req.getEveryDuration().getMonthPart() != 0 + || req.getStartOffsetDuration().getMonthPart() != 0 + || req.getEndOffsetDuration().getMonthPart() != 0; + java.time.ZoneId zone = calendarAware ? java.time.ZoneId.of(req.zoneId) : null; + long boundary = + duration.monthDuration != 0 + && req.isSetBoundaryExplicit() + && !req.isBoundaryExplicit() + ? CQCalendarUtils.localEpochBoundary(zone) + : req.boundaryTime; + nextOccurrenceIndex = + CQCalendarUtils.firstOccurrenceIndex( + boundary, duration, plan.getFirstExecutionTime(), zone); + } CQEntry cqEntry = - new CQEntry( - plan.getReq(), - plan.getCqToken(), - plan.getFirstExecutionTime() - plan.getReq().everyInterval); + new CQEntry(plan.getReq(), plan.getCqToken(), lastExecutionTime, nextOccurrenceIndex); cqMap.put(cqId, cqEntry); res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); } @@ -213,6 +263,22 @@ public TSStatus updateCQLastExecutionTime(UpdateCQLastExecTimePlan plan) { } else if (!cqToken.equals(cqEntry.cqToken)) { res.code = TSStatusCode.NO_SUCH_CQ.getStatusCode(); res.message = String.format(CQ_TOKEN_NOT_MATCH_FORMAT, cqId); + } else if (plan.hasOccurrenceIndex()) { + if (cqEntry.nextOccurrenceIndex < 0) { + res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); + res.message = ManagerMessages.MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C; + } else if (cqEntry.nextOccurrenceIndex > plan.getExpectedIndex()) { + res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); + res.message = ManagerMessages.MESSAGE_CQ_OCCURRENCE_CALLBACK_IS_STALE_36C5FBFC; + } else if (cqEntry.nextOccurrenceIndex < plan.getExpectedIndex()) { + res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); + res.message = + ManagerMessages.MESSAGE_CQ_OCCURRENCE_INDEX_IS_AHEAD_OF_THE_CALLBACK_8A18ECC9; + } else { + cqEntry.nextOccurrenceIndex = plan.getTargetIndex(); + cqEntry.lastExecutionTime = plan.getExecutionTime(); + res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } } else if (cqEntry.lastExecutionTime >= plan.getExecutionTime()) { res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); res.message = @@ -253,16 +319,52 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept private void serialize(OutputStream stream) throws IOException { ReadWriteIOUtils.write(cqMap.size(), stream); for (CQEntry entry : cqMap.values()) { - entry.serialize(stream); + entry.serializeLegacy(stream); + } + ReadWriteIOUtils.write(SNAPSHOT_EXTENSION_MARKER, stream); + ReadWriteIOUtils.write(cqMap.size(), stream); + for (CQEntry entry : cqMap.values()) { + entry.serializeExtension(stream); } } private void deserialize(InputStream stream) throws IOException { int size = ReadWriteIOUtils.readInt(stream); + if (size < 0) { + throw new IOException( + String.format( + ManagerMessages.EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035, size)); + } for (int i = 0; i < size; i++) { - CQEntry cqEntry = CQEntry.deserialize(stream); + CQEntry cqEntry = CQEntry.deserializeLegacy(stream); cqMap.put(cqEntry.cqId, cqEntry); } + final int extensionMarker; + try { + extensionMarker = ReadWriteIOUtils.readInt(stream); + } catch (EOFException e) { + // Legacy snapshots end immediately after the legacy records. + return; + } + if (extensionMarker != SNAPSHOT_EXTENSION_MARKER) { + return; + } + int extensionSize = ReadWriteIOUtils.readInt(stream); + if (extensionSize < 0) { + throw new IOException( + String.format( + ManagerMessages.EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035, + extensionSize)); + } + for (int i = 0; i < extensionSize; i++) { + String cqId = ReadWriteIOUtils.readString(stream); + CQEntry cqEntry = cqMap.get(cqId); + if (cqEntry == null) { + CQEntry.skipExtension(stream); + } else { + cqEntry.deserializeExtension(stream); + } + } } @Override @@ -321,11 +423,21 @@ public static class CQEntry { private final String zoneId; private final String username; + private org.apache.tsfile.utils.TimeDuration everyDuration; + private org.apache.tsfile.utils.TimeDuration startTimeOffsetDuration; + private org.apache.tsfile.utils.TimeDuration endTimeOffsetDuration; + private boolean boundaryExplicit; private CQState state; private long lastExecutionTime; + private long nextOccurrenceIndex; private CQEntry(TCreateCQReq req, String cqToken, long lastExecutionTime) { + this(req, cqToken, lastExecutionTime, -1); + } + + private CQEntry( + TCreateCQReq req, String cqToken, long lastExecutionTime, long nextOccurrenceIndex) { this( req.cqId, req.everyInterval, @@ -338,8 +450,20 @@ private CQEntry(TCreateCQReq req, String cqToken, long lastExecutionTime) { cqToken, req.zoneId, req.username, + durationFromReq( + req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval), + durationFromReq( + req, + req.isSetStartOffsetDuration() ? req.getStartOffsetDuration() : null, + req.startTimeOffset), + durationFromReq( + req, + req.isSetEndOffsetDuration() ? req.getEndOffsetDuration() : null, + req.endTimeOffset), + req.isSetBoundaryExplicit() && req.isBoundaryExplicit(), CQState.INACTIVE, - lastExecutionTime); + lastExecutionTime, + nextOccurrenceIndex); } private CQEntry(CQEntry other) { @@ -355,8 +479,13 @@ private CQEntry(CQEntry other) { other.cqToken, other.zoneId, other.username, + other.everyDuration, + other.startTimeOffsetDuration, + other.endTimeOffsetDuration, + other.boundaryExplicit, other.state, - other.lastExecutionTime); + other.lastExecutionTime, + other.nextOccurrenceIndex); } @SuppressWarnings("squid:S107") @@ -372,8 +501,13 @@ private CQEntry( String cqToken, String zoneId, String username, + org.apache.tsfile.utils.TimeDuration everyDuration, + org.apache.tsfile.utils.TimeDuration startTimeOffsetDuration, + org.apache.tsfile.utils.TimeDuration endTimeOffsetDuration, + boolean boundaryExplicit, CQState state, - long lastExecutionTime) { + long lastExecutionTime, + long nextOccurrenceIndex) { this.cqId = cqId; this.everyInterval = everyInterval; this.boundaryTime = boundaryTime; @@ -385,11 +519,16 @@ private CQEntry( this.cqToken = cqToken; this.zoneId = zoneId; this.username = username; + this.everyDuration = everyDuration; + this.startTimeOffsetDuration = startTimeOffsetDuration; + this.endTimeOffsetDuration = endTimeOffsetDuration; + this.boundaryExplicit = boundaryExplicit; this.state = state; this.lastExecutionTime = lastExecutionTime; + this.nextOccurrenceIndex = nextOccurrenceIndex; } - private void serialize(OutputStream stream) throws IOException { + private void serializeLegacy(OutputStream stream) throws IOException { ReadWriteIOUtils.write(cqId, stream); ReadWriteIOUtils.write(everyInterval, stream); ReadWriteIOUtils.write(boundaryTime, stream); @@ -405,7 +544,19 @@ private void serialize(OutputStream stream) throws IOException { ReadWriteIOUtils.write(lastExecutionTime, stream); } - private static CQEntry deserialize(InputStream stream) throws IOException { + private void serializeExtension(OutputStream stream) throws IOException { + ReadWriteIOUtils.write(cqId, stream); + ReadWriteIOUtils.write(everyDuration.monthDuration, stream); + ReadWriteIOUtils.write(everyDuration.nonMonthDuration, stream); + ReadWriteIOUtils.write(startTimeOffsetDuration.monthDuration, stream); + ReadWriteIOUtils.write(startTimeOffsetDuration.nonMonthDuration, stream); + ReadWriteIOUtils.write(endTimeOffsetDuration.monthDuration, stream); + ReadWriteIOUtils.write(endTimeOffsetDuration.nonMonthDuration, stream); + ReadWriteIOUtils.write(boundaryExplicit, stream); + ReadWriteIOUtils.write(nextOccurrenceIndex, stream); + } + + private static CQEntry deserializeLegacy(InputStream stream) throws IOException { String cqId = ReadWriteIOUtils.readString(stream); long everyInterval = ReadWriteIOUtils.readLong(stream); long boundaryTime = ReadWriteIOUtils.readLong(stream); @@ -431,8 +582,38 @@ private static CQEntry deserialize(InputStream stream) throws IOException { cqToken, zoneId, username, + new org.apache.tsfile.utils.TimeDuration(0, everyInterval), + new org.apache.tsfile.utils.TimeDuration(0, startTimeOffset), + new org.apache.tsfile.utils.TimeDuration(0, endTimeOffset), + false, state, - lastExecutionTime); + lastExecutionTime, + -1); + } + + private void deserializeExtension(InputStream stream) throws IOException { + everyDuration = + new org.apache.tsfile.utils.TimeDuration( + ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + startTimeOffsetDuration = + new org.apache.tsfile.utils.TimeDuration( + ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + endTimeOffsetDuration = + new org.apache.tsfile.utils.TimeDuration( + ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + boundaryExplicit = ReadWriteIOUtils.readBool(stream); + nextOccurrenceIndex = ReadWriteIOUtils.readLong(stream); + } + + private static void skipExtension(InputStream stream) throws IOException { + ReadWriteIOUtils.readInt(stream); + ReadWriteIOUtils.readLong(stream); + ReadWriteIOUtils.readInt(stream); + ReadWriteIOUtils.readLong(stream); + ReadWriteIOUtils.readInt(stream); + ReadWriteIOUtils.readLong(stream); + ReadWriteIOUtils.readBool(stream); + ReadWriteIOUtils.readLong(stream); } public String getCqId() { @@ -479,6 +660,10 @@ public long getLastExecutionTime() { return lastExecutionTime; } + public long getNextOccurrenceIndex() { + return nextOccurrenceIndex; + } + public String getZoneId() { return zoneId; } @@ -487,6 +672,39 @@ public String getUsername() { return username; } + public org.apache.tsfile.utils.TimeDuration getEveryDuration() { + return everyDuration; + } + + public org.apache.tsfile.utils.TimeDuration getStartTimeOffsetDuration() { + return startTimeOffsetDuration; + } + + public org.apache.tsfile.utils.TimeDuration getEndTimeOffsetDuration() { + return endTimeOffsetDuration; + } + + public boolean isBoundaryExplicit() { + return boundaryExplicit; + } + + public boolean hasCalendarDuration() { + return everyDuration.monthDuration != 0 + || startTimeOffsetDuration.monthDuration != 0 + || endTimeOffsetDuration.monthDuration != 0; + } + + private static org.apache.tsfile.utils.TimeDuration durationFromReq( + TCreateCQReq req, TCQDuration d, long legacy) { + if (req.isSetDurationEncodingVersion() + && req.getDurationEncodingVersion() == 1 + && d != null) { + return new org.apache.tsfile.utils.TimeDuration( + Math.toIntExact(d.getMonthPart()), d.getNonMonthDuration()); + } + return new org.apache.tsfile.utils.TimeDuration(0, legacy); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -501,6 +719,7 @@ public boolean equals(Object o) { && startTimeOffset == cqEntry.startTimeOffset && endTimeOffset == cqEntry.endTimeOffset && lastExecutionTime == cqEntry.lastExecutionTime + && nextOccurrenceIndex == cqEntry.nextOccurrenceIndex && Objects.equals(cqId, cqEntry.cqId) && timeoutPolicy == cqEntry.timeoutPolicy && Objects.equals(queryBody, cqEntry.queryBody) @@ -508,6 +727,10 @@ public boolean equals(Object o) { && Objects.equals(cqToken, cqEntry.cqToken) && Objects.equals(zoneId, cqEntry.zoneId) && Objects.equals(username, cqEntry.username) + && Objects.equals(everyDuration, cqEntry.everyDuration) + && Objects.equals(startTimeOffsetDuration, cqEntry.startTimeOffsetDuration) + && Objects.equals(endTimeOffsetDuration, cqEntry.endTimeOffsetDuration) + && boundaryExplicit == cqEntry.boundaryExplicit && state == cqEntry.state; } @@ -525,8 +748,13 @@ public int hashCode() { cqToken, zoneId, username, + everyDuration, + startTimeOffsetDuration, + endTimeOffsetDuration, + boundaryExplicit, state, - lastExecutionTime); + lastExecutionTime, + nextOccurrenceIndex); } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java index d98a05c8a8cdc..a6878b963cbed 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java @@ -68,6 +68,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -722,6 +723,19 @@ private void serializeVersionInfo(OutputStream outputStream) throws IOException ReadWriteIOUtils.write(entry.getValue().getVersion(), outputStream); ReadWriteIOUtils.write(entry.getValue().getBuildInfo(), outputStream); } + // Optional tail; legacy snapshots end after the two strings above. + ReadWriteIOUtils.write(0x43515631, outputStream); + ReadWriteIOUtils.write(nodeVersionInfo.size(), outputStream); + for (Entry entry : nodeVersionInfo.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + Set capabilities = entry.getValue().getSupportedCQDurationEncodingVersions(); + ReadWriteIOUtils.write(capabilities == null ? 0 : capabilities.size(), outputStream); + if (capabilities != null) { + for (short capability : capabilities) { + ReadWriteIOUtils.write(capability, outputStream); + } + } + } } @Override @@ -827,6 +841,31 @@ private void deserializeBuildInfo(InputStream inputStream) throws IOException { nodeVersionInfo.put(nodeId, new TNodeVersionInfo(version, buildInfo)); size--; } + if (inputStream.available() >= Integer.BYTES) { + inputStream.mark(Integer.BYTES); + int marker = ReadWriteIOUtils.readInt(inputStream); + if (marker == 0x43515631) { + int capabilitySize = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < capabilitySize; i++) { + int nodeId = ReadWriteIOUtils.readInt(inputStream); + int sizeOfCapabilities = ReadWriteIOUtils.readInt(inputStream); + TNodeVersionInfo info = nodeVersionInfo.get(nodeId); + if (info != null) { + Set capabilities = new java.util.HashSet<>(); + for (int j = 0; j < sizeOfCapabilities; j++) { + capabilities.add(ReadWriteIOUtils.readShort(inputStream)); + } + info.setSupportedCQDurationEncodingVersions(capabilities); + } else { + for (int j = 0; j < sizeOfCapabilities; j++) { + ReadWriteIOUtils.readShort(inputStream); + } + } + } + } else { + inputStream.reset(); + } + } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java index 7066815ef141e..a22b9c9148610 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java @@ -28,6 +28,7 @@ import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan; import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; import org.apache.iotdb.confignode.i18n.ProcedureMessages; +import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; import org.apache.iotdb.confignode.persistence.cq.CQInfo; @@ -41,6 +42,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,8 +84,39 @@ public CreateCQProcedure(TCreateCQReq req, ScheduledExecutorService executor) { this.req = req; this.cqToken = generateCQToken(); this.executor = executor; - this.firstExecutionTime = - CQScheduleTask.getFirstExecutionTime(req.boundaryTime, req.everyInterval); + TimeDuration everyDuration = + req.isSetEveryDuration() + ? new TimeDuration( + Math.toIntExact(req.getEveryDuration().getMonthPart()), + req.getEveryDuration().getNonMonthDuration()) + : new TimeDuration(0, req.everyInterval); + if (everyDuration.monthDuration != 0) { + java.time.ZoneId zone = java.time.ZoneId.of(req.zoneId); + long boundary = + req.isSetBoundaryExplicit() && !req.isBoundaryExplicit() + ? CQCalendarUtils.localEpochBoundary(zone) + : req.boundaryTime; + long now = + System.currentTimeMillis() + * ("ns" + .equals( + org.apache.iotdb.commons.conf.CommonDescriptor.getInstance() + .getConfig() + .getTimestampPrecision()) + ? 1_000_000L + : "us" + .equals( + org.apache.iotdb.commons.conf.CommonDescriptor.getInstance() + .getConfig() + .getTimestampPrecision()) + ? 1_000L + : 1L); + long index = CQCalendarUtils.firstOccurrenceIndex(boundary, everyDuration, now, zone); + this.firstExecutionTime = CQCalendarUtils.occurrence(boundary, everyDuration, index, zone); + } else { + this.firstExecutionTime = + CQScheduleTask.getFirstExecutionTime(req.boundaryTime, everyDuration.nonMonthDuration); + } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java index a3a5ffb41de4e..ad6f7233bf1b4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java @@ -36,7 +36,9 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; /** add config node procedure */ public class AddConfigNodeProcedure extends AbstractNodeProcedure { @@ -156,6 +158,15 @@ public void serialize(DataOutputStream stream) throws IOException { ThriftConfigNodeSerDeUtils.serializeTConfigNodeLocation(tConfigNodeLocation, stream); ReadWriteIOUtils.write(versionInfo.getVersion(), stream); ReadWriteIOUtils.write(versionInfo.getBuildInfo(), stream); + // Optional tail preserves the joining node's capability across procedure recovery while old + // payloads remain readable because the tail is absent in their serialized form. + ReadWriteIOUtils.write(versionInfo.isSetSupportedCQDurationEncodingVersions(), stream); + if (versionInfo.isSetSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(versionInfo.getSupportedCQDurationEncodingVersions().size(), stream); + for (short version : versionInfo.getSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(version, stream); + } + } } @Override @@ -169,6 +180,14 @@ public void deserialize(ByteBuffer byteBuffer) { versionInfo = new TNodeVersionInfo( ReadWriteIOUtils.readString(byteBuffer), ReadWriteIOUtils.readString(byteBuffer)); + if (byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer)) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + Set capabilities = new HashSet<>(); + for (int i = 0; i < size; i++) { + capabilities.add(ReadWriteIOUtils.readShort(byteBuffer)); + } + versionInfo.setSupportedCQDurationEncodingVersions(capabilities); + } } else { versionInfo = new TNodeVersionInfo("Unknown", "Unknown"); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index cf49a3916bf2a..5bb628a82157e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -78,6 +78,7 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -97,6 +98,11 @@ public class ConfigNode extends ServerCommandLine implements ConfigNodeMBean { private static final int INIT_NON_SEED_CONFIG_NODE_ID = -1; + private static TNodeVersionInfo currentNodeVersionInfo() { + return new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + } + private static final String CONFIGURATION = "IoTDB configuration: {}"; private final String mbeanName = @@ -192,9 +198,7 @@ public void active() { TSStatus status = configManager .getNodeManager() - .updateConfigNodeIfNecessary( - configNodeId, - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + .updateConfigNodeIfNecessary(configNodeId, currentNodeVersionInfo()); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { break; } else { @@ -233,7 +237,7 @@ public void active() { .getNodeManager() .applyConfigNode( CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(SEED_CONFIG_NODE_ID), - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + currentNodeVersionInfo()); setUpMetricService(); // Notice: We always set up Seed-ConfigNode's RPC service lastly to ensure // that the external service is not provided until Seed-ConfigNode is fully initialized @@ -376,7 +380,7 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio configManager.getClusterParameters(), CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(INIT_NON_SEED_CONFIG_NODE_ID)); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo(currentNodeVersionInfo()); TEndPoint seedConfigNode = CONF.getSeedConfigNode(); if (seedConfigNode == null) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java new file mode 100644 index 0000000000000..a4d183850f3f8 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.confignode; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataOutputStream; +import java.nio.ByteBuffer; +import java.util.Collections; + +public class UpdateVersionInfoPlanTest { + + @Test + public void capabilitiesRoundTrip() throws Exception { + TNodeVersionInfo versionInfo = + new TNodeVersionInfo("2.0.0", "build") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + UpdateVersionInfoPlan plan = new UpdateVersionInfoPlan(versionInfo, 3); + UpdateVersionInfoPlan restored = + (UpdateVersionInfoPlan) ConfigPhysicalPlan.Factory.create(plan.serializeToByteBuffer()); + Assert.assertEquals(plan, restored); + Assert.assertTrue( + restored.getVersionInfo().getSupportedCQDurationEncodingVersions().contains((short) 1)); + } + + @Test + public void legacyPayloadWithoutCapabilityTailStillDeserializes() throws Exception { + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write( + org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType.UpdateVersionInfo + .getPlanType(), + outputStream); + ReadWriteIOUtils.write(7, outputStream); + ReadWriteIOUtils.write("1.3.0", outputStream); + ReadWriteIOUtils.write("legacy", outputStream); + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + UpdateVersionInfoPlan restored = + (UpdateVersionInfoPlan) ConfigPhysicalPlan.Factory.create(buffer); + Assert.assertEquals(7, restored.getNodeId()); + Assert.assertEquals("1.3.0", restored.getVersionInfo().getVersion()); + Assert.assertFalse(restored.getVersionInfo().isSetSupportedCQDurationEncodingVersions()); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java index a0bc5a523ba70..b068d35ba4bf0 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java @@ -24,6 +24,9 @@ import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.rpc.TSStatusCode; @@ -34,6 +37,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class CQManagerTest { @@ -82,6 +86,106 @@ public void newTokenShouldCancelPreviousLocallyScheduledTask() { } } + @Test + public void mixedCalendarAndFixedDurationsReachCapabilityValidation() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + NodeManager nodeManager = Mockito.mock(NodeManager.class); + Mockito.when(configManager.getNodeManager()).thenReturn(nodeManager); + Mockito.when(nodeManager.getNodeVersionInfo()).thenReturn(java.util.Collections.emptyMap()); + CQManager cqManager = new CQManager(configManager); + + TCreateCQReq req = + new TCreateCQReq( + "mixedDurationCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq mixedDurationCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 86_400_000)); + req.setStartOffsetDuration(new TCQDuration(1, 86_400_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + try { + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void markerlessCreateCQIsRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = + new TCreateCQReq( + "legacyCq", + 1_000, + 0, + 1_000, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq legacyCq", + "UTC", + "root"); + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void invalidStructuredDurationIsRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = + new TCreateCQReq( + "invalidDurationCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq invalidDurationCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 0)); + req.setStartOffsetDuration(new TCQDuration(0, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .EXCEPTION_CQ_EVERY_DURATION_MUST_BE_POSITIVE_69C29D26, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + @SuppressWarnings("unchecked") private CQScheduleTask newScheduledTask( ConfigManager configManager, ScheduledFuture scheduledFuture, String cqToken) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java index 44f8536ee0fdc..2df17ed9fd980 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java @@ -18,10 +18,19 @@ */ package org.apache.iotdb.confignode.cq; +import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; +import org.apache.tsfile.utils.TimeDuration; import org.junit.Test; +import java.lang.reflect.Field; +import java.time.ZoneId; +import java.time.ZonedDateTime; + import static org.junit.Assert.assertEquals; public class CQScheduleTaskTest { @@ -41,4 +50,119 @@ public void testGetFirstExecutionTime2() { long everyInterval = 30L; assertEquals(110L, CQScheduleTask.getFirstExecutionTime(boundaryTime, everyInterval, now)); } + + @Test + public void testFixedDurationCqDoesNotRequireCanonicalZoneId() { + new CQScheduleTask( + "testCq", + 1000, + 0, + 1000, + TimeoutPolicy.BLOCKED, + "select s1 into root.backup.d1.s1 from root.sg.d1", + "token", + "Asia", + "root", + null, + null, + 1000); + } + + @Test + public void testFixedDurationVersionedCqUsesLegacySchedulerPath() { + TCreateCQReq req = + new TCreateCQReq( + "fixedVersionedCq", + 1000, + 0, + 1000, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq fixedVersionedCq", + "Asia", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + // A versioned fixed-duration CQ must not enter the calendar path (which requires a ZoneId). + new CQScheduleTask(req, 1000, "token", null, null); + } + + @Test + public void testCalendarConstructorKeepsProcedureSelectedFirstOccurrence() throws Exception { + TCreateCQReq req = + new TCreateCQReq( + "calendarCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq calendarCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(false); + long firstOccurrence = + ZonedDateTime.of(2030, 2, 1, 0, 0, 0, 0, ZoneId.of("UTC")).toInstant().toEpochMilli(); + + CQScheduleTask task = new CQScheduleTask(req, firstOccurrence, "token", null, null); + Field executionTime = CQScheduleTask.class.getDeclaredField("executionTime"); + executionTime.setAccessible(true); + assertEquals(firstOccurrence, executionTime.getLong(task)); + } + + @Test(expected = IllegalArgumentException.class) + public void testCalendarConstructorRejectsMismatchedFirstOccurrence() { + TCreateCQReq req = + new TCreateCQReq( + "calendarCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq calendarCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(false); + new CQScheduleTask(req, 1L, "token", null, null); + } + + @Test + public void testCalendarOccurrencesRecomputeFromOriginalBoundary() { + ZoneId zone = ZoneId.of("UTC"); + long boundary = ZonedDateTime.of(2024, 1, 31, 0, 0, 0, 0, zone).toInstant().toEpochMilli(); + TimeDuration month = new TimeDuration(1, 0); + assertEquals( + ZonedDateTime.of(2024, 2, 29, 0, 0, 0, 0, zone).toInstant().toEpochMilli(), + CQCalendarUtils.occurrence(boundary, month, 1, zone)); + assertEquals( + ZonedDateTime.of(2024, 3, 31, 0, 0, 0, 0, zone).toInstant().toEpochMilli(), + CQCalendarUtils.occurrence(boundary, month, 2, zone)); + } + + @Test + public void testDiscardLowerBoundNeverMovesBeforeCurrentOccurrence() { + ZoneId zone = ZoneId.of("UTC"); + long boundary = ZonedDateTime.of(2024, 1, 1, 0, 0, 0, 0, zone).toInstant().toEpochMilli(); + TimeDuration month = new TimeDuration(1, 0); + long current = CQCalendarUtils.occurrence(boundary, month, 2, zone); + long lowerBound = CQCalendarUtils.firstOccurrenceIndex(boundary, month, current, zone); + assertEquals(2, lowerBound); + assertEquals(3, Math.max(2 + 1, lowerBound)); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java new file mode 100644 index 0000000000000..cb3402b434e46 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.confignode.manager.cq; + +import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; + +import org.apache.tsfile.utils.TimeDuration; +import org.junit.Test; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; + +public class CQCalendarUtilsTest { + + private static final ZoneId UTC = ZoneId.of("UTC"); + + @Test + public void testMonthEndSequenceIsAnchoredToOriginalBoundary() { + long boundary = epochTimestamp(2024, 1, 31, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + + assertEquals( + epochTimestamp(2024, 2, 29, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, every, 1, UTC)); + assertEquals( + epochTimestamp(2024, 3, 31, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, every, 2, UTC)); + assertEquals( + epochTimestamp(2024, 4, 30, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, every, 3, UTC)); + } + + @Test + public void testLeapDayYearSequenceReachesNextLeapYear() { + long boundary = epochTimestamp(2020, 2, 29, 0, 0, UTC); + TimeDuration everyYear = new TimeDuration(12, 0); + + assertEquals( + epochTimestamp(2021, 2, 28, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, everyYear, 1, UTC)); + assertEquals( + epochTimestamp(2024, 2, 29, 0, 0, UTC), + CQCalendarUtils.occurrence(boundary, everyYear, 4, UTC)); + } + + @Test + public void testOmittedBoundaryUsesLocalEpochInPersistedZone() { + ZoneId shanghai = ZoneId.of("Asia/Shanghai"); + assertEquals( + epochTimestamp(1970, 1, 1, 0, 0, shanghai), CQCalendarUtils.localEpochBoundary(shanghai)); + } + + @Test + public void testRangeEndpointsAreDerivedFromBoundaryVector() { + long boundary = epochTimestamp(2024, 1, 31, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long start = CQCalendarUtils.applyVector(boundary, 1, 0, UTC); + long end = CQCalendarUtils.applyVector(boundary, 2, 0, UTC); + + assertEquals(epochTimestamp(2024, 2, 29, 0, 0, UTC), start); + assertEquals(epochTimestamp(2024, 3, 31, 0, 0, UTC), end); + assertEquals(end, CQCalendarUtils.occurrence(boundary, every, 2, UTC)); + } + + @Test + public void testFixedCadenceRangeUsesCurrentCalendarMonth() { + long day = TimestampPrecisionUtils.currPrecision.convert(1, TimeUnit.DAYS); + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + long execution = epochTimestamp(2024, 4, 1, 0, 0, UTC); + TimeDuration start = new TimeDuration(1, day); + TimeDuration end = new TimeDuration(1, 0); + CQScheduleTask task = + calendarTask(boundary, execution, new TimeDuration(0, day), start, end, UTC); + + // EVERY 1d RANGE 1mo1d, 1mo passes the component-wise duration validation. Its two + // offsets must use March's calendar boundary, rather than January's 31-day length. + assertEquals( + epochTimestamp(2024, 2, 29, 0, 0, UTC), task.calculateCalendarRangeEndpoint(start, 91)); + assertEquals( + epochTimestamp(2024, 3, 1, 0, 0, UTC), task.calculateCalendarRangeEndpoint(end, 91)); + + // The calculation must remain anchored to the original boundary even when the fixed cadence + // has not reached a month boundary. Subtracting RANGE from the occurrence would yield Jan 29. + long marchFirst = epochTimestamp(2024, 3, 1, 0, 0, UTC); + task = calendarTask(boundary, marchFirst, new TimeDuration(0, day), start, end, UTC); + assertEquals( + epochTimestamp(2024, 1, 29, 0, 0, UTC), task.calculateCalendarRangeEndpoint(start, 60)); + } + + @Test + public void testFixedCadenceRangePreservesMonthClampingAndDst() { + long day = TimestampPrecisionUtils.currPrecision.convert(1, TimeUnit.DAYS); + TimeDuration every = new TimeDuration(0, day); + TimeDuration start = new TimeDuration(1, day); + TimeDuration end = new TimeDuration(0, 0); + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + long execution = epochTimestamp(2024, 3, 31, 0, 0, UTC); + CQScheduleTask task = calendarTask(boundary, execution, every, start, end, UTC); + assertEquals( + epochTimestamp(2024, 2, 28, 0, 0, UTC), task.calculateCalendarRangeEndpoint(start, 90)); + assertEquals(execution, task.calculateCalendarRangeEndpoint(end, 90)); + + ZoneId newYork = ZoneId.of("America/New_York"); + boundary = epochTimestamp(2024, 1, 1, 11, 0, newYork); + execution = epochTimestamp(2024, 4, 10, 12, 0, newYork); + task = calendarTask(boundary, execution, every, start, end, newYork); + // Subtracting one calendar month reaches March 10 at noon; subtracting another 24 hours + // crosses the spring DST transition and reaches March 9 at 11:00. + assertEquals( + epochTimestamp(2024, 3, 9, 11, 0, newYork), + task.calculateCalendarRangeEndpoint(start, 100)); + } + + @Test + public void testCalendarCadenceRangeRetainsOriginalMonthEndAnchor() { + long boundary = epochTimestamp(2024, 1, 31, 0, 0, UTC); + long execution = epochTimestamp(2024, 4, 30, 0, 0, UTC); + TimeDuration month = new TimeDuration(1, 0); + TimeDuration zero = new TimeDuration(0, 0); + CQScheduleTask task = calendarTask(boundary, execution, month, month, zero, UTC); + + assertEquals( + epochTimestamp(2024, 3, 31, 0, 0, UTC), task.calculateCalendarRangeEndpoint(month, 3)); + assertEquals(execution, task.calculateCalendarRangeEndpoint(zero, 3)); + } + + private static CQScheduleTask calendarTask( + long boundary, + long execution, + TimeDuration every, + TimeDuration start, + TimeDuration end, + ZoneId zone) { + TCreateCQReq req = + new TCreateCQReq( + "rangeCq", + 0, + boundary, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select s1 into root.backup.d1.s1 from root.sg.d1", + "create cq rangeCq", + zone.getId(), + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(every.monthDuration, every.nonMonthDuration)); + req.setStartOffsetDuration(new TCQDuration(start.monthDuration, start.nonMonthDuration)); + req.setEndOffsetDuration(new TCQDuration(end.monthDuration, end.nonMonthDuration)); + req.setBoundaryExplicit(true); + return new CQScheduleTask(req, execution, "token", null, null); + } + + @Test + public void testCalendarOccurrenceMatchesGroupByTimeHelper() { + ZoneId newYork = ZoneId.of("America/New_York"); + long boundary = epochTimestamp(2024, 1, 3, 1, 30, newYork); + TimeDuration tenMonths = new TimeDuration(10, 0); + long cqOccurrence = CQCalendarUtils.occurrence(boundary, tenMonths, 1, newYork); + long groupBy = + org.apache.iotdb.commons.queryengine.utils.DateTimeUtils.calcPositiveIntervalByMonth( + boundary, tenMonths, newYork); + assertEquals(groupBy, cqOccurrence); + // DST overlap: GROUP BY / atZone selects the earlier offset (-04:00 = 05:30Z). + assertEquals(epochTimestamp(2024, 11, 3, 1, 30, newYork), cqOccurrence); + } + + @Test + public void testDstUsesZoneRulesForCalendarAndElapsedParts() { + ZoneId newYork = ZoneId.of("America/New_York"); + long monthBoundary = epochTimestamp(2024, 2, 10, 2, 30, newYork); + long monthOccurrence = + CQCalendarUtils.occurrence(monthBoundary, new TimeDuration(1, 0), 1, newYork); + assertEquals(epochTimestamp(2024, 3, 10, 3, 30, newYork), monthOccurrence); + + long elapsedBoundary = epochTimestamp(2024, 3, 9, 12, 0, newYork); + long oneDay = TimestampPrecisionUtils.currPrecision.convert(1, TimeUnit.DAYS); + long elapsedOccurrence = + CQCalendarUtils.occurrence(elapsedBoundary, new TimeDuration(0, oneDay), 1, newYork); + assertEquals(epochTimestamp(2024, 3, 10, 13, 0, newYork), elapsedOccurrence); + } + + @Test + public void testBlockedKeepsTheNextOccurrenceWhileDiscardSkipsMissedOccurrences() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long callbackTime = epochTimestamp(2024, 1, 4, 12, 0, UTC); + long executionTime = CQCalendarUtils.occurrence(boundary, every, 1, UTC); + + assertEquals( + 2, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.BLOCKED, 1, callbackTime, executionTime, 0, 1, boundary, every, UTC)); + assertEquals( + 2, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.DISCARD, 1, callbackTime, executionTime, 0, 1, boundary, every, UTC)); + } + + @Test + public void testDiscardNeverMovesBeforeTheCurrentOccurrence() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long currentOccurrence = CQCalendarUtils.occurrence(boundary, every, 2, UTC); + + assertEquals( + 3, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.DISCARD, + 2, + currentOccurrence, + currentOccurrence, + 0, + 2, + boundary, + every, + UTC)); + } + + private static long epochTimestamp( + int year, int month, int day, int hour, int minute, ZoneId zone) { + Instant instant = ZonedDateTime.of(year, month, day, hour, minute, 0, 0, zone).toInstant(); + return Math.addExact( + TimestampPrecisionUtils.currPrecision.convert(instant.getEpochSecond(), TimeUnit.SECONDS), + TimestampPrecisionUtils.currPrecision.convert(instant.getNano(), TimeUnit.NANOSECONDS)); + } + + private static long epochTimestamp(int year, int month, int day, int hour, ZoneId zone) { + return epochTimestamp(year, month, day, hour, 0, zone); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java index f39ecd46d5c6a..60434905b4180 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java @@ -20,16 +20,25 @@ package org.apache.iotdb.confignode.manager.node; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; +import org.apache.iotdb.confignode.consensus.response.datanode.DataNodeRegisterResp; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; +import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.consensus.IConsensus; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -57,6 +66,8 @@ public class NodeManagerTest { private IConsensus consensus; private ConsensusManager consensusManager; + private NodeInfo nodeInfo; + private CQManager cqManager; private NodeManager nodeManager; private int originalCnConnectionTimeout; private int originalTransferLeaderTimeout; @@ -73,10 +84,12 @@ public void setUp() { consensus = Mockito.mock(IConsensus.class); consensusManager = Mockito.mock(ConsensusManager.class); IManager configManager = Mockito.mock(IManager.class); - NodeInfo nodeInfo = Mockito.mock(NodeInfo.class); + nodeInfo = Mockito.mock(NodeInfo.class); LoadManager loadManager = Mockito.mock(LoadManager.class); + cqManager = Mockito.mock(CQManager.class); Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); Mockito.when(configManager.getLoadManager()).thenReturn(loadManager); Mockito.when(consensusManager.getConsensusImpl()).thenReturn(consensus); Mockito.when(consensusManager.getConsensusGroupId()) @@ -126,6 +139,64 @@ public void transferLeaderShouldTryAnotherCandidateAfterTransientFailure() throw peerCaptor.getAllValues().stream().map(Peer::getNodeId).collect(Collectors.toList())); } + @Test + public void configNodeWithoutCalendarDurationCapabilityIsRejectedWhileCalendarCQExists() { + TNodeVersionInfo unsupportedVersion = new TNodeVersionInfo("old", "old"); + TConfigNodeRegisterReq registerReq = + new TConfigNodeRegisterReq().setConfigNodeLocation(firstCandidate); + registerReq.setVersionInfo(unsupportedVersion); + + // The CQ manager is mocked in setUp; make the persisted metadata barrier active for this test. + Mockito.when(cqManager.hasCalendarDurationCQ()).thenReturn(true); + + TConfigNodeRegisterResp registerResp = nodeManager.registerConfigNode(registerReq); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), registerResp.getStatus().getCode()); + Assert.assertEquals(-1, registerResp.getConfigNodeId()); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + nodeManager + .updateConfigNodeIfNecessary(firstCandidate.getConfigNodeId(), unsupportedVersion) + .getCode()); + Mockito.verify(nodeInfo, Mockito.never()).generateNextNodeId(); + } + + @Test + public void dataNodeWithoutCalendarDurationCapabilityIsRejectedWhileCalendarCQExists() { + TNodeVersionInfo unsupportedVersion = new TNodeVersionInfo("old", "old"); + TDataNodeLocation location = + new TDataNodeLocation( + 11, + new TEndPoint("127.0.0.1", 6667), + new TEndPoint("127.0.0.1", 10730), + new TEndPoint("127.0.0.1", 10740), + new TEndPoint("127.0.0.1", 10750), + new TEndPoint("127.0.0.1", 10760)); + TDataNodeConfiguration configuration = new TDataNodeConfiguration().setLocation(location); + TDataNodeRegisterReq registerReq = + new TDataNodeRegisterReq() + .setClusterName("cluster") + .setDataNodeConfiguration(configuration) + .setVersionInfo(unsupportedVersion); + + Mockito.when(cqManager.hasCalendarDurationCQ()).thenReturn(true); + + DataNodeRegisterResp registerResp = + (DataNodeRegisterResp) nodeManager.registerDataNode(registerReq); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), registerResp.getStatus().getCode()); + Mockito.verify(nodeInfo, Mockito.never()).generateNextNodeId(); + + TDataNodeRestartReq restartReq = + new TDataNodeRestartReq() + .setClusterName("cluster") + .setDataNodeConfiguration(configuration) + .setVersionInfo(unsupportedVersion); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + nodeManager.updateDataNodeIfNecessary(restartReq).getStatus().getCode()); + } + @Test public void transferLeaderShouldRedirectToActualLeaderAfterFailedResponse() throws Exception { Mockito.when(consensusManager.getLeaderLocation()) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java index 64bbd69c5b6a7..56547ac1e391b 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.confignode.consensus.request.write.cq.UpdateCQLastExecTimePlan; import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; import org.apache.iotdb.confignode.persistence.cq.CQInfo; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.rpc.TSStatusCode; @@ -36,6 +37,8 @@ import java.io.File; import java.io.IOException; +import java.time.ZoneId; +import java.time.ZonedDateTime; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -159,4 +162,180 @@ public void testShowCQCanFilterByCQId() throws Exception { Assert.assertEquals(1, showCQResp.getCqList().size()); Assert.assertEquals("testCq4", showCQResp.getCqList().get(0).getCqId()); } + + @Test + public void testOccurrenceIndexCasFencesOutOfOrderCallbacks() { + TCreateCQReq req = + new TCreateCQReq( + "indexedCq", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select 1", + "create cq indexedCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "indexedToken", 1000)).getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("indexedCq", 1000, "indexedToken", 1, 2)) + .getCode()); + Assert.assertEquals( + TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("indexedCq", 2000, "indexedToken", 1, 2)) + .getCode()); + } + + @Test + public void testCommittedProgressRetryIsDetectedAsStaleForReconciliation() throws TException { + TCreateCQReq req = + new TCreateCQReq( + "lostAckCq", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select 1", + "create cq lostAckCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "lostAckToken", 1000)).getCode()); + UpdateCQLastExecTimePlan advance = + new UpdateCQLastExecTimePlan("lostAckCq", 1000, "lostAckToken", 1, 2); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.updateCQLastExecutionTime(advance).getCode()); + // A retry after the first response was lost must be recognized as stale and reconciled. + Assert.assertEquals( + TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(), + cqInfo.updateCQLastExecutionTime(advance).getCode()); + } + + @Test + public void testMixedCalendarDurationUsesStructuredFixedEveryForLastExecution() { + TCreateCQReq req = + new TCreateCQReq( + "mixedFixedEveryCq", + 0, + 0, + 0, + 0, + (byte) 0, + "select 1", + "create cq mixedFixedEveryCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1_000)); + req.setStartOffsetDuration(new TCQDuration(1, 1_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "mixedFixedEveryToken", 10_000)).getCode()); + + CQInfo.CQEntry entry = cqInfo.showCQ(new ShowCQPlan("mixedFixedEveryCq")).getCqList().get(0); + Assert.assertEquals(9_000, entry.getLastExecutionTime()); + Assert.assertEquals(10, entry.getNextOccurrenceIndex()); + } + + @Test + public void testFixedVersionedDurationKeepsLegacyZoneOpaque() { + TCreateCQReq req = + new TCreateCQReq( + "fixedVersionedCq", + 1_000, + 0, + 1_000, + 0, + (byte) 0, + "select 1", + "create cq fixedVersionedCq", + "Asia", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1_000)); + req.setStartOffsetDuration(new TCQDuration(0, 1_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(false); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "fixedVersionedToken", 10_000)).getCode()); + } + + @Test + public void testCalendarOccurrenceProgressSurvivesSnapshotRecovery() throws Exception { + File calendarSnapshotDir = new File(BASE_OUTPUT_PATH, "snapshot-calendar"); + if (calendarSnapshotDir.exists()) { + FileUtils.deleteDirectory(calendarSnapshotDir); + } + calendarSnapshotDir.mkdirs(); + + ZoneId zone = ZoneId.of("UTC"); + long boundary = ZonedDateTime.of(2024, 1, 31, 0, 0, 0, 0, zone).toInstant().toEpochMilli(); + long firstOccurrence = + ZonedDateTime.of(2024, 2, 29, 0, 0, 0, 0, zone).toInstant().toEpochMilli(); + TCreateCQReq req = + new TCreateCQReq( + "snapshotCalendarCq", + 0, + boundary, + 0, + 0, + (byte) 0, + "select 1", + "create cq snapshotCalendarCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "snapshotCalendarToken", firstOccurrence)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan( + "snapshotCalendarCq", firstOccurrence, "snapshotCalendarToken", 1, 2)) + .getCode()); + + cqInfo.processTakeSnapshot(calendarSnapshotDir); + CQInfo restored = new CQInfo(); + restored.processLoadSnapshot(calendarSnapshotDir); + + ShowCQResp response = restored.showCQ(new ShowCQPlan("snapshotCalendarCq")); + Assert.assertEquals(1, response.getCqList().size()); + Assert.assertEquals(2, response.getCqList().get(0).getNextOccurrenceIndex()); + Assert.assertEquals(firstOccurrence, response.getCqList().get(0).getLastExecutionTime()); + FileUtils.deleteDirectory(calendarSnapshotDir); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java index e3ced5b069428..3b2a74634b5bd 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java @@ -26,8 +26,10 @@ import org.apache.iotdb.common.rpc.thrift.TNodeResource; import org.apache.iotdb.commons.exception.StartupException; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; @@ -38,6 +40,7 @@ import java.io.File; import java.io.IOException; +import java.util.Collections; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -73,6 +76,35 @@ public void testSnapshot() throws TException, IOException { Assert.assertEquals(nodeInfo, nodeInfo1); } + @Test + public void testSnapshotPreservesDurationEncodingCapabilities() throws TException, IOException { + File capabilitySnapshotDir = new File(BASE_OUTPUT_PATH, "snapshot-node-capabilities"); + if (capabilitySnapshotDir.exists()) { + FileUtils.deleteDirectory(capabilitySnapshotDir); + } + capabilitySnapshotDir.mkdirs(); + try { + registerConfigNodes(); + registerDataNodes(); + TNodeVersionInfo versionInfo = + new TNodeVersionInfo("2.0.0", "build") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + nodeInfo.updateVersionInfo(new UpdateVersionInfoPlan(versionInfo, 10000)); + Assert.assertTrue(nodeInfo.processTakeSnapshot(capabilitySnapshotDir)); + + NodeInfo restored = new NodeInfo(); + restored.processLoadSnapshot(capabilitySnapshotDir); + Assert.assertEquals(nodeInfo, restored); + Assert.assertTrue( + restored + .getVersionInfo(10000) + .getSupportedCQDurationEncodingVersions() + .contains((short) 1)); + } finally { + FileUtils.deleteDirectory(capabilitySnapshotDir); + } + } + private void registerConfigNodes() { for (int i = 0; i < 3; i++) { ApplyConfigNodePlan applyConfigNodePlan = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java index 3e7fd2052ad53..024eaf25ceb6c 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.service.ConfigNode; @@ -66,6 +67,34 @@ public void tokenShouldBeUniqueForSameCQId() { } } + @Test + public void mixedCalendarRangeWithFixedEveryUsesStructuredEveryDuration() { + TCreateCQReq req = + new TCreateCQReq( + "mixedFixedEveryCq", + 0, + 0, + 0, + 0, + (byte) 0, + "select s1 into root.backup.d1(s1) from root.sg.d1", + "create cq mixedFixedEveryCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1_000)); + req.setStartOffsetDuration(new TCQDuration(1, 1_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + try { + new CreateCQProcedure(req, executor); + } finally { + executor.shutdown(); + } + } + @Test public void serializeDeserializeTest() { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java index 682b478134664..835e21dca9c00 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java @@ -31,6 +31,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; public class AddConfigNodeProcedureTest { @@ -40,7 +41,8 @@ public void serDeTest() throws IOException { new AddConfigNodeProcedure( new TConfigNodeLocation( 0, new TEndPoint("127.0.0.1", 10710), new TEndPoint("0.0.0.0", 10720)), - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { @@ -50,4 +52,21 @@ public void serDeTest() throws IOException { Assert.assertEquals(procedure0, ProcedureFactory.getInstance().create(buffer)); } } + + @Test + public void deserializeLegacyPayloadWithoutCapabilities() throws IOException { + AddConfigNodeProcedure procedure0 = + new AddConfigNodeProcedure( + new TConfigNodeLocation( + 0, new TEndPoint("127.0.0.1", 10710), new TEndPoint("0.0.0.0", 10720)), + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + procedure0.serialize(outputStream); + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size() - 1); + Assert.assertEquals(procedure0, ProcedureFactory.getInstance().create(buffer)); + } + } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 2c25ba7460643..f3ea7245a5ee8 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -1229,6 +1229,16 @@ public final class DataNodeQueryMessages { "CQ: The start time offset should be greater than 0."; public static final String CQ_THE_END_TIME_OFFSET_SHOULD_BE_GREATER = "CQ: The end time offset should be greater than or equal to 0."; + public static final String EXCEPTION_CQ_DURATION_CANNOT_BE_EMPTY_C7269AB2 = + "CQ duration cannot be empty"; + public static final String EXCEPTION_INVALID_CQ_DURATION_ARG_F4917D5C = + "Invalid CQ duration: %s"; + public static final String EXCEPTION_CQ_DURATION_COMPONENT_OVERFLOWS_ARG_ED5B0962 = + "CQ duration component overflows: %s"; + public static final String EXCEPTION_CQ_DURATION_MONTH_COMPONENT_OVERFLOWS_ARG_EBF2A2B6 = + "CQ duration month component overflows: %s"; + public static final String EXCEPTION_CQ_EVERY_INTERVAL_MUST_BE_POSITIVE_26259019 = + "CQ: The every interval should be greater than 0."; public static final String CQ_THE_QUERY_BODY_MISSES_AN_INTO_CLAUSE = "CQ: The query body misses an INTO clause."; public static final String CQ_SPECIFYING_TIME_FILTERS_IN_THE_QUERY_BODY = @@ -1934,6 +1944,8 @@ public final class DataNodeQueryMessages { "Sorting by device is only supported in ALIGN BY DEVICE queries."; public static final String CQ_EVERY_INTERVAL_D_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL = "CQ: Every interval [%d] should not be lower than the `continuous_query_minimum_every_interval` [%d] configured."; + public static final String CQ_EVERY_INTERVAL_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL = + "CQ: Every interval [%s] should not be lower than the `continuous_query_minimum_every_interval` [%d] configured."; public static final String CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_END_TIME_OFFSET = "CQ: The start time offset should be greater than end time offset."; public static final String CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_INTERVAL = @@ -3877,4 +3889,6 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "Only InMemoryDeviceEntryDataSet supports get inline device entries"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3 = + "CQ calendar duration cannot be created until all cluster nodes support duration encoding version 1"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index c6a55ebad3e06..efea296f6d57b 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -1211,6 +1211,16 @@ public final class DataNodeQueryMessages { "连续查询：起始时间偏移量应大于 0。"; public static final String CQ_THE_END_TIME_OFFSET_SHOULD_BE_GREATER = "连续查询：结束时间偏移量应大于或等于 0。"; + public static final String EXCEPTION_CQ_DURATION_CANNOT_BE_EMPTY_C7269AB2 = + "CQ duration 不能为空"; + public static final String EXCEPTION_INVALID_CQ_DURATION_ARG_F4917D5C = + "无效的 CQ duration：%s"; + public static final String EXCEPTION_CQ_DURATION_COMPONENT_OVERFLOWS_ARG_ED5B0962 = + "CQ duration 组件溢出：%s"; + public static final String EXCEPTION_CQ_DURATION_MONTH_COMPONENT_OVERFLOWS_ARG_EBF2A2B6 = + "CQ duration 月份组件溢出：%s"; + public static final String EXCEPTION_CQ_EVERY_INTERVAL_MUST_BE_POSITIVE_26259019 = + "CQ：every interval 应大于 0。"; public static final String CQ_THE_QUERY_BODY_MISSES_AN_INTO_CLAUSE = "连续查询：查询体缺少 INTO 子句。"; public static final String CQ_SPECIFYING_TIME_FILTERS_IN_THE_QUERY_BODY = @@ -2065,6 +2075,9 @@ public final class DataNodeQueryMessages { public static final String CQ_EVERY_INTERVAL_D_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL = "CQ：every interval [%d] 不应小于配置项 `continuous_query_minimum_every_interval` [%d]。"; + public static final String CQ_EVERY_INTERVAL_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL = + + "CQ：every interval [%s] 不应小于配置项 `continuous_query_minimum_every_interval` [%d]。"; public static final String CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_END_TIME_OFFSET = "CQ：开始时间偏移量应大于结束时间偏移量。"; @@ -4635,4 +4648,6 @@ private DataNodeQueryMessages() {} public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "只有 InMemoryDeviceEntryDataSet 支持获取内存中的设备条目"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3 = + "集群所有节点支持 duration encoding version 1 后才能创建 CQ 日历 duration"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java index d8692a3573b3a..1244879ed425c 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java @@ -63,7 +63,7 @@ private StorageEngineMessages() {} public static final String FILE_NO_SUCH_TIME_SERIES = "文件中不包含该时间序列 {}。"; // ======================== Resource Control - Disk ======================== - + public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY = "所有目录已满，切换系统为只读模式。"; public static final String FAILED_TO_PROCESS_FOLDER = "处理目录失败 '"; public static final String FAIL_TO_GET_CANONICAL_PATH = "获取数据目录 {} 的规范路径失败"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 3ddd2531200dc..f7d921b29df7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -112,6 +112,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTimeSeriesReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterTopicReq; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCountDatabaseResp; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; @@ -156,6 +157,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetUDFTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetUdfTableReq; import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TPipeConfigTransferReq; import org.apache.iotdb.confignode.rpc.thrift.TPipeConfigTransferResp; import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq; @@ -4157,24 +4159,77 @@ public SettableFuture createContinuousQuery( createContinuousQueryStatement.semanticCheck(); final String queryBody = createContinuousQueryStatement.getQueryBody(); + // CQ body group-by bounds are populated only when the CQ executes. Month-aware filters + // cannot initialize an empty [0, 0) range during this validation pass, so provide a minimal + // non-empty placeholder; the execution RPC replaces these bounds with the real window. + if (createContinuousQueryStatement.getQueryBodyStatement().getGroupByTimeComponent() != null + && createContinuousQueryStatement + .getQueryBodyStatement() + .getGroupByTimeComponent() + .getInterval() + .containsMonth()) { + createContinuousQueryStatement + .getQueryBodyStatement() + .getGroupByTimeComponent() + .setStartTime(0); + createContinuousQueryStatement + .getQueryBodyStatement() + .getGroupByTimeComponent() + .setEndTime(1); + } // TODO Do not modify Statement in Analyzer Analyzer.analyze(createContinuousQueryStatement.getQueryBodyStatement(), context); final SettableFuture future = SettableFuture.create(); try (final ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + final boolean hasCalendarDuration = + createContinuousQueryStatement.getEveryDuration().monthDuration != 0 + || createContinuousQueryStatement.getStartTimeOffsetDuration().monthDuration != 0 + || createContinuousQueryStatement.getEndTimeOffsetDuration().monthDuration != 0; + if (hasCalendarDuration && !allClusterNodesSupportCQDurationEncoding(client.showCluster())) { + future.setException( + new SemanticException( + DataNodeQueryMessages + .MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3)); + return future; + } + // The legacy fields are still required on the wire. If any structured component contains a + // calendar month, none of the legacy fields has a valid representation and all three use the + // zero sentinel. This prevents an old reader from interpreting a mixed request as a partially + // valid fixed-duration CQ. + final long legacyEvery = + hasCalendarDuration ? 0 : createContinuousQueryStatement.getEveryInterval(); + final long legacyStart = + hasCalendarDuration ? 0 : createContinuousQueryStatement.getStartTimeOffset(); + final long legacyEnd = + hasCalendarDuration ? 0 : createContinuousQueryStatement.getEndTimeOffset(); final TCreateCQReq tCreateCQReq = new TCreateCQReq( createContinuousQueryStatement.getCqId(), - createContinuousQueryStatement.getEveryInterval(), + legacyEvery, createContinuousQueryStatement.getBoundaryTime(), - createContinuousQueryStatement.getStartTimeOffset(), - createContinuousQueryStatement.getEndTimeOffset(), + legacyStart, + legacyEnd, createContinuousQueryStatement.getTimeoutPolicy().getType(), queryBody, context.getSql(), context.getZoneId().getId(), context.getSession() == null ? null : context.getSession().getUserName()); + tCreateCQReq.setDurationEncodingVersion((short) 1); + tCreateCQReq.setEveryDuration( + new TCQDuration( + createContinuousQueryStatement.getEveryDuration().monthDuration, + createContinuousQueryStatement.getEveryDuration().nonMonthDuration)); + tCreateCQReq.setStartOffsetDuration( + new TCQDuration( + createContinuousQueryStatement.getStartTimeOffsetDuration().monthDuration, + createContinuousQueryStatement.getStartTimeOffsetDuration().nonMonthDuration)); + tCreateCQReq.setEndOffsetDuration( + new TCQDuration( + createContinuousQueryStatement.getEndTimeOffsetDuration().monthDuration, + createContinuousQueryStatement.getEndTimeOffsetDuration().nonMonthDuration)); + tCreateCQReq.setBoundaryExplicit(createContinuousQueryStatement.isBoundaryExplicit()); final TSStatus executionStatus = client.createCQ(tCreateCQReq); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != executionStatus.getCode()) { future.setException(new IoTDBException(executionStatus)); @@ -4187,6 +4242,35 @@ public SettableFuture createContinuousQuery( return future; } + private boolean allClusterNodesSupportCQDurationEncoding(TShowClusterResp response) { + if (response == null + || response.getConfigNodeList() == null + || response.getDataNodeList() == null + || response.getNodeVersionInfo() == null) { + return false; + } + if (response.getConfigNodeList().isEmpty() && response.getDataNodeList().isEmpty()) { + return false; + } + for (TConfigNodeLocation node : response.getConfigNodeList()) { + if (!supportsCQDurationEncoding(response.getNodeVersionInfo().get(node.getConfigNodeId()))) { + return false; + } + } + for (TDataNodeLocation node : response.getDataNodeList()) { + if (!supportsCQDurationEncoding(response.getNodeVersionInfo().get(node.getDataNodeId()))) { + return false; + } + } + return true; + } + + private boolean supportsCQDurationEncoding(TNodeVersionInfo info) { + return info != null + && info.isSetSupportedCQDurationEncodingVersions() + && info.getSupportedCQDurationEncodingVersions().contains((short) 1); + } + @Override public SettableFuture dropContinuousQuery(final String cqId) { final SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 1a9b8297ec44c..455dfbe687f68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -1240,10 +1240,10 @@ public Statement visitCreateContinuousQuery(IoTDBSqlParser.CreateContinuousQuery .CQ_AT_LEAST_ONE_OF_THE_PARAMETERS_EVERY_INTERVAL_AND_GROUP_BY_INTERVAL_NEEDS_TO_BE); } - long interval = - queryStatement.getGroupByTimeComponent().getInterval().getTotalDuration(currPrecision); - statement.setEveryInterval(interval); - statement.setStartTimeOffset(interval); + org.apache.tsfile.utils.TimeDuration interval = + queryStatement.getGroupByTimeComponent().getInterval(); + statement.setEveryDuration(interval); + statement.setStartTimeOffsetDuration(interval); } if (ctx.timeoutPolicyClause() != null) { @@ -1256,8 +1256,8 @@ public Statement visitCreateContinuousQuery(IoTDBSqlParser.CreateContinuousQuery private void parseResampleClause( IoTDBSqlParser.ResampleClauseContext ctx, CreateContinuousQueryStatement statement) { if (ctx.EVERY() != null) { - statement.setEveryInterval( - DataNodeDateTimeUtils.convertDurationStrToLong(ctx.everyInterval.getText())); + statement.setEveryDuration( + DataNodeDateTimeUtils.constructTimeDurationForCQ(ctx.everyInterval.getText())); } else { QueryStatement queryStatement = statement.getQueryBodyStatement(); if (!queryStatement.isGroupByTime()) { @@ -1265,24 +1265,24 @@ private void parseResampleClause( DataNodeQueryMessages .CQ_AT_LEAST_ONE_OF_THE_PARAMETERS_EVERY_INTERVAL_AND_GROUP_BY_INTERVAL_NEEDS_TO_BE); } - statement.setEveryInterval( - queryStatement.getGroupByTimeComponent().getInterval().getTotalDuration(currPrecision)); + statement.setEveryDuration(queryStatement.getGroupByTimeComponent().getInterval()); } if (ctx.BOUNDARY() != null) { + statement.setBoundaryExplicit(true); statement.setBoundaryTime( parseTimeValue(ctx.boundaryTime, CommonDateTimeUtils.currentTime())); } if (ctx.RANGE() != null) { - statement.setStartTimeOffset( - DataNodeDateTimeUtils.convertDurationStrToLong(ctx.startTimeOffset.getText())); + statement.setStartTimeOffsetDuration( + DataNodeDateTimeUtils.constructTimeDurationForCQ(ctx.startTimeOffset.getText())); if (ctx.endTimeOffset != null) { - statement.setEndTimeOffset( - DataNodeDateTimeUtils.convertDurationStrToLong(ctx.endTimeOffset.getText())); + statement.setEndTimeOffsetDuration( + DataNodeDateTimeUtils.constructTimeDurationForCQ(ctx.endTimeOffset.getText())); } } else { - statement.setStartTimeOffset(statement.getEveryInterval()); + statement.setStartTimeOffsetDuration(statement.getEveryDuration()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java index ef7506c8d20b2..b10fc9ac89ab5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.cq.TimeoutPolicy; import org.apache.iotdb.commons.exception.SemanticException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.queryengine.utils.TimestampPrecisionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils; @@ -33,6 +34,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.component.GroupByTimeComponent; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.tsfile.utils.TimeDuration; + import java.util.Collections; import java.util.List; @@ -52,6 +55,13 @@ public class CreateContinuousQueryStatement extends Statement implements IConfig // The end time of each query execution, default value is 0. private long endTimeOffset = 0; + // Structured representations retained for calendar-aware CQ scheduling. The legacy long + // fields above are kept as wire/backward-compatible projections for fixed-only durations. + private TimeDuration everyDuration = new TimeDuration(0, 0); + private TimeDuration startTimeOffsetDuration = new TimeDuration(0, 0); + private TimeDuration endTimeOffsetDuration = new TimeDuration(0, 0); + private boolean boundaryExplicit; + // Specify how we deal with the cq task whose previous time interval execution is not finished // while the next execution time has reached, default value is BLOCKED. private TimeoutPolicy timeoutPolicy = TimeoutPolicy.BLOCKED; @@ -78,6 +88,19 @@ public long getEveryInterval() { public void setEveryInterval(long everyInterval) { this.everyInterval = everyInterval; + this.everyDuration = new TimeDuration(0, everyInterval); + } + + public TimeDuration getEveryDuration() { + return everyDuration; + } + + public void setEveryDuration(TimeDuration everyDuration) { + this.everyDuration = everyDuration; + this.everyInterval = + everyDuration.monthDuration == 0 + ? everyDuration.getTotalDuration(TimestampPrecisionUtils.currPrecision) + : 0; } public long getBoundaryTime() { @@ -88,12 +111,33 @@ public void setBoundaryTime(long boundaryTime) { this.boundaryTime = boundaryTime; } + public boolean isBoundaryExplicit() { + return boundaryExplicit; + } + + public void setBoundaryExplicit(boolean boundaryExplicit) { + this.boundaryExplicit = boundaryExplicit; + } + public long getStartTimeOffset() { return startTimeOffset; } public void setStartTimeOffset(long startTimeOffset) { this.startTimeOffset = startTimeOffset; + this.startTimeOffsetDuration = new TimeDuration(0, startTimeOffset); + } + + public TimeDuration getStartTimeOffsetDuration() { + return startTimeOffsetDuration; + } + + public void setStartTimeOffsetDuration(TimeDuration duration) { + this.startTimeOffsetDuration = duration; + this.startTimeOffset = + duration.monthDuration == 0 + ? duration.getTotalDuration(TimestampPrecisionUtils.currPrecision) + : 0; } public long getEndTimeOffset() { @@ -102,6 +146,19 @@ public long getEndTimeOffset() { public void setEndTimeOffset(long endTimeOffset) { this.endTimeOffset = endTimeOffset; + this.endTimeOffsetDuration = new TimeDuration(0, endTimeOffset); + } + + public TimeDuration getEndTimeOffsetDuration() { + return endTimeOffsetDuration; + } + + public void setEndTimeOffsetDuration(TimeDuration duration) { + this.endTimeOffsetDuration = duration; + this.endTimeOffset = + duration.monthDuration == 0 + ? duration.getTotalDuration(TimestampPrecisionUtils.currPrecision) + : 0; } public TimeoutPolicy getTimeoutPolicy() { @@ -135,11 +192,28 @@ public String constructFormattedSQL() { StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("CREATE CQ ").append(cqId).append('\n'); sqlBuilder.append("RESAMPLE\n"); - sqlBuilder.append('\t').append("EVERY ").append(everyInterval).append("ms\n"); + sqlBuilder + .append('\t') + .append("EVERY ") + .append( + everyDuration.monthDuration == 0 ? everyInterval + "ms" : formatDuration(everyDuration)) + .append('\n'); sqlBuilder.append('\t').append("BOUNDARY ").append(boundaryTime).append("\n"); - sqlBuilder.append('\t').append("RANGE ").append(startTimeOffset).append("ms"); - if (endTimeOffset != 0) { - sqlBuilder.append(", ").append(endTimeOffset).append("ms\n"); + sqlBuilder + .append('\t') + .append("RANGE ") + .append( + startTimeOffsetDuration.monthDuration == 0 + ? startTimeOffset + "ms" + : formatDuration(startTimeOffsetDuration)); + if (endTimeOffset != 0 || endTimeOffsetDuration.monthDuration != 0) { + sqlBuilder + .append(", ") + .append( + endTimeOffsetDuration.monthDuration == 0 + ? endTimeOffset + "ms" + : formatDuration(endTimeOffsetDuration)) + .append('\n'); } else { sqlBuilder.append("\n"); } @@ -154,6 +228,24 @@ public String constructFormattedSQL() { return sqlBuilder.toString(); } + private static String formatDuration(TimeDuration duration) { + StringBuilder result = new StringBuilder(); + int months = duration.monthDuration; + if (months >= 12) { + result.append(months / 12).append('y'); + months %= 12; + } + if (months != 0) { + result.append(months).append("mo"); + } + if (duration.nonMonthDuration != 0) { + result.append(duration.nonMonthDuration).append(TimestampPrecisionUtils.TIMESTAMP_PRECISION); + } + return result.length() == 0 + ? "0" + TimestampPrecisionUtils.TIMESTAMP_PRECISION + : result.toString(); + } + @Override public QueryType getQueryType() { return QueryType.OTHER; @@ -170,26 +262,49 @@ public R accept(StatementVisitor visitor, C context) { } public void semanticCheck() { - if (everyInterval - < IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval()) { + long minimumEvery = + IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval(); + long minimumElapsed = + everyDuration.monthDuration == 0 + ? everyDuration.nonMonthDuration + : Math.subtractExact( + Math.addExact( + Math.multiplyExact( + (long) everyDuration.monthDuration, + TimestampPrecisionUtils.currPrecision.convert( + 28L * 86_400_000L, java.util.concurrent.TimeUnit.MILLISECONDS)), + everyDuration.nonMonthDuration), + TimestampPrecisionUtils.currPrecision.convert( + 36L * 3_600_000L, java.util.concurrent.TimeUnit.MILLISECONDS)); + if (minimumElapsed < minimumEvery) { throw new SemanticException( - String.format( - DataNodeQueryMessages - .CQ_EVERY_INTERVAL_D_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL, - everyInterval, - IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval())); + everyDuration.monthDuration == 0 + ? String.format( + DataNodeQueryMessages + .CQ_EVERY_INTERVAL_D_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL, + everyInterval, + minimumEvery) + : String.format( + DataNodeQueryMessages + .CQ_EVERY_INTERVAL_SHOULD_NOT_BE_LOWER_THAN_THE_CONTINUOUS_QUERY_MINIMUM_EVERY_INTERVAL, + formatDuration(everyDuration), + minimumEvery)); } - if (startTimeOffset <= 0) { + if (!isPositive(everyDuration)) { + throw new SemanticException( + DataNodeQueryMessages.EXCEPTION_CQ_EVERY_INTERVAL_MUST_BE_POSITIVE_26259019); + } + if (!isPositive(startTimeOffsetDuration)) { throw new SemanticException(DataNodeQueryMessages.CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER); } - if (endTimeOffset < 0) { + if (endTimeOffsetDuration.monthDuration < 0 || endTimeOffsetDuration.nonMonthDuration < 0) { throw new SemanticException(DataNodeQueryMessages.CQ_THE_END_TIME_OFFSET_SHOULD_BE_GREATER); } - if (startTimeOffset <= endTimeOffset) { + if (!dominates(startTimeOffsetDuration, endTimeOffsetDuration, true)) { throw new SemanticException( DataNodeQueryMessages.CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_END_TIME_OFFSET); } - if (everyInterval > startTimeOffset) { + if (!dominates(startTimeOffsetDuration, everyDuration, false)) { throw new SemanticException( DataNodeQueryMessages .CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER_THAN_OR_EQUAL_TO_EVERY_INTERVAL); @@ -211,4 +326,20 @@ public void semanticCheck() { DataNodeQueryMessages.CQ_SPECIFYING_TIME_FILTERS_IN_THE_QUERY_BODY); } } + + private static boolean dominates(TimeDuration left, TimeDuration right, boolean strict) { + boolean result = + left.monthDuration >= right.monthDuration + && left.nonMonthDuration >= right.nonMonthDuration; + if (!result) { + return false; + } + return !strict + || left.monthDuration != right.monthDuration + || left.nonMonthDuration != right.nonMonthDuration; + } + + private static boolean isPositive(TimeDuration duration) { + return duration.monthDuration > 0 || duration.nonMonthDuration > 0; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 5ce282db7a701..1e1659c74bd1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -561,7 +561,9 @@ private void sendRegisterRequestToConfigNode(boolean isPreCheck) req.setPreCheck(isPreCheck); req.setDataNodeConfiguration(generateDataNodeConfiguration()); req.setClusterName(config.getClusterName()); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo( + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); TDataNodeRegisterResp dataNodeRegisterResp = null; while (retry > 0) { try (ConfigNodeClient configNodeClient = @@ -737,7 +739,9 @@ private void sendRestartRequestToConfigNode() throws StartupException { req.setClusterName( config.getClusterName() == null ? DEFAULT_CLUSTER_NAME : config.getClusterName()); req.setDataNodeConfiguration(generateDataNodeConfiguration()); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo( + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); req.setClusterId(config.getClusterId()); TDataNodeRestartResp dataNodeRestartResp = null; while (retry > 0) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java index 617572d0f8643..a5222ab7256da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java @@ -38,8 +38,70 @@ import java.time.ZoneId; import java.util.Calendar; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class DataNodeDateTimeUtils { + private static final Pattern CQ_DURATION_COMPONENT = + // Match multi-character units before their one-character prefixes (for example, ms before + // m), otherwise 1ms would be tokenized as 1m followed by an invalid trailing s. + Pattern.compile("(\\d+)(y|mo|w|d|h|ms|us|ns|m|s)", Pattern.CASE_INSENSITIVE); + + /** + * Parses the CQ duration grammar while retaining calendar months. Full aliases are deliberate not + * accepted here; CQ uses the same mo/y abbreviations as Tree SQL. + */ + public static TimeDuration constructTimeDurationForCQ(String duration) { + if (duration == null || duration.isEmpty()) { + throw new IllegalArgumentException( + DataNodeQueryMessages.EXCEPTION_CQ_DURATION_CANNOT_BE_EMPTY_C7269AB2); + } + Matcher matcher = CQ_DURATION_COMPONENT.matcher(duration); + long months = 0; + long fixed = 0; + int end = 0; + String precision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + while (matcher.find()) { + if (matcher.start() != end) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_INVALID_CQ_DURATION_ARG_F4917D5C, duration)); + } + long value; + try { + value = Long.parseLong(matcher.group(1)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_CQ_DURATION_COMPONENT_OVERFLOWS_ARG_ED5B0962, + duration), + e); + } + String unit = matcher.group(2).toLowerCase(Locale.ROOT); + if (unit.equals("y")) { + months = Math.addExact(months, Math.multiplyExact(value, 12)); + } else if (unit.equals("mo")) { + months = Math.addExact(months, value); + } else { + fixed = Math.addExact(fixed, convertDurationStrToLong(-1, value, unit, precision)); + } + end = matcher.end(); + } + if (end != duration.length()) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_INVALID_CQ_DURATION_ARG_F4917D5C, duration)); + } + if (months > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + DataNodeQueryMessages.EXCEPTION_CQ_DURATION_MONTH_COMPONENT_OVERFLOWS_ARG_EBF2A2B6, + duration)); + } + return new TimeDuration((int) months, fixed); + } + public static Long parseDateTimeExpressionToLong(String dateExpression, ZoneId zoneId) { ASTVisitor astVisitor = new ASTVisitor(); astVisitor.setZoneId(zoneId); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java new file mode 100644 index 0000000000000..9e5e1852560fd --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.metadata; + +import org.apache.iotdb.commons.exception.SemanticException; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.time.ZoneId; + +public class CreateContinuousQueryStatementTest { + + private long originalMinimumEvery; + + @Before + public void setUp() { + originalMinimumEvery = + IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval(); + } + + @After + public void tearDown() { + IoTDBDescriptor.getInstance() + .getConfig() + .setContinuousQueryMinimumEveryInterval(originalMinimumEvery); + } + + @Test + public void calendarEveryAndRangeAreAccepted() { + CreateContinuousQueryStatement statement = parse(calendarSql("1mo", "1mo")); + statement.semanticCheck(); + Assert.assertEquals(1, statement.getEveryDuration().monthDuration); + Assert.assertEquals(0, statement.getEveryDuration().nonMonthDuration); + Assert.assertEquals(1, statement.getStartTimeOffsetDuration().monthDuration); + } + + @Test + public void inheritedGroupByMonthKeepsCalendarEvery() { + CreateContinuousQueryStatement statement = + parse( + "CREATE CQ cq_inherited BEGIN " + + "SELECT max_value(s1) INTO root.sg.d1(s1_max) FROM root.sg.d1 GROUP BY(1mo) END"); + statement.semanticCheck(); + Assert.assertEquals(1, statement.getEveryDuration().monthDuration); + Assert.assertEquals(1, statement.getStartTimeOffsetDuration().monthDuration); + } + + @Test + public void compoundCalendarDurationDominatesAndIsAccepted() { + parse(calendarSql("1mo", "1mo3d")).semanticCheck(); + } + + @Test + public void incomparableCalendarAndFixedDurationsAreRejected() { + try { + parse(calendarSql("1mo", "30d")).semanticCheck(); + Assert.fail("expected incomparable RANGE vs EVERY to be rejected"); + } catch (SemanticException e) { + Assert.assertTrue( + e.getMessage() + .contains("The start time offset should be greater than or equal to every interval")); + } + } + + @Test + public void monthAndYearAliasesAreRejectedByTheParser() { + assertParseFails(calendarSql("1month", "1month")); + assertParseFails(calendarSql("1year", "1year")); + assertParseFails(calendarSql("2months", "2months")); + assertParseFails(calendarSql("2years", "2years")); + } + + @Test + public void calendarMinimumEveryUsesElapsedLowerBound() { + // 1mo lower bound is M * 28d - 36h. Reject when the configured minimum exceeds that bound. + IoTDBDescriptor.getInstance() + .getConfig() + .setContinuousQueryMinimumEveryInterval(2_289_600_001L); + try { + parse(calendarSql("1mo", "1mo")).semanticCheck(); + Assert.fail("expected 1mo to fail the conservative minimum-EVERY bound"); + } catch (SemanticException e) { + Assert.assertTrue(e.getMessage().contains("1mo")); + Assert.assertFalse(e.getMessage().contains("[0]")); + } + } + + private static CreateContinuousQueryStatement parse(String sql) { + return (CreateContinuousQueryStatement) + StatementGenerator.createStatement(sql, ZoneId.of("UTC")); + } + + private static void assertParseFails(String sql) { + try { + StatementGenerator.createStatement(sql, ZoneId.of("UTC")); + Assert.fail("expected parser to reject " + sql); + } catch (Exception ignored) { + // Tree SQL duration literals only accept y/mo abbreviations. + } + } + + private static String calendarSql(String every, String range) { + return "CREATE CQ cq_calendar RESAMPLE EVERY " + + every + + " RANGE " + + range + + " BEGIN SELECT max_value(s1) INTO root.sg.d1(s1_max) FROM root.sg.d1 GROUP BY(1mo) END"; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java index 0e48ffb883a9b..dcef69bb6f0d1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java @@ -394,6 +394,28 @@ public void testConstructTimeDuration() { Assert.assertEquals(10000000000L, timeDuration.nonMonthDuration); } + @Test + public void testConstructTimeDurationForCQSupportsMinuteAndSecond() { + TimeDuration timeDuration = DataNodeDateTimeUtils.constructTimeDurationForCQ("5m30s"); + Assert.assertEquals(0, timeDuration.monthDuration); + Assert.assertEquals(330_000L, timeDuration.nonMonthDuration); + + timeDuration = DataNodeDateTimeUtils.constructTimeDurationForCQ("1ms"); + Assert.assertEquals(0, timeDuration.monthDuration); + Assert.assertEquals(1L, timeDuration.nonMonthDuration); + + timeDuration = DataNodeDateTimeUtils.constructTimeDurationForCQ("1y2mo"); + Assert.assertEquals(14, timeDuration.monthDuration); + Assert.assertEquals(0, timeDuration.nonMonthDuration); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> DataNodeDateTimeUtils.constructTimeDurationForCQ("1month")); + Assert.assertThrows( + IllegalArgumentException.class, + () -> DataNodeDateTimeUtils.constructTimeDurationForCQ("1year")); + } + @Test public void testConstructTimeDurationOverflow() { Assert.assertThrows( diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 512e2e05f05fc..8ce6213a44004 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -653,6 +653,8 @@ struct TGetClusterIdResp { struct TNodeVersionInfo { 1: required string version; 2: required string buildInfo; + // Capabilities are optional so older nodes can still register. Missing means unsupported. + 3: optional set supportedCQDurationEncodingVersions; } struct TNodeActivateInfo { @@ -1132,6 +1134,11 @@ struct TGetCommitProgressResp { // ==================================================== // CQ // ==================================================== +struct TCQDuration { + 1: required i64 monthPart + 2: required i64 nonMonthDuration +} + struct TCreateCQReq { 1: required string cqId, 2: required i64 everyInterval @@ -1143,6 +1150,12 @@ struct TCreateCQReq { 8: required string sql 9: required string zoneId 10: required string username + // Versioned calendar-aware duration representation. Legacy fields above remain unchanged. + 11: optional i16 durationEncodingVersion + 12: optional TCQDuration everyDuration + 13: optional TCQDuration startOffsetDuration + 14: optional TCQDuration endOffsetDuration + 15: optional bool boundaryExplicit } struct TDropCQReq { From b76fdecbb46ac81dbd6bcf02a51eb20163751fb1 Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Sun, 20 Sep 2026 14:57:28 +0800 Subject: [PATCH 02/10] fix: recover legacy CQ snapshots and stale last-exec updates ReadWriteIOUtils.readInt throws a generic IOException at EOF, so CQInfo now treats a short remaining stream as a pre-extension snapshot instead of catching EOFException. Legacy CQs also reconcile durable progress after CQ_UPDATE_LAST_EXEC_TIME_ERROR so they do not stall. --- .../confignode/manager/cq/CQScheduleTask.java | 2 + .../confignode/persistence/cq/CQInfo.java | 9 +-- .../confignode/cq/CQScheduleTaskTest.java | 73 +++++++++++++++++++ .../confignode/persistence/CQInfoTest.java | 60 +++++++++++++++ 4 files changed, 138 insertions(+), 6 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java index 8d3b9b8454f00..4810dd2b0a2c1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java @@ -500,6 +500,8 @@ && needSubmit()) { () -> persistProgress(targetIndex, callbackTime), retryWaitTimeInMS, TimeUnit.MILLISECONDS); + } else if (needSubmit()) { + configManager.getCQManager().reconcileCQ(cqId, cqToken); } return; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java index 5af0d9794e034..c17047745ce27 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java @@ -44,7 +44,6 @@ import javax.annotation.concurrent.ThreadSafe; -import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -339,13 +338,11 @@ private void deserialize(InputStream stream) throws IOException { CQEntry cqEntry = CQEntry.deserializeLegacy(stream); cqMap.put(cqEntry.cqId, cqEntry); } - final int extensionMarker; - try { - extensionMarker = ReadWriteIOUtils.readInt(stream); - } catch (EOFException e) { - // Legacy snapshots end immediately after the legacy records. + if (stream.available() < Integer.BYTES) { + // Pre-extension snapshots end after the legacy records. return; } + int extensionMarker = ReadWriteIOUtils.readInt(stream); if (extensionMarker != SNAPSHOT_EXTENSION_MARKER) { return; } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java index 2df17ed9fd980..3cecae795a92e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java @@ -18,20 +18,32 @@ */ package org.apache.iotdb.confignode.cq; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; +import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.TimeDuration; import org.junit.Test; +import org.mockito.Mockito; +import java.lang.reflect.Constructor; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class CQScheduleTaskTest { @@ -165,4 +177,65 @@ public void testDiscardLowerBoundNeverMovesBeforeCurrentOccurrence() { assertEquals(2, lowerBound); assertEquals(3, Math.max(2 + 1, lowerBound)); } + + @Test + public void staleLastExecUpdateOnLegacyCqReconciles() throws Exception { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + ConsensusManager consensusManager = Mockito.mock(ConsensusManager.class); + CQManager cqManager = Mockito.mock(CQManager.class); + Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); + Mockito.when(consensusManager.isLeader()).thenReturn(true); + Mockito.when(consensusManager.write(Mockito.any())) + .thenReturn(new TSStatus(TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode())); + + ScheduledExecutorService executor = Mockito.mock(ScheduledExecutorService.class); + Mockito.when(executor.isShutdown()).thenReturn(false); + ScheduledFuture future = Mockito.mock(ScheduledFuture.class); + Mockito.when( + executor.schedule( + Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.any(TimeUnit.class))) + .thenReturn((ScheduledFuture) future); + + CQScheduleTask task = + new CQScheduleTask( + "legacyCq", + 1000, + 0, + 1000, + TimeoutPolicy.BLOCKED, + "select 1", + "token", + "Asia", + "root", + executor, + configManager, + 10_000); + + Field occurrenceIndex = CQScheduleTask.class.getDeclaredField("occurrenceIndex"); + occurrenceIndex.setAccessible(true); + assertEquals(-1L, occurrenceIndex.getLong(task)); + + Class callbackClass = null; + for (Class nested : CQScheduleTask.class.getDeclaredClasses()) { + if (nested.getSimpleName().equals("AsyncExecuteCQCallback")) { + callbackClass = nested; + break; + } + } + assertNotNull(callbackClass); + Constructor constructor = + callbackClass.getDeclaredConstructor( + CQScheduleTask.class, long.class, long.class, long.class); + constructor.setAccessible(true); + Object callback = constructor.newInstance(task, 0L, 1000L, 0L); + Method onComplete = callbackClass.getMethod("onComplete", TSStatus.class); + onComplete.setAccessible(true); + onComplete.invoke(callback, new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + Mockito.verify(consensusManager, Mockito.times(1)).write(Mockito.any()); + Mockito.verify(cqManager).reconcileCQ("legacyCq", "token"); + Mockito.verify(executor, Mockito.never()) + .schedule(Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.any(TimeUnit.class)); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java index 56547ac1e391b..2a135312da34a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java @@ -37,6 +37,8 @@ import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.file.Files; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -338,4 +340,62 @@ public void testCalendarOccurrenceProgressSurvivesSnapshotRecovery() throws Exce Assert.assertEquals(firstOccurrence, response.getCqList().get(0).getLastExecutionTime()); FileUtils.deleteDirectory(calendarSnapshotDir); } + + @Test + public void testPreExtensionSnapshotLoadsAsLegacyCq() throws Exception { + File legacyDir = new File(BASE_OUTPUT_PATH, "snapshot-pre-extension"); + if (legacyDir.exists()) { + FileUtils.deleteDirectory(legacyDir); + } + legacyDir.mkdirs(); + try { + TCreateCQReq req = + new TCreateCQReq( + "legacySnapshotCq", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select 1", + "create cq legacySnapshotCq", + "Asia", + "root"); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "legacySnapshotToken", 10_000)).getCode()); + Assert.assertTrue(cqInfo.processTakeSnapshot(legacyDir)); + + File snapshotFile = new File(legacyDir, "cq_info.snapshot"); + byte[] bytes = Files.readAllBytes(snapshotFile.toPath()); + byte[] marker = new byte[] {0x43, 0x51, 0x56, 0x31}; + int markerAt = indexOf(bytes, marker); + Assert.assertTrue(markerAt >= 0); + try (RandomAccessFile raf = new RandomAccessFile(snapshotFile, "rw")) { + raf.setLength(markerAt); + } + + CQInfo restored = new CQInfo(); + restored.processLoadSnapshot(legacyDir); + ShowCQResp response = restored.showCQ(new ShowCQPlan("legacySnapshotCq")); + Assert.assertEquals(1, response.getCqList().size()); + Assert.assertEquals("legacySnapshotCq", response.getCqList().get(0).getCqId()); + Assert.assertEquals(-1, response.getCqList().get(0).getNextOccurrenceIndex()); + } finally { + FileUtils.deleteDirectory(legacyDir); + } + } + + private static int indexOf(byte[] haystack, byte[] needle) { + outer: + for (int i = 0; i <= haystack.length - needle.length; i++) { + for (int j = 0; j < needle.length; j++) { + if (haystack[i + j] != needle[j]) { + continue outer; + } + } + return i; + } + return -1; + } } From c2a044e028cc64cb1f222d94ee9a849e09d5e1e6 Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Sun, 20 Sep 2026 16:16:29 +0800 Subject: [PATCH 03/10] fix: keep mixed CQ retry delay after recovery Fixed EVERY + calendar RANGE CQs persist everyInterval as 0. Recovery must use calendarAware when computing retry delay so restart/leader failover does not busy-spin on failure. --- .../confignode/manager/cq/CQScheduleTask.java | 7 +++- .../confignode/cq/CQScheduleTaskTest.java | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java index 4810dd2b0a2c1..eab3ceac4ed3b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java @@ -195,8 +195,13 @@ public CQScheduleTask( // Fixed-duration CQs do not need calendar arithmetic. Keep the zone opaque in that case so // legacy persisted entries containing a non-canonical zone string remain compatible. this.scheduleZone = calendarAware ? ZoneId.of(entry.getZoneId()) : null; - if (scheduleCalendarAware) { + // Mixed CQs (fixed EVERY + calendar RANGE) persist everyInterval as the zero sentinel. + // Retry delay must follow calendarAware, matching the create constructor; otherwise + // recovery inherits everyInterval / FACTOR == 0 and busy-spins on failure. + if (calendarAware) { this.retryWaitTimeInMS = calculateRetryWaitTime(everyDuration); + } + if (scheduleCalendarAware) { if (!entry.isBoundaryExplicit()) { this.boundaryTime = CQCalendarUtils.localEpochBoundary(scheduleZone); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java index 3cecae795a92e..1097c8d6bfb84 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQScheduleTaskTest.java @@ -20,11 +20,14 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.confignode.consensus.request.read.cq.ShowCQPlan; +import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.persistence.cq.CQInfo; import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.rpc.TSStatusCode; @@ -178,6 +181,39 @@ public void testDiscardLowerBoundNeverMovesBeforeCurrentOccurrence() { assertEquals(3, Math.max(2 + 1, lowerBound)); } + @Test + public void recoveredMixedCalendarRangeKeepsPositiveRetryWait() throws Exception { + TCreateCQReq req = + new TCreateCQReq( + "mixedRetryCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq mixedRetryCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1_000)); + req.setStartOffsetDuration(new TCQDuration(1, 1_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + CQInfo cqInfo = new CQInfo(); + assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "mixedRetryToken", 10_000)).getCode()); + CQInfo.CQEntry entry = cqInfo.showCQ(new ShowCQPlan("mixedRetryCq")).getCqList().get(0); + + CQScheduleTask recovered = new CQScheduleTask(entry, null, null); + Field retryWaitTimeInMS = CQScheduleTask.class.getDeclaredField("retryWaitTimeInMS"); + retryWaitTimeInMS.setAccessible(true); + // Structured EVERY is 1000ms; the persisted legacy everyInterval is the zero sentinel. + assertEquals(1_000L, retryWaitTimeInMS.getLong(recovered)); + } + @Test public void staleLastExecUpdateOnLegacyCqReconciles() throws Exception { ConfigManager configManager = Mockito.mock(ConfigManager.class); From 7fe3c2b1e8da2094d93f1b55fdb28be59ca3c367 Mon Sep 17 00:00:00 2001 From: DaZuiZui Date: Sun, 20 Sep 2026 16:59:01 +0800 Subject: [PATCH 04/10] test: cover calendar CQ natural-month windows and encoding rejects Add n=0 RANGE, DISCARD skip, mixed-version ingress, and an execution IT that asserts EVERY 1mo RANGE 1mo uses the just-finished natural month. --- .../apache/iotdb/db/it/cq/IoTDBCQExecIT.java | 85 ++++++++++++++ .../iotdb/confignode/cq/CQManagerTest.java | 109 ++++++++++++++++++ .../manager/cq/CQCalendarUtilsTest.java | 94 +++++++++++++++ .../confignode/persistence/CQInfoTest.java | 37 ++++++ .../ClusterConfigTaskExecutorCQTest.java | 77 +++++++++++++ .../CreateContinuousQueryStatementTest.java | 11 ++ .../iotdb/db/utils/DateTimeUtilsTest.java | 3 + 7 files changed, 416 insertions(+) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java index ea00bbfa646a9..6476c9cc1fce1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java @@ -32,6 +32,9 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.Statement; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.concurrent.TimeUnit; import static org.apache.iotdb.itbase.constant.TestConstant.TIMESTAMP_STR; @@ -52,6 +55,88 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } + @Test + public void testCalendarMonthCQExecutionUsesNaturalMonthWindow() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + connection.setClientInfo("time_zone", "UTC"); + long now = System.currentTimeMillis(); + long firstExecutionTime = now + 10_000; + ZoneId utc = ZoneId.of("UTC"); + long calendarStart = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(firstExecutionTime), utc) + .minusMonths(1) + .toInstant() + .toEpochMilli(); + long thirtyDayStart = firstExecutionTime - TimeUnit.DAYS.toMillis(30); + + statement.execute("create timeseries root.sg.calendar.s1 WITH DATATYPE=INT64"); + statement.execute("create timeseries root.sg.calendar.s1_max WITH DATATYPE=INT64"); + statement.execute("INSERT INTO root.sg.calendar(time, s1) VALUES (0,0)"); + statement.execute( + String.format( + "INSERT INTO root.sg.calendar(time, s1) VALUES (%d, 777), (%d, 100), (%d, 10), (%d, 999)", + calendarStart - 1, calendarStart, firstExecutionTime - 1, firstExecutionTime)); + // On 28/29-day months the 30-day window starts before the natural month. A 30d RANGE + // would include this 888; a calendar RANGE 1mo must not. + if (thirtyDayStart < calendarStart - 1) { + statement.execute( + String.format( + "INSERT INTO root.sg.calendar(time, s1) VALUES (%d, 888)", thirtyDayStart)); + } + + statement.execute( + "CREATE CONTINUOUS QUERY cq_calendar_month\n" + + "RESAMPLE EVERY 1mo\n" + + String.format("BOUNDARY %d\n", firstExecutionTime) + + "RANGE 1mo\n" + + "BEGIN\n" + + " SELECT max_value(s1)\n" + + " INTO root.sg.calendar(s1_max)\n" + + " FROM root.sg.calendar\n" + + " GROUP BY(1mo)\n" + + "END"); + + if (System.currentTimeMillis() > firstExecutionTime) { + statement.execute("DROP CQ cq_calendar_month"); + return; + } + + long targetTime = firstExecutionTime + 10_000; + while (System.currentTimeMillis() - targetTime < 0) { + TimeUnit.SECONDS.sleep(1); + } + + try (ResultSet resultSet = statement.executeQuery("select s1_max from root.sg.calendar")) { + boolean sawNaturalMonthBucket = false; + while (resultSet.next()) { + long time = resultSet.getLong(TIMESTAMP_STR); + long value = resultSet.getLong("root.sg.calendar.s1_max"); + assertEquals( + "CQ RANGE must stay inside the just-finished natural month", + true, + time >= calendarStart && time < firstExecutionTime); + assertEquals( + "points before/after the natural month must not contribute", true, value != 777); + assertEquals("the BOUNDARY instant is exclusive", true, value != 999); + assertEquals("a 30-day-only point must not become the monthly max", true, value != 888); + if (time == calendarStart && value == 100) { + sawNaturalMonthBucket = true; + } + } + assertEquals( + "the first GROUP BY month bucket should start at BOUNDARY-1mo", + true, + sawNaturalMonthBucket); + } finally { + statement.execute("DROP CQ cq_calendar_month"); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + @Test public void testCQExecution1() { String insertTemplate = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java index b068d35ba4bf0..71b990c75bfdd 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java @@ -186,6 +186,115 @@ public void invalidStructuredDurationIsRejectedAtRpcIngress() { } } + @Test + public void unknownDurationEncodingVersionIsRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = versionedReq("unknownVersionCq", 0, 1_000, 0, 1_000); + req.setDurationEncodingVersion((short) 2); + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void partialStructuredDurationFieldsAreRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = + new TCreateCQReq( + "partialFieldsCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq partialFieldsCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setBoundaryExplicit(true); + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_INVALID_CQ_DURATION_ENCODING_VERSION_1_REQUIRES_ALL_STRUCTURED_FIELDS_FEAD7F92, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void calendarLegacySentinelConflictIsRejectedAtRpcIngress() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = versionedReq("sentinelConflictCq", 1, 0, 1, 0); + req.everyInterval = 2_592_000_000L; + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + @Test + public void fixedLegacyFieldsMustMirrorStructuredDurations() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + try { + TCreateCQReq req = versionedReq("fixedConflictCq", 0, 1_000, 0, 1_000); + req.everyInterval = 2_000; + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + + private static TCreateCQReq versionedReq( + String cqId, long everyMonths, long everyFixed, long startMonths, long startFixed) { + boolean calendar = everyMonths != 0 || startMonths != 0; + TCreateCQReq req = + new TCreateCQReq( + cqId, + calendar ? 0 : everyFixed, + 0, + calendar ? 0 : startFixed, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq " + cqId, + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(everyMonths, everyFixed)); + req.setStartOffsetDuration(new TCQDuration(startMonths, startFixed)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + return req; + } + @SuppressWarnings("unchecked") private CQScheduleTask newScheduledTask( ConfigManager configManager, ScheduledFuture scheduledFuture, String cqToken) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java index cb3402b434e46..65d35c750d663 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java @@ -146,6 +146,69 @@ public void testCalendarCadenceRangeRetainsOriginalMonthEndAnchor() { assertEquals(execution, task.calculateCalendarRangeEndpoint(zero, 3)); } + @Test + public void testFirstOccurrenceRangeIsTheJustFinishedNaturalMonth() { + long boundary = epochTimestamp(2024, 3, 1, 0, 0, UTC); + TimeDuration month = new TimeDuration(1, 0); + TimeDuration zero = new TimeDuration(0, 0); + CQScheduleTask task = calendarTask(boundary, boundary, month, month, zero, UTC); + + // Occurrence 0 is the user-visible first fire. RANGE 1mo must look backward from the original + // boundary, including the negative month vector, rather than subtracting from a later clamped + // occurrence. + assertEquals( + epochTimestamp(2024, 2, 1, 0, 0, UTC), task.calculateCalendarRangeEndpoint(month, 0)); + assertEquals(boundary, task.calculateCalendarRangeEndpoint(zero, 0)); + assertEquals( + epochTimestamp(2024, 2, 1, 0, 0, UTC), CQCalendarUtils.applyVector(boundary, -1, 0, UTC)); + } + + @Test + public void testMonthEndFirstOccurrenceRangeClampsFebruary() { + long boundary = epochTimestamp(2024, 3, 31, 0, 0, UTC); + TimeDuration month = new TimeDuration(1, 0); + TimeDuration zero = new TimeDuration(0, 0); + CQScheduleTask task = calendarTask(boundary, boundary, month, month, zero, UTC); + + assertEquals( + epochTimestamp(2024, 2, 29, 0, 0, UTC), task.calculateCalendarRangeEndpoint(month, 0)); + assertEquals(boundary, task.calculateCalendarRangeEndpoint(zero, 0)); + } + + @Test + public void testCalendarTimeoutUsesActualAdjacentOccurrenceDistance() throws Exception { + long boundary = epochTimestamp(2024, 1, 31, 0, 0, UTC); + TimeDuration month = new TimeDuration(1, 0); + TimeDuration zero = new TimeDuration(0, 0); + java.lang.reflect.Method timeout = + CQScheduleTask.class.getDeclaredMethod("calculateCalendarTimeoutMillis", long.class); + timeout.setAccessible(true); + + long januaryToFebruary = epochTimestamp(2024, 2, 29, 0, 0, UTC) - boundary; + long februaryToMarch = + epochTimestamp(2024, 3, 31, 0, 0, UTC) - epochTimestamp(2024, 2, 29, 0, 0, UTC); + CQScheduleTask first = calendarTask(boundary, boundary, month, month, zero, UTC); + CQScheduleTask second = + calendarTask(boundary, epochTimestamp(2024, 2, 29, 0, 0, UTC), month, month, zero, UTC); + // n=0 spans 29 days (Jan 31 -> Feb 29); n=1 spans 31 days (Feb 29 -> Mar 31). Neither is 30d. + assertEquals(januaryToFebruary, timeout.invoke(first, 0L)); + assertEquals(februaryToMarch, timeout.invoke(second, 1L)); + } + + @Test + public void testCheckedCalendarArithmeticRejectsOverflow() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + try { + CQCalendarUtils.apply(boundary, new TimeDuration(1, 0), Integer.MAX_VALUE + 1L, UTC); + org.junit.Assert.fail("expected month multiplication to overflow"); + } catch (IllegalArgumentException e) { + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .EXCEPTION_CQ_TIMESTAMP_OVERFLOWS_CONFIGURED_PRECISION_F5FB230C, + e.getMessage()); + } + } + private static CQScheduleTask calendarTask( long boundary, long execution, @@ -239,6 +302,37 @@ public void testDiscardNeverMovesBeforeTheCurrentOccurrence() { UTC)); } + @Test + public void testDiscardSkipsMultipleMissedCalendarOccurrences() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long executionTime = CQCalendarUtils.occurrence(boundary, every, 1, UTC); + long callbackTime = epochTimestamp(2024, 5, 15, 12, 0, UTC); + + // Missed March/April/May. The next durable index is June (n=5), not February+1. + assertEquals( + 5, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.DISCARD, 1, callbackTime, executionTime, 0, 1, boundary, every, UTC)); + assertEquals( + 2, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.BLOCKED, 1, callbackTime, executionTime, 0, 1, boundary, every, UTC)); + } + + @Test + public void testDiscardClockRollbackKeepsTheNextOccurrence() { + long boundary = epochTimestamp(2024, 1, 1, 0, 0, UTC); + TimeDuration every = new TimeDuration(1, 0); + long executionTime = CQCalendarUtils.occurrence(boundary, every, 2, UTC); + long callbackTime = epochTimestamp(2023, 12, 1, 0, 0, UTC); + + assertEquals( + 3, + CQScheduleTask.calculateNextOccurrenceIndex( + TimeoutPolicy.DISCARD, 2, callbackTime, executionTime, 0, 2, boundary, every, UTC)); + } + private static long epochTimestamp( int year, int month, int day, int hour, int minute, ZoneId zone) { Instant instant = ZonedDateTime.of(year, month, day, hour, minute, 0, 0, zone).toInstant(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java index 2a135312da34a..ccca5bc035035 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java @@ -202,6 +202,43 @@ public void testOccurrenceIndexCasFencesOutOfOrderCallbacks() { .getCode()); } + @Test + public void testOccurrenceIndexCasRejectsAheadCallbackAndTokenMismatch() { + TCreateCQReq req = + new TCreateCQReq( + "fencedCq", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select 1", + "create cq fencedCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 1000)); + req.setStartOffsetDuration(new TCQDuration(0, 1000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.addCQ(new AddCQPlan(req, "fencedToken", 1000)).getCode()); + + Assert.assertEquals( + TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("fencedCq", 2000, "fencedToken", 2, 3)) + .getCode()); + Assert.assertEquals( + TSStatusCode.NO_SUCH_CQ.getStatusCode(), + cqInfo + .updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("fencedCq", 1000, "otherToken", 1, 2)) + .getCode()); + } + @Test public void testCommittedProgressRetryIsDetectedAsStaleForReconciliation() throws TException { TCreateCQReq req = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java new file mode 100644 index 0000000000000..c8065a975ca97 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.executor; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class ClusterConfigTaskExecutorCQTest { + + @Test + public void mixedVersionClusterIsRejectedAtDataNodeIngress() throws Exception { + Method method = + ClusterConfigTaskExecutor.class.getDeclaredMethod( + "allClusterNodesSupportCQDurationEncoding", TShowClusterResp.class); + method.setAccessible(true); + ClusterConfigTaskExecutor executor = ClusterConfigTaskExecutor.getInstance(); + + Assert.assertFalse((Boolean) method.invoke(executor, new TShowClusterResp())); + Assert.assertFalse((Boolean) method.invoke(executor, (Object) null)); + + TConfigNodeLocation configNode = + new TConfigNodeLocation( + 1, new TEndPoint("127.0.0.1", 10710), new TEndPoint("127.0.0.1", 10720)); + TDataNodeLocation dataNode = + new TDataNodeLocation( + 11, + new TEndPoint("127.0.0.1", 6667), + new TEndPoint("127.0.0.1", 10730), + new TEndPoint("127.0.0.1", 10740), + new TEndPoint("127.0.0.1", 10750), + new TEndPoint("127.0.0.1", 10760)); + TNodeVersionInfo supported = + new TNodeVersionInfo("2.0.0", "new") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + TNodeVersionInfo unsupported = new TNodeVersionInfo("1.3.0", "old"); + + TShowClusterResp mixed = new TShowClusterResp(); + mixed.setConfigNodeList(Collections.singletonList(configNode)); + mixed.setDataNodeList(Collections.singletonList(dataNode)); + Map versions = new HashMap<>(); + versions.put(configNode.getConfigNodeId(), supported); + versions.put(dataNode.getDataNodeId(), unsupported); + mixed.setNodeVersionInfo(versions); + Assert.assertFalse((Boolean) method.invoke(executor, mixed)); + + versions.put(dataNode.getDataNodeId(), supported); + Assert.assertTrue((Boolean) method.invoke(executor, mixed)); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java index 9e5e1852560fd..747ec62977693 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatementTest.java @@ -92,6 +92,17 @@ public void monthAndYearAliasesAreRejectedByTheParser() { assertParseFails(calendarSql("2years", "2years")); } + @Test + public void inheritedGroupByYearKeepsCalendarEvery() { + CreateContinuousQueryStatement statement = + parse( + "CREATE CQ cq_inherited_year BEGIN " + + "SELECT max_value(s1) INTO root.sg.d1(s1_max) FROM root.sg.d1 GROUP BY(1y) END"); + statement.semanticCheck(); + Assert.assertEquals(12, statement.getEveryDuration().monthDuration); + Assert.assertEquals(12, statement.getStartTimeOffsetDuration().monthDuration); + } + @Test public void calendarMinimumEveryUsesElapsedLowerBound() { // 1mo lower bound is M * 28d - 36h. Reject when the configured minimum exceeds that bound. diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java index dcef69bb6f0d1..321dd0699b981 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/DateTimeUtilsTest.java @@ -414,6 +414,9 @@ public void testConstructTimeDurationForCQSupportsMinuteAndSecond() { Assert.assertThrows( IllegalArgumentException.class, () -> DataNodeDateTimeUtils.constructTimeDurationForCQ("1year")); + Assert.assertThrows( + IllegalArgumentException.class, + () -> DataNodeDateTimeUtils.constructTimeDurationForCQ("1mo 1d")); } @Test From fac80393060a071a9aafda0e27a3a31cc250ca6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:56:59 +0800 Subject: [PATCH 05/10] fix: harden CQ snapshot recovery stream and drop dead legacy branch CQInfo.processLoadSnapshot used a raw FileInputStream, whose available() does not contractually guarantee the remaining byte count, for the extension-marker presence check in deserialize(). Read the snapshot into memory via ByteArrayInputStream instead, matching the pattern already used in NodeInfo for the same kind of optional-trailing-data probe. Also remove the occurrenceIndex < 0 branch in CQScheduleTask#occurrenceAt: every call site (both constructors and persistProgress's legacy-path guard) only invokes it once occurrenceIndex is already non-negative, so the branch was unreachable. --- .../iotdb/confignode/manager/cq/CQScheduleTask.java | 6 +++--- .../apache/iotdb/confignode/persistence/cq/CQInfo.java | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java index eab3ceac4ed3b..c5b473df9045b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java @@ -401,9 +401,9 @@ private static long calculateRetryWaitTime(TimeDuration duration) { } private long occurrenceAt(long index) { - if (occurrenceIndex < 0) { - return executionTime + (index - occurrenceIndex) * everyInterval; - } + // All current call sites only invoke this once occurrenceIndex has been established + // as non-negative (see the constructors and the persistProgress legacy-path guard), + // so this always resolves through calendar-aware arithmetic. return CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, scheduleZone); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java index c17047745ce27..269e9d38f6790 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java @@ -44,12 +44,13 @@ import javax.annotation.concurrent.ThreadSafe; +import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -374,11 +375,12 @@ public void processLoadSnapshot(File snapshotDir) throws TException, IOException return; } lock.writeLock().lock(); - try (FileInputStream fileInputStream = new FileInputStream(snapshotFile)) { + try (ByteArrayInputStream inputStream = + new ByteArrayInputStream(Files.readAllBytes(snapshotFile.toPath()))) { clear(); - deserialize(fileInputStream); + deserialize(inputStream); } finally { lock.writeLock().unlock(); From 857388d8ab8ed4df72f1f33c9f7b1b482076c23e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:10:04 +0800 Subject: [PATCH 06/10] Remove out-of-scope mixed-version cluster capability negotiation from CQ calendar durations Issue #18428 only asks for calendar-aware EVERY/RANGE duration support in continuous queries. The previous commits on this branch additionally introduced a cluster-wide node capability negotiation mechanism (TNodeVersionInfo.supportedCQDurationEncodingVersions, snapshot/procedure/ plan serialization tails guarded by a magic marker, and "fail closed" gating in NodeManager/CQManager/ClusterConfigTaskExecutor that rejects calendar-duration CQs unless every node in the cluster advertises support). That mechanism addresses a mixed-version-cluster rollout concern that the issue never raised, and added a second, independent surface for bugs. This trims the branch back to what the issue actually asks for: the TCQDuration wire format, TCreateCQReq's structured duration fields, and CQManager's structural/semantic validation stay; the capability-negotiation scaffolding and its tests are removed. --- .../confignode/i18n/ManagerMessages.java | 2 - .../confignode/i18n/ManagerMessages.java | 2 - .../confignode/UpdateVersionInfoPlan.java | 19 ----- .../confignode/manager/cq/CQManager.java | 67 ---------------- .../confignode/manager/node/NodeManager.java | 34 -------- .../confignode/persistence/cq/CQInfo.java | 6 -- .../confignode/persistence/node/NodeInfo.java | 39 ---------- .../impl/node/AddConfigNodeProcedure.java | 19 ----- .../iotdb/confignode/service/ConfigNode.java | 14 ++-- .../confignode/UpdateVersionInfoPlanTest.java | 69 ----------------- .../iotdb/confignode/cq/CQManagerTest.java | 39 ---------- .../manager/node/NodeManagerTest.java | 73 +----------------- .../confignode/persistence/NodeInfoTest.java | 32 -------- .../impl/node/AddConfigNodeProcedureTest.java | 21 +---- .../iotdb/db/i18n/DataNodeQueryMessages.java | 3 - .../iotdb/db/i18n/DataNodeQueryMessages.java | 3 - .../executor/ClusterConfigTaskExecutor.java | 37 --------- .../org/apache/iotdb/db/service/DataNode.java | 8 +- .../ClusterConfigTaskExecutorCQTest.java | 77 ------------------- .../src/main/thrift/confignode.thrift | 2 - 20 files changed, 9 insertions(+), 557 deletions(-) delete mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 885dc8339f75c..89b981cefebeb 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -723,8 +723,6 @@ private ManagerMessages() {} public static final String MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67 = "CQ legacy duration fields conflict with structured duration fields"; - public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072 = - "CQ calendar duration requires all cluster nodes to support duration encoding version 1"; public static final String MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A = "CQ duration encoding marker is required for new requests"; public static final String MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C = diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index b56336a27fc73..7aed04c396249 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -702,8 +702,6 @@ private ManagerMessages() {} public static final String MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67 = "CQ legacy duration fields 与结构化 duration 字段冲突"; - public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072 = - "CQ 日历 duration 要求集群所有节点支持 duration encoding version 1"; public static final String MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A = "新建 CQ 请求必须包含 duration encoding marker"; public static final String MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java index d2894392e26bc..62f2cc71e6926 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java @@ -59,14 +59,6 @@ protected void serializeImpl(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(nodeId, stream); ReadWriteIOUtils.write(versionInfo.getVersion(), stream); ReadWriteIOUtils.write(versionInfo.getBuildInfo(), stream); - // Optional tail keeps this plan readable by pre-calendar-duration ConfigNodes. - ReadWriteIOUtils.write(versionInfo.isSetSupportedCQDurationEncodingVersions(), stream); - if (versionInfo.isSetSupportedCQDurationEncodingVersions()) { - ReadWriteIOUtils.write(versionInfo.getSupportedCQDurationEncodingVersions().size(), stream); - for (short version : versionInfo.getSupportedCQDurationEncodingVersions()) { - ReadWriteIOUtils.write(version, stream); - } - } } @Override @@ -75,17 +67,6 @@ protected void deserializeImpl(ByteBuffer buffer) { versionInfo = new TNodeVersionInfo( ReadWriteIOUtils.readString(buffer), ReadWriteIOUtils.readString(buffer)); - if (buffer.hasRemaining()) { - boolean hasCapabilities = ReadWriteIOUtils.readBool(buffer); - if (hasCapabilities) { - int size = ReadWriteIOUtils.readInt(buffer); - java.util.Set capabilities = new java.util.HashSet<>(); - for (int i = 0; i < size; i++) { - capabilities.add(ReadWriteIOUtils.readShort(buffer)); - } - versionInfo.setSupportedCQDurationEncodingVersions(capabilities); - } - } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java index 0159cc3af3be3..9ef6009f1e2b2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java @@ -34,7 +34,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; -import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowCQResp; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -145,11 +144,6 @@ private TSStatus validateDurationEncoding(TCreateCQReq req) { if (semanticValidation != null) { return semanticValidation; } - if (hasCalendarDuration && !allClusterNodesSupportDurationEncodingV1()) { - return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) - .setMessage( - ManagerMessages.MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072); - } return null; } @@ -197,67 +191,6 @@ private static boolean dominates(TCQDuration left, TCQDuration right, boolean st || left.getNonMonthDuration() != right.getNonMonthDuration()); } - /** Returns true when any persisted CQ requires the structured calendar-duration reader. */ - public boolean hasCalendarDurationCQ() { - try { - DataSet response = configManager.getConsensusManager().read(new ShowCQPlan()); - if (!(response instanceof ShowCQResp)) { - // Do not allow a node with unknown metadata to join while the reader barrier is active. - return true; - } - if (((ShowCQResp) response).getCqList() == null) { - // A malformed response is just as unsafe as an unavailable response for this barrier. - return true; - } - return ((ShowCQResp) response) - .getCqList().stream().anyMatch(CQInfo.CQEntry::hasCalendarDuration); - } catch (ConsensusException e) { - // A failed metadata read must fail closed: an old reader must never be admitted blindly. - LOGGER.warn(ManagerMessages.UNEXPECTED_ERROR_HAPPENED_WHILE_FETCHING_CQ_LIST, e); - return true; - } - } - - private boolean allClusterNodesSupportDurationEncodingV1() { - java.util.Map versionInfo = - configManager.getNodeManager().getNodeVersionInfo(); - if (versionInfo == null || versionInfo.isEmpty()) { - return false; - } - boolean hasRegisteredNode = false; - // Check every registered node explicitly. A missing heartbeat/version entry must not allow a - // calendar CQ to be created during a rolling upgrade. - List configNodes = - configManager.getNodeManager().getRegisteredConfigNodes(); - List dataNodes = - configManager.getNodeManager().getRegisteredDataNodes(); - if (configNodes == null || dataNodes == null) { - return false; - } - for (org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation node : configNodes) { - hasRegisteredNode = true; - if (!supportsDurationEncodingV1(versionInfo.get(node.getConfigNodeId()))) { - return false; - } - } - for (org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration node : dataNodes) { - hasRegisteredNode = true; - if (!supportsDurationEncodingV1(versionInfo.get(node.getLocation().getDataNodeId()))) { - return false; - } - } - return hasRegisteredNode; - } - - private boolean supportsDurationEncodingV1(TNodeVersionInfo info) { - if (info == null - || !info.isSetSupportedCQDurationEncodingVersions() - || !info.getSupportedCQDurationEncodingVersions().contains((short) 1)) { - return false; - } - return true; - } - public TSStatus dropCQ(TDropCQReq req) { lock.readLock().lock(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 35a51b0929c49..598c47e692d26 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -331,11 +331,6 @@ private TRuntimeConfiguration getRuntimeConfiguration(int dataNodeId) { public DataSet registerDataNode(TDataNodeRegisterReq req) { DataNodeRegisterResp resp = new DataNodeRegisterResp(); resp.setConfigNodeList(getRegisteredConfigNodes()); - TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); - if (capabilityStatus != null) { - resp.setStatus(capabilityStatus); - return resp; - } // Create a new DataNodeHeartbeatCache and force update NodeStatus int dataNodeId = nodeInfo.generateNextNodeId(); @@ -378,11 +373,6 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { public TDataNodeRestartResp updateDataNodeIfNecessary(TDataNodeRestartReq req) { TDataNodeRestartResp resp = new TDataNodeRestartResp(); resp.setConfigNodeList(getRegisteredConfigNodes()); - TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); - if (capabilityStatus != null) { - resp.setStatus(capabilityStatus); - return resp; - } final String clusterId = configManager .getClusterManager() @@ -483,10 +473,6 @@ public DataSet removeDataNode(RemoveDataNodePlan removeDataNodePlan) { } public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { - TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); - if (capabilityStatus != null) { - return new TConfigNodeRegisterResp().setStatus(capabilityStatus).setConfigNodeId(-1); - } int nodeId = nodeInfo.generateNextNodeId(); req.getConfigNodeLocation().setConfigNodeId(nodeId); configManager.getProcedureManager().addConfigNode(req); @@ -496,10 +482,6 @@ public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { } public TSStatus updateConfigNodeIfNecessary(int configNodeId, TNodeVersionInfo versionInfo) { - TSStatus capabilityStatus = validateDurationCapability(versionInfo); - if (capabilityStatus != null) { - return capabilityStatus; - } TNodeVersionInfo recordVersionInfo = nodeInfo.getVersionInfo(configNodeId); if (!recordVersionInfo.equals(versionInfo)) { // Update versionInfo when modified during restart @@ -514,22 +496,6 @@ public TSStatus updateConfigNodeIfNecessary(int configNodeId, TNodeVersionInfo v return ClusterNodeStartUtils.ACCEPT_NODE_RESTART; } - private TSStatus validateDurationCapability(TNodeVersionInfo versionInfo) { - if (!supportsDurationEncodingV1(versionInfo) - && configManager.getCQManager().hasCalendarDurationCQ()) { - return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) - .setMessage( - ManagerMessages.MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072); - } - return null; - } - - private static boolean supportsDurationEncodingV1(TNodeVersionInfo versionInfo) { - return versionInfo != null - && versionInfo.isSetSupportedCQDurationEncodingVersions() - && versionInfo.getSupportedCQDurationEncodingVersions().contains((short) 1); - } - public List getRegisteredAINodeInfoList() { List aiNodeInfoList = new ArrayList<>(); for (TAINodeConfiguration aiNodeConfiguration : getRegisteredAINodes()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java index 269e9d38f6790..8596df0ddcf7b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java @@ -687,12 +687,6 @@ public boolean isBoundaryExplicit() { return boundaryExplicit; } - public boolean hasCalendarDuration() { - return everyDuration.monthDuration != 0 - || startTimeOffsetDuration.monthDuration != 0 - || endTimeOffsetDuration.monthDuration != 0; - } - private static org.apache.tsfile.utils.TimeDuration durationFromReq( TCreateCQReq req, TCQDuration d, long legacy) { if (req.isSetDurationEncodingVersion() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java index a6878b963cbed..d98a05c8a8cdc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java @@ -68,7 +68,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -723,19 +722,6 @@ private void serializeVersionInfo(OutputStream outputStream) throws IOException ReadWriteIOUtils.write(entry.getValue().getVersion(), outputStream); ReadWriteIOUtils.write(entry.getValue().getBuildInfo(), outputStream); } - // Optional tail; legacy snapshots end after the two strings above. - ReadWriteIOUtils.write(0x43515631, outputStream); - ReadWriteIOUtils.write(nodeVersionInfo.size(), outputStream); - for (Entry entry : nodeVersionInfo.entrySet()) { - ReadWriteIOUtils.write(entry.getKey(), outputStream); - Set capabilities = entry.getValue().getSupportedCQDurationEncodingVersions(); - ReadWriteIOUtils.write(capabilities == null ? 0 : capabilities.size(), outputStream); - if (capabilities != null) { - for (short capability : capabilities) { - ReadWriteIOUtils.write(capability, outputStream); - } - } - } } @Override @@ -841,31 +827,6 @@ private void deserializeBuildInfo(InputStream inputStream) throws IOException { nodeVersionInfo.put(nodeId, new TNodeVersionInfo(version, buildInfo)); size--; } - if (inputStream.available() >= Integer.BYTES) { - inputStream.mark(Integer.BYTES); - int marker = ReadWriteIOUtils.readInt(inputStream); - if (marker == 0x43515631) { - int capabilitySize = ReadWriteIOUtils.readInt(inputStream); - for (int i = 0; i < capabilitySize; i++) { - int nodeId = ReadWriteIOUtils.readInt(inputStream); - int sizeOfCapabilities = ReadWriteIOUtils.readInt(inputStream); - TNodeVersionInfo info = nodeVersionInfo.get(nodeId); - if (info != null) { - Set capabilities = new java.util.HashSet<>(); - for (int j = 0; j < sizeOfCapabilities; j++) { - capabilities.add(ReadWriteIOUtils.readShort(inputStream)); - } - info.setSupportedCQDurationEncodingVersions(capabilities); - } else { - for (int j = 0; j < sizeOfCapabilities; j++) { - ReadWriteIOUtils.readShort(inputStream); - } - } - } - } else { - inputStream.reset(); - } - } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java index ad6f7233bf1b4..a3a5ffb41de4e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java @@ -36,9 +36,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; -import java.util.Set; /** add config node procedure */ public class AddConfigNodeProcedure extends AbstractNodeProcedure { @@ -158,15 +156,6 @@ public void serialize(DataOutputStream stream) throws IOException { ThriftConfigNodeSerDeUtils.serializeTConfigNodeLocation(tConfigNodeLocation, stream); ReadWriteIOUtils.write(versionInfo.getVersion(), stream); ReadWriteIOUtils.write(versionInfo.getBuildInfo(), stream); - // Optional tail preserves the joining node's capability across procedure recovery while old - // payloads remain readable because the tail is absent in their serialized form. - ReadWriteIOUtils.write(versionInfo.isSetSupportedCQDurationEncodingVersions(), stream); - if (versionInfo.isSetSupportedCQDurationEncodingVersions()) { - ReadWriteIOUtils.write(versionInfo.getSupportedCQDurationEncodingVersions().size(), stream); - for (short version : versionInfo.getSupportedCQDurationEncodingVersions()) { - ReadWriteIOUtils.write(version, stream); - } - } } @Override @@ -180,14 +169,6 @@ public void deserialize(ByteBuffer byteBuffer) { versionInfo = new TNodeVersionInfo( ReadWriteIOUtils.readString(byteBuffer), ReadWriteIOUtils.readString(byteBuffer)); - if (byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer)) { - int size = ReadWriteIOUtils.readInt(byteBuffer); - Set capabilities = new HashSet<>(); - for (int i = 0; i < size; i++) { - capabilities.add(ReadWriteIOUtils.readShort(byteBuffer)); - } - versionInfo.setSupportedCQDurationEncodingVersions(capabilities); - } } else { versionInfo = new TNodeVersionInfo("Unknown", "Unknown"); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index 5bb628a82157e..cf49a3916bf2a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -78,7 +78,6 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -98,11 +97,6 @@ public class ConfigNode extends ServerCommandLine implements ConfigNodeMBean { private static final int INIT_NON_SEED_CONFIG_NODE_ID = -1; - private static TNodeVersionInfo currentNodeVersionInfo() { - return new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) - .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); - } - private static final String CONFIGURATION = "IoTDB configuration: {}"; private final String mbeanName = @@ -198,7 +192,9 @@ public void active() { TSStatus status = configManager .getNodeManager() - .updateConfigNodeIfNecessary(configNodeId, currentNodeVersionInfo()); + .updateConfigNodeIfNecessary( + configNodeId, + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { break; } else { @@ -237,7 +233,7 @@ public void active() { .getNodeManager() .applyConfigNode( CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(SEED_CONFIG_NODE_ID), - currentNodeVersionInfo()); + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); setUpMetricService(); // Notice: We always set up Seed-ConfigNode's RPC service lastly to ensure // that the external service is not provided until Seed-ConfigNode is fully initialized @@ -380,7 +376,7 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio configManager.getClusterParameters(), CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(INIT_NON_SEED_CONFIG_NODE_ID)); - req.setVersionInfo(currentNodeVersionInfo()); + req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); TEndPoint seedConfigNode = CONF.getSeedConfigNode(); if (seedConfigNode == null) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java deleted file mode 100644 index a4d183850f3f8..0000000000000 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.consensus.request.write.confignode; - -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; -import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; - -import org.apache.tsfile.utils.PublicBAOS; -import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.junit.Assert; -import org.junit.Test; - -import java.io.DataOutputStream; -import java.nio.ByteBuffer; -import java.util.Collections; - -public class UpdateVersionInfoPlanTest { - - @Test - public void capabilitiesRoundTrip() throws Exception { - TNodeVersionInfo versionInfo = - new TNodeVersionInfo("2.0.0", "build") - .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); - UpdateVersionInfoPlan plan = new UpdateVersionInfoPlan(versionInfo, 3); - UpdateVersionInfoPlan restored = - (UpdateVersionInfoPlan) ConfigPhysicalPlan.Factory.create(plan.serializeToByteBuffer()); - Assert.assertEquals(plan, restored); - Assert.assertTrue( - restored.getVersionInfo().getSupportedCQDurationEncodingVersions().contains((short) 1)); - } - - @Test - public void legacyPayloadWithoutCapabilityTailStillDeserializes() throws Exception { - try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); - DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { - ReadWriteIOUtils.write( - org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType.UpdateVersionInfo - .getPlanType(), - outputStream); - ReadWriteIOUtils.write(7, outputStream); - ReadWriteIOUtils.write("1.3.0", outputStream); - ReadWriteIOUtils.write("legacy", outputStream); - ByteBuffer buffer = - ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); - UpdateVersionInfoPlan restored = - (UpdateVersionInfoPlan) ConfigPhysicalPlan.Factory.create(buffer); - Assert.assertEquals(7, restored.getNodeId()); - Assert.assertEquals("1.3.0", restored.getVersionInfo().getVersion()); - Assert.assertFalse(restored.getVersionInfo().isSetSupportedCQDurationEncodingVersions()); - } - } -} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java index 71b990c75bfdd..50445fe25814a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java @@ -24,7 +24,6 @@ import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; -import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; @@ -86,44 +85,6 @@ public void newTokenShouldCancelPreviousLocallyScheduledTask() { } } - @Test - public void mixedCalendarAndFixedDurationsReachCapabilityValidation() { - ConfigManager configManager = Mockito.mock(ConfigManager.class); - NodeManager nodeManager = Mockito.mock(NodeManager.class); - Mockito.when(configManager.getNodeManager()).thenReturn(nodeManager); - Mockito.when(nodeManager.getNodeVersionInfo()).thenReturn(java.util.Collections.emptyMap()); - CQManager cqManager = new CQManager(configManager); - - TCreateCQReq req = - new TCreateCQReq( - "mixedDurationCq", - 0, - 0, - 0, - 0, - TimeoutPolicy.BLOCKED.getType(), - "select 1", - "create cq mixedDurationCq", - "UTC", - "root"); - req.setDurationEncodingVersion((short) 1); - req.setEveryDuration(new TCQDuration(0, 86_400_000)); - req.setStartOffsetDuration(new TCQDuration(1, 86_400_000)); - req.setEndOffsetDuration(new TCQDuration(0, 0)); - req.setBoundaryExplicit(true); - - try { - TSStatus status = cqManager.createCQ(req); - assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); - assertEquals( - org.apache.iotdb.confignode.i18n.ManagerMessages - .MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072, - status.getMessage()); - } finally { - cqManager.stopCQScheduler(); - } - } - @Test public void markerlessCreateCQIsRejectedAtRpcIngress() { ConfigManager configManager = Mockito.mock(ConfigManager.class); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java index 60434905b4180..f39ecd46d5c6a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java @@ -20,25 +20,16 @@ package org.apache.iotdb.confignode.manager.node; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; -import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; -import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; -import org.apache.iotdb.confignode.consensus.response.datanode.DataNodeRegisterResp; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; -import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.persistence.node.NodeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; -import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; -import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq; -import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartReq; -import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.consensus.IConsensus; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -66,8 +57,6 @@ public class NodeManagerTest { private IConsensus consensus; private ConsensusManager consensusManager; - private NodeInfo nodeInfo; - private CQManager cqManager; private NodeManager nodeManager; private int originalCnConnectionTimeout; private int originalTransferLeaderTimeout; @@ -84,12 +73,10 @@ public void setUp() { consensus = Mockito.mock(IConsensus.class); consensusManager = Mockito.mock(ConsensusManager.class); IManager configManager = Mockito.mock(IManager.class); - nodeInfo = Mockito.mock(NodeInfo.class); + NodeInfo nodeInfo = Mockito.mock(NodeInfo.class); LoadManager loadManager = Mockito.mock(LoadManager.class); - cqManager = Mockito.mock(CQManager.class); Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); - Mockito.when(configManager.getCQManager()).thenReturn(cqManager); Mockito.when(configManager.getLoadManager()).thenReturn(loadManager); Mockito.when(consensusManager.getConsensusImpl()).thenReturn(consensus); Mockito.when(consensusManager.getConsensusGroupId()) @@ -139,64 +126,6 @@ public void transferLeaderShouldTryAnotherCandidateAfterTransientFailure() throw peerCaptor.getAllValues().stream().map(Peer::getNodeId).collect(Collectors.toList())); } - @Test - public void configNodeWithoutCalendarDurationCapabilityIsRejectedWhileCalendarCQExists() { - TNodeVersionInfo unsupportedVersion = new TNodeVersionInfo("old", "old"); - TConfigNodeRegisterReq registerReq = - new TConfigNodeRegisterReq().setConfigNodeLocation(firstCandidate); - registerReq.setVersionInfo(unsupportedVersion); - - // The CQ manager is mocked in setUp; make the persisted metadata barrier active for this test. - Mockito.when(cqManager.hasCalendarDurationCQ()).thenReturn(true); - - TConfigNodeRegisterResp registerResp = nodeManager.registerConfigNode(registerReq); - Assert.assertEquals( - TSStatusCode.SEMANTIC_ERROR.getStatusCode(), registerResp.getStatus().getCode()); - Assert.assertEquals(-1, registerResp.getConfigNodeId()); - Assert.assertEquals( - TSStatusCode.SEMANTIC_ERROR.getStatusCode(), - nodeManager - .updateConfigNodeIfNecessary(firstCandidate.getConfigNodeId(), unsupportedVersion) - .getCode()); - Mockito.verify(nodeInfo, Mockito.never()).generateNextNodeId(); - } - - @Test - public void dataNodeWithoutCalendarDurationCapabilityIsRejectedWhileCalendarCQExists() { - TNodeVersionInfo unsupportedVersion = new TNodeVersionInfo("old", "old"); - TDataNodeLocation location = - new TDataNodeLocation( - 11, - new TEndPoint("127.0.0.1", 6667), - new TEndPoint("127.0.0.1", 10730), - new TEndPoint("127.0.0.1", 10740), - new TEndPoint("127.0.0.1", 10750), - new TEndPoint("127.0.0.1", 10760)); - TDataNodeConfiguration configuration = new TDataNodeConfiguration().setLocation(location); - TDataNodeRegisterReq registerReq = - new TDataNodeRegisterReq() - .setClusterName("cluster") - .setDataNodeConfiguration(configuration) - .setVersionInfo(unsupportedVersion); - - Mockito.when(cqManager.hasCalendarDurationCQ()).thenReturn(true); - - DataNodeRegisterResp registerResp = - (DataNodeRegisterResp) nodeManager.registerDataNode(registerReq); - Assert.assertEquals( - TSStatusCode.SEMANTIC_ERROR.getStatusCode(), registerResp.getStatus().getCode()); - Mockito.verify(nodeInfo, Mockito.never()).generateNextNodeId(); - - TDataNodeRestartReq restartReq = - new TDataNodeRestartReq() - .setClusterName("cluster") - .setDataNodeConfiguration(configuration) - .setVersionInfo(unsupportedVersion); - Assert.assertEquals( - TSStatusCode.SEMANTIC_ERROR.getStatusCode(), - nodeManager.updateDataNodeIfNecessary(restartReq).getStatus().getCode()); - } - @Test public void transferLeaderShouldRedirectToActualLeaderAfterFailedResponse() throws Exception { Mockito.when(consensusManager.getLeaderLocation()) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java index 3b2a74634b5bd..e3ced5b069428 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java @@ -26,10 +26,8 @@ import org.apache.iotdb.common.rpc.thrift.TNodeResource; import org.apache.iotdb.commons.exception.StartupException; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; -import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.persistence.node.NodeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; @@ -40,7 +38,6 @@ import java.io.File; import java.io.IOException; -import java.util.Collections; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -76,35 +73,6 @@ public void testSnapshot() throws TException, IOException { Assert.assertEquals(nodeInfo, nodeInfo1); } - @Test - public void testSnapshotPreservesDurationEncodingCapabilities() throws TException, IOException { - File capabilitySnapshotDir = new File(BASE_OUTPUT_PATH, "snapshot-node-capabilities"); - if (capabilitySnapshotDir.exists()) { - FileUtils.deleteDirectory(capabilitySnapshotDir); - } - capabilitySnapshotDir.mkdirs(); - try { - registerConfigNodes(); - registerDataNodes(); - TNodeVersionInfo versionInfo = - new TNodeVersionInfo("2.0.0", "build") - .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); - nodeInfo.updateVersionInfo(new UpdateVersionInfoPlan(versionInfo, 10000)); - Assert.assertTrue(nodeInfo.processTakeSnapshot(capabilitySnapshotDir)); - - NodeInfo restored = new NodeInfo(); - restored.processLoadSnapshot(capabilitySnapshotDir); - Assert.assertEquals(nodeInfo, restored); - Assert.assertTrue( - restored - .getVersionInfo(10000) - .getSupportedCQDurationEncodingVersions() - .contains((short) 1)); - } finally { - FileUtils.deleteDirectory(capabilitySnapshotDir); - } - } - private void registerConfigNodes() { for (int i = 0; i < 3; i++) { ApplyConfigNodePlan applyConfigNodePlan = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java index 835e21dca9c00..682b478134664 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java @@ -31,30 +31,11 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collections; public class AddConfigNodeProcedureTest { @Test public void serDeTest() throws IOException { - AddConfigNodeProcedure procedure0 = - new AddConfigNodeProcedure( - new TConfigNodeLocation( - 0, new TEndPoint("127.0.0.1", 10710), new TEndPoint("0.0.0.0", 10720)), - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) - .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); - - try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); - DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { - procedure0.serialize(outputStream); - ByteBuffer buffer = - ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); - Assert.assertEquals(procedure0, ProcedureFactory.getInstance().create(buffer)); - } - } - - @Test - public void deserializeLegacyPayloadWithoutCapabilities() throws IOException { AddConfigNodeProcedure procedure0 = new AddConfigNodeProcedure( new TConfigNodeLocation( @@ -65,7 +46,7 @@ public void deserializeLegacyPayloadWithoutCapabilities() throws IOException { DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { procedure0.serialize(outputStream); ByteBuffer buffer = - ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size() - 1); + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); Assert.assertEquals(procedure0, ProcedureFactory.getInstance().create(buffer)); } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index f3ea7245a5ee8..6e5e3ba8207a0 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3888,7 +3888,4 @@ private DataNodeQueryMessages() {} "No more DeviceEntry records are available"; public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "Only InMemoryDeviceEntryDataSet supports get inline device entries"; - - public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3 = - "CQ calendar duration cannot be created until all cluster nodes support duration encoding version 1"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index efea296f6d57b..c8f41c50f8d44 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4647,7 +4647,4 @@ private DataNodeQueryMessages() {} "没有更多可用的 DeviceEntry 记录"; public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "只有 InMemoryDeviceEntryDataSet 支持获取内存中的设备条目"; - - public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3 = - "集群所有节点支持 duration encoding version 1 后才能创建 CQ 日历 duration"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index f7d921b29df7e..6d67dc912bb91 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -157,7 +157,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetUDFTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetUdfTableReq; import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq; -import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TPipeConfigTransferReq; import org.apache.iotdb.confignode.rpc.thrift.TPipeConfigTransferResp; import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq; @@ -4187,13 +4186,6 @@ public SettableFuture createContinuousQuery( createContinuousQueryStatement.getEveryDuration().monthDuration != 0 || createContinuousQueryStatement.getStartTimeOffsetDuration().monthDuration != 0 || createContinuousQueryStatement.getEndTimeOffsetDuration().monthDuration != 0; - if (hasCalendarDuration && !allClusterNodesSupportCQDurationEncoding(client.showCluster())) { - future.setException( - new SemanticException( - DataNodeQueryMessages - .MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3)); - return future; - } // The legacy fields are still required on the wire. If any structured component contains a // calendar month, none of the legacy fields has a valid representation and all three use the // zero sentinel. This prevents an old reader from interpreting a mixed request as a partially @@ -4242,35 +4234,6 @@ public SettableFuture createContinuousQuery( return future; } - private boolean allClusterNodesSupportCQDurationEncoding(TShowClusterResp response) { - if (response == null - || response.getConfigNodeList() == null - || response.getDataNodeList() == null - || response.getNodeVersionInfo() == null) { - return false; - } - if (response.getConfigNodeList().isEmpty() && response.getDataNodeList().isEmpty()) { - return false; - } - for (TConfigNodeLocation node : response.getConfigNodeList()) { - if (!supportsCQDurationEncoding(response.getNodeVersionInfo().get(node.getConfigNodeId()))) { - return false; - } - } - for (TDataNodeLocation node : response.getDataNodeList()) { - if (!supportsCQDurationEncoding(response.getNodeVersionInfo().get(node.getDataNodeId()))) { - return false; - } - } - return true; - } - - private boolean supportsCQDurationEncoding(TNodeVersionInfo info) { - return info != null - && info.isSetSupportedCQDurationEncodingVersions() - && info.getSupportedCQDurationEncodingVersions().contains((short) 1); - } - @Override public SettableFuture dropContinuousQuery(final String cqId) { final SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 1e1659c74bd1f..5ce282db7a701 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -561,9 +561,7 @@ private void sendRegisterRequestToConfigNode(boolean isPreCheck) req.setPreCheck(isPreCheck); req.setDataNodeConfiguration(generateDataNodeConfiguration()); req.setClusterName(config.getClusterName()); - req.setVersionInfo( - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) - .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); + req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); TDataNodeRegisterResp dataNodeRegisterResp = null; while (retry > 0) { try (ConfigNodeClient configNodeClient = @@ -739,9 +737,7 @@ private void sendRestartRequestToConfigNode() throws StartupException { req.setClusterName( config.getClusterName() == null ? DEFAULT_CLUSTER_NAME : config.getClusterName()); req.setDataNodeConfiguration(generateDataNodeConfiguration()); - req.setVersionInfo( - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) - .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); + req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); req.setClusterId(config.getClusterId()); TDataNodeRestartResp dataNodeRestartResp = null; while (retry > 0) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java deleted file mode 100644 index c8065a975ca97..0000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.execution.config.executor; - -import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; -import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; - -import org.junit.Assert; -import org.junit.Test; - -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public class ClusterConfigTaskExecutorCQTest { - - @Test - public void mixedVersionClusterIsRejectedAtDataNodeIngress() throws Exception { - Method method = - ClusterConfigTaskExecutor.class.getDeclaredMethod( - "allClusterNodesSupportCQDurationEncoding", TShowClusterResp.class); - method.setAccessible(true); - ClusterConfigTaskExecutor executor = ClusterConfigTaskExecutor.getInstance(); - - Assert.assertFalse((Boolean) method.invoke(executor, new TShowClusterResp())); - Assert.assertFalse((Boolean) method.invoke(executor, (Object) null)); - - TConfigNodeLocation configNode = - new TConfigNodeLocation( - 1, new TEndPoint("127.0.0.1", 10710), new TEndPoint("127.0.0.1", 10720)); - TDataNodeLocation dataNode = - new TDataNodeLocation( - 11, - new TEndPoint("127.0.0.1", 6667), - new TEndPoint("127.0.0.1", 10730), - new TEndPoint("127.0.0.1", 10740), - new TEndPoint("127.0.0.1", 10750), - new TEndPoint("127.0.0.1", 10760)); - TNodeVersionInfo supported = - new TNodeVersionInfo("2.0.0", "new") - .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); - TNodeVersionInfo unsupported = new TNodeVersionInfo("1.3.0", "old"); - - TShowClusterResp mixed = new TShowClusterResp(); - mixed.setConfigNodeList(Collections.singletonList(configNode)); - mixed.setDataNodeList(Collections.singletonList(dataNode)); - Map versions = new HashMap<>(); - versions.put(configNode.getConfigNodeId(), supported); - versions.put(dataNode.getDataNodeId(), unsupported); - mixed.setNodeVersionInfo(versions); - Assert.assertFalse((Boolean) method.invoke(executor, mixed)); - - versions.put(dataNode.getDataNodeId(), supported); - Assert.assertTrue((Boolean) method.invoke(executor, mixed)); - } -} diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 8ce6213a44004..a74d8dcb312a7 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -653,8 +653,6 @@ struct TGetClusterIdResp { struct TNodeVersionInfo { 1: required string version; 2: required string buildInfo; - // Capabilities are optional so older nodes can still register. Missing means unsupported. - 3: optional set supportedCQDurationEncodingVersions; } struct TNodeActivateInfo { From 76e59ce8eb459a2e35475885ae286db9c5e61034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Wed, 23 Sep 2026 01:15:47 +0800 Subject: [PATCH 07/10] Revert "Remove out-of-scope mixed-version cluster capability negotiation from CQ calendar durations" This reverts commit 857388d8ab8ed4df72f1f33c9f7b1b482076c23e. --- .../confignode/i18n/ManagerMessages.java | 2 + .../confignode/i18n/ManagerMessages.java | 2 + .../confignode/UpdateVersionInfoPlan.java | 19 +++++ .../confignode/manager/cq/CQManager.java | 67 ++++++++++++++++ .../confignode/manager/node/NodeManager.java | 34 ++++++++ .../confignode/persistence/cq/CQInfo.java | 6 ++ .../confignode/persistence/node/NodeInfo.java | 39 ++++++++++ .../impl/node/AddConfigNodeProcedure.java | 19 +++++ .../iotdb/confignode/service/ConfigNode.java | 14 ++-- .../confignode/UpdateVersionInfoPlanTest.java | 69 +++++++++++++++++ .../iotdb/confignode/cq/CQManagerTest.java | 39 ++++++++++ .../manager/node/NodeManagerTest.java | 73 +++++++++++++++++- .../confignode/persistence/NodeInfoTest.java | 32 ++++++++ .../impl/node/AddConfigNodeProcedureTest.java | 21 ++++- .../iotdb/db/i18n/DataNodeQueryMessages.java | 3 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 3 + .../executor/ClusterConfigTaskExecutor.java | 37 +++++++++ .../org/apache/iotdb/db/service/DataNode.java | 8 +- .../ClusterConfigTaskExecutorCQTest.java | 77 +++++++++++++++++++ .../src/main/thrift/confignode.thrift | 2 + 20 files changed, 557 insertions(+), 9 deletions(-) create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 89b981cefebeb..885dc8339f75c 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -723,6 +723,8 @@ private ManagerMessages() {} public static final String MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67 = "CQ legacy duration fields conflict with structured duration fields"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072 = + "CQ calendar duration requires all cluster nodes to support duration encoding version 1"; public static final String MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A = "CQ duration encoding marker is required for new requests"; public static final String MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C = diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 7aed04c396249..b56336a27fc73 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -702,6 +702,8 @@ private ManagerMessages() {} public static final String MESSAGE_CQ_LEGACY_DURATION_FIELDS_CONFLICT_WITH_STRUCTURED_DURATION_FIELDS_4D6C6D67 = "CQ legacy duration fields 与结构化 duration 字段冲突"; + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072 = + "CQ 日历 duration 要求集群所有节点支持 duration encoding version 1"; public static final String MESSAGE_CQ_DURATION_ENCODING_MARKER_REQUIRED_9035980A = "新建 CQ 请求必须包含 duration encoding marker"; public static final String MESSAGE_CQ_DOES_NOT_HAVE_OCCURRENCE_INDEX_METADATA_929A7F0C = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java index 62f2cc71e6926..d2894392e26bc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlan.java @@ -59,6 +59,14 @@ protected void serializeImpl(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(nodeId, stream); ReadWriteIOUtils.write(versionInfo.getVersion(), stream); ReadWriteIOUtils.write(versionInfo.getBuildInfo(), stream); + // Optional tail keeps this plan readable by pre-calendar-duration ConfigNodes. + ReadWriteIOUtils.write(versionInfo.isSetSupportedCQDurationEncodingVersions(), stream); + if (versionInfo.isSetSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(versionInfo.getSupportedCQDurationEncodingVersions().size(), stream); + for (short version : versionInfo.getSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(version, stream); + } + } } @Override @@ -67,6 +75,17 @@ protected void deserializeImpl(ByteBuffer buffer) { versionInfo = new TNodeVersionInfo( ReadWriteIOUtils.readString(buffer), ReadWriteIOUtils.readString(buffer)); + if (buffer.hasRemaining()) { + boolean hasCapabilities = ReadWriteIOUtils.readBool(buffer); + if (hasCapabilities) { + int size = ReadWriteIOUtils.readInt(buffer); + java.util.Set capabilities = new java.util.HashSet<>(); + for (int i = 0; i < size; i++) { + capabilities.add(ReadWriteIOUtils.readShort(buffer)); + } + versionInfo.setSupportedCQDurationEncodingVersions(capabilities); + } + } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java index 9ef6009f1e2b2..0159cc3af3be3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java @@ -34,6 +34,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowCQResp; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -144,6 +145,11 @@ private TSStatus validateDurationEncoding(TCreateCQReq req) { if (semanticValidation != null) { return semanticValidation; } + if (hasCalendarDuration && !allClusterNodesSupportDurationEncodingV1()) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages.MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072); + } return null; } @@ -191,6 +197,67 @@ private static boolean dominates(TCQDuration left, TCQDuration right, boolean st || left.getNonMonthDuration() != right.getNonMonthDuration()); } + /** Returns true when any persisted CQ requires the structured calendar-duration reader. */ + public boolean hasCalendarDurationCQ() { + try { + DataSet response = configManager.getConsensusManager().read(new ShowCQPlan()); + if (!(response instanceof ShowCQResp)) { + // Do not allow a node with unknown metadata to join while the reader barrier is active. + return true; + } + if (((ShowCQResp) response).getCqList() == null) { + // A malformed response is just as unsafe as an unavailable response for this barrier. + return true; + } + return ((ShowCQResp) response) + .getCqList().stream().anyMatch(CQInfo.CQEntry::hasCalendarDuration); + } catch (ConsensusException e) { + // A failed metadata read must fail closed: an old reader must never be admitted blindly. + LOGGER.warn(ManagerMessages.UNEXPECTED_ERROR_HAPPENED_WHILE_FETCHING_CQ_LIST, e); + return true; + } + } + + private boolean allClusterNodesSupportDurationEncodingV1() { + java.util.Map versionInfo = + configManager.getNodeManager().getNodeVersionInfo(); + if (versionInfo == null || versionInfo.isEmpty()) { + return false; + } + boolean hasRegisteredNode = false; + // Check every registered node explicitly. A missing heartbeat/version entry must not allow a + // calendar CQ to be created during a rolling upgrade. + List configNodes = + configManager.getNodeManager().getRegisteredConfigNodes(); + List dataNodes = + configManager.getNodeManager().getRegisteredDataNodes(); + if (configNodes == null || dataNodes == null) { + return false; + } + for (org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation node : configNodes) { + hasRegisteredNode = true; + if (!supportsDurationEncodingV1(versionInfo.get(node.getConfigNodeId()))) { + return false; + } + } + for (org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration node : dataNodes) { + hasRegisteredNode = true; + if (!supportsDurationEncodingV1(versionInfo.get(node.getLocation().getDataNodeId()))) { + return false; + } + } + return hasRegisteredNode; + } + + private boolean supportsDurationEncodingV1(TNodeVersionInfo info) { + if (info == null + || !info.isSetSupportedCQDurationEncodingVersions() + || !info.getSupportedCQDurationEncodingVersions().contains((short) 1)) { + return false; + } + return true; + } + public TSStatus dropCQ(TDropCQReq req) { lock.readLock().lock(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 598c47e692d26..35a51b0929c49 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -331,6 +331,11 @@ private TRuntimeConfiguration getRuntimeConfiguration(int dataNodeId) { public DataSet registerDataNode(TDataNodeRegisterReq req) { DataNodeRegisterResp resp = new DataNodeRegisterResp(); resp.setConfigNodeList(getRegisteredConfigNodes()); + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + resp.setStatus(capabilityStatus); + return resp; + } // Create a new DataNodeHeartbeatCache and force update NodeStatus int dataNodeId = nodeInfo.generateNextNodeId(); @@ -373,6 +378,11 @@ public DataSet registerDataNode(TDataNodeRegisterReq req) { public TDataNodeRestartResp updateDataNodeIfNecessary(TDataNodeRestartReq req) { TDataNodeRestartResp resp = new TDataNodeRestartResp(); resp.setConfigNodeList(getRegisteredConfigNodes()); + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + resp.setStatus(capabilityStatus); + return resp; + } final String clusterId = configManager .getClusterManager() @@ -473,6 +483,10 @@ public DataSet removeDataNode(RemoveDataNodePlan removeDataNodePlan) { } public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { + TSStatus capabilityStatus = validateDurationCapability(req.getVersionInfo()); + if (capabilityStatus != null) { + return new TConfigNodeRegisterResp().setStatus(capabilityStatus).setConfigNodeId(-1); + } int nodeId = nodeInfo.generateNextNodeId(); req.getConfigNodeLocation().setConfigNodeId(nodeId); configManager.getProcedureManager().addConfigNode(req); @@ -482,6 +496,10 @@ public TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req) { } public TSStatus updateConfigNodeIfNecessary(int configNodeId, TNodeVersionInfo versionInfo) { + TSStatus capabilityStatus = validateDurationCapability(versionInfo); + if (capabilityStatus != null) { + return capabilityStatus; + } TNodeVersionInfo recordVersionInfo = nodeInfo.getVersionInfo(configNodeId); if (!recordVersionInfo.equals(versionInfo)) { // Update versionInfo when modified during restart @@ -496,6 +514,22 @@ public TSStatus updateConfigNodeIfNecessary(int configNodeId, TNodeVersionInfo v return ClusterNodeStartUtils.ACCEPT_NODE_RESTART; } + private TSStatus validateDurationCapability(TNodeVersionInfo versionInfo) { + if (!supportsDurationEncodingV1(versionInfo) + && configManager.getCQManager().hasCalendarDurationCQ()) { + return new TSStatus(TSStatusCode.SEMANTIC_ERROR.getStatusCode()) + .setMessage( + ManagerMessages.MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072); + } + return null; + } + + private static boolean supportsDurationEncodingV1(TNodeVersionInfo versionInfo) { + return versionInfo != null + && versionInfo.isSetSupportedCQDurationEncodingVersions() + && versionInfo.getSupportedCQDurationEncodingVersions().contains((short) 1); + } + public List getRegisteredAINodeInfoList() { List aiNodeInfoList = new ArrayList<>(); for (TAINodeConfiguration aiNodeConfiguration : getRegisteredAINodes()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java index 8596df0ddcf7b..269e9d38f6790 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java @@ -687,6 +687,12 @@ public boolean isBoundaryExplicit() { return boundaryExplicit; } + public boolean hasCalendarDuration() { + return everyDuration.monthDuration != 0 + || startTimeOffsetDuration.monthDuration != 0 + || endTimeOffsetDuration.monthDuration != 0; + } + private static org.apache.tsfile.utils.TimeDuration durationFromReq( TCreateCQReq req, TCQDuration d, long legacy) { if (req.isSetDurationEncodingVersion() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java index d98a05c8a8cdc..a6878b963cbed 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java @@ -68,6 +68,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -722,6 +723,19 @@ private void serializeVersionInfo(OutputStream outputStream) throws IOException ReadWriteIOUtils.write(entry.getValue().getVersion(), outputStream); ReadWriteIOUtils.write(entry.getValue().getBuildInfo(), outputStream); } + // Optional tail; legacy snapshots end after the two strings above. + ReadWriteIOUtils.write(0x43515631, outputStream); + ReadWriteIOUtils.write(nodeVersionInfo.size(), outputStream); + for (Entry entry : nodeVersionInfo.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + Set capabilities = entry.getValue().getSupportedCQDurationEncodingVersions(); + ReadWriteIOUtils.write(capabilities == null ? 0 : capabilities.size(), outputStream); + if (capabilities != null) { + for (short capability : capabilities) { + ReadWriteIOUtils.write(capability, outputStream); + } + } + } } @Override @@ -827,6 +841,31 @@ private void deserializeBuildInfo(InputStream inputStream) throws IOException { nodeVersionInfo.put(nodeId, new TNodeVersionInfo(version, buildInfo)); size--; } + if (inputStream.available() >= Integer.BYTES) { + inputStream.mark(Integer.BYTES); + int marker = ReadWriteIOUtils.readInt(inputStream); + if (marker == 0x43515631) { + int capabilitySize = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < capabilitySize; i++) { + int nodeId = ReadWriteIOUtils.readInt(inputStream); + int sizeOfCapabilities = ReadWriteIOUtils.readInt(inputStream); + TNodeVersionInfo info = nodeVersionInfo.get(nodeId); + if (info != null) { + Set capabilities = new java.util.HashSet<>(); + for (int j = 0; j < sizeOfCapabilities; j++) { + capabilities.add(ReadWriteIOUtils.readShort(inputStream)); + } + info.setSupportedCQDurationEncodingVersions(capabilities); + } else { + for (int j = 0; j < sizeOfCapabilities; j++) { + ReadWriteIOUtils.readShort(inputStream); + } + } + } + } else { + inputStream.reset(); + } + } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java index a3a5ffb41de4e..ad6f7233bf1b4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedure.java @@ -36,7 +36,9 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; /** add config node procedure */ public class AddConfigNodeProcedure extends AbstractNodeProcedure { @@ -156,6 +158,15 @@ public void serialize(DataOutputStream stream) throws IOException { ThriftConfigNodeSerDeUtils.serializeTConfigNodeLocation(tConfigNodeLocation, stream); ReadWriteIOUtils.write(versionInfo.getVersion(), stream); ReadWriteIOUtils.write(versionInfo.getBuildInfo(), stream); + // Optional tail preserves the joining node's capability across procedure recovery while old + // payloads remain readable because the tail is absent in their serialized form. + ReadWriteIOUtils.write(versionInfo.isSetSupportedCQDurationEncodingVersions(), stream); + if (versionInfo.isSetSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(versionInfo.getSupportedCQDurationEncodingVersions().size(), stream); + for (short version : versionInfo.getSupportedCQDurationEncodingVersions()) { + ReadWriteIOUtils.write(version, stream); + } + } } @Override @@ -169,6 +180,14 @@ public void deserialize(ByteBuffer byteBuffer) { versionInfo = new TNodeVersionInfo( ReadWriteIOUtils.readString(byteBuffer), ReadWriteIOUtils.readString(byteBuffer)); + if (byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer)) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + Set capabilities = new HashSet<>(); + for (int i = 0; i < size; i++) { + capabilities.add(ReadWriteIOUtils.readShort(byteBuffer)); + } + versionInfo.setSupportedCQDurationEncodingVersions(capabilities); + } } else { versionInfo = new TNodeVersionInfo("Unknown", "Unknown"); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java index cf49a3916bf2a..5bb628a82157e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java @@ -78,6 +78,7 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -97,6 +98,11 @@ public class ConfigNode extends ServerCommandLine implements ConfigNodeMBean { private static final int INIT_NON_SEED_CONFIG_NODE_ID = -1; + private static TNodeVersionInfo currentNodeVersionInfo() { + return new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + } + private static final String CONFIGURATION = "IoTDB configuration: {}"; private final String mbeanName = @@ -192,9 +198,7 @@ public void active() { TSStatus status = configManager .getNodeManager() - .updateConfigNodeIfNecessary( - configNodeId, - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + .updateConfigNodeIfNecessary(configNodeId, currentNodeVersionInfo()); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { break; } else { @@ -233,7 +237,7 @@ public void active() { .getNodeManager() .applyConfigNode( CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(SEED_CONFIG_NODE_ID), - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + currentNodeVersionInfo()); setUpMetricService(); // Notice: We always set up Seed-ConfigNode's RPC service lastly to ensure // that the external service is not provided until Seed-ConfigNode is fully initialized @@ -376,7 +380,7 @@ private void sendRegisterConfigNodeRequest() throws StartupException, IOExceptio configManager.getClusterParameters(), CONF.generateLocalConfigNodeLocationWithSpecifiedNodeId(INIT_NON_SEED_CONFIG_NODE_ID)); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo(currentNodeVersionInfo()); TEndPoint seedConfigNode = CONF.getSeedConfigNode(); if (seedConfigNode == null) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java new file mode 100644 index 0000000000000..a4d183850f3f8 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateVersionInfoPlanTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.confignode; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataOutputStream; +import java.nio.ByteBuffer; +import java.util.Collections; + +public class UpdateVersionInfoPlanTest { + + @Test + public void capabilitiesRoundTrip() throws Exception { + TNodeVersionInfo versionInfo = + new TNodeVersionInfo("2.0.0", "build") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + UpdateVersionInfoPlan plan = new UpdateVersionInfoPlan(versionInfo, 3); + UpdateVersionInfoPlan restored = + (UpdateVersionInfoPlan) ConfigPhysicalPlan.Factory.create(plan.serializeToByteBuffer()); + Assert.assertEquals(plan, restored); + Assert.assertTrue( + restored.getVersionInfo().getSupportedCQDurationEncodingVersions().contains((short) 1)); + } + + @Test + public void legacyPayloadWithoutCapabilityTailStillDeserializes() throws Exception { + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write( + org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType.UpdateVersionInfo + .getPlanType(), + outputStream); + ReadWriteIOUtils.write(7, outputStream); + ReadWriteIOUtils.write("1.3.0", outputStream); + ReadWriteIOUtils.write("legacy", outputStream); + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + UpdateVersionInfoPlan restored = + (UpdateVersionInfoPlan) ConfigPhysicalPlan.Factory.create(buffer); + Assert.assertEquals(7, restored.getNodeId()); + Assert.assertEquals("1.3.0", restored.getVersionInfo().getVersion()); + Assert.assertFalse(restored.getVersionInfo().isSetSupportedCQDurationEncodingVersions()); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java index 50445fe25814a..71b990c75bfdd 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; @@ -85,6 +86,44 @@ public void newTokenShouldCancelPreviousLocallyScheduledTask() { } } + @Test + public void mixedCalendarAndFixedDurationsReachCapabilityValidation() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + NodeManager nodeManager = Mockito.mock(NodeManager.class); + Mockito.when(configManager.getNodeManager()).thenReturn(nodeManager); + Mockito.when(nodeManager.getNodeVersionInfo()).thenReturn(java.util.Collections.emptyMap()); + CQManager cqManager = new CQManager(configManager); + + TCreateCQReq req = + new TCreateCQReq( + "mixedDurationCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq mixedDurationCq", + "UTC", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(0, 86_400_000)); + req.setStartOffsetDuration(new TCQDuration(1, 86_400_000)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(true); + + try { + TSStatus status = cqManager.createCQ(req); + assertEquals(TSStatusCode.SEMANTIC_ERROR.getStatusCode(), status.getCode()); + assertEquals( + org.apache.iotdb.confignode.i18n.ManagerMessages + .MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_49534072, + status.getMessage()); + } finally { + cqManager.stopCQScheduler(); + } + } + @Test public void markerlessCreateCQIsRejectedAtRpcIngress() { ConfigManager configManager = Mockito.mock(ConfigManager.class); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java index f39ecd46d5c6a..60434905b4180 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerTest.java @@ -20,16 +20,25 @@ package org.apache.iotdb.confignode.manager.node; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; +import org.apache.iotdb.confignode.consensus.response.datanode.DataNodeRegisterResp; import org.apache.iotdb.confignode.manager.IManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; +import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.consensus.IConsensus; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -57,6 +66,8 @@ public class NodeManagerTest { private IConsensus consensus; private ConsensusManager consensusManager; + private NodeInfo nodeInfo; + private CQManager cqManager; private NodeManager nodeManager; private int originalCnConnectionTimeout; private int originalTransferLeaderTimeout; @@ -73,10 +84,12 @@ public void setUp() { consensus = Mockito.mock(IConsensus.class); consensusManager = Mockito.mock(ConsensusManager.class); IManager configManager = Mockito.mock(IManager.class); - NodeInfo nodeInfo = Mockito.mock(NodeInfo.class); + nodeInfo = Mockito.mock(NodeInfo.class); LoadManager loadManager = Mockito.mock(LoadManager.class); + cqManager = Mockito.mock(CQManager.class); Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); Mockito.when(configManager.getLoadManager()).thenReturn(loadManager); Mockito.when(consensusManager.getConsensusImpl()).thenReturn(consensus); Mockito.when(consensusManager.getConsensusGroupId()) @@ -126,6 +139,64 @@ public void transferLeaderShouldTryAnotherCandidateAfterTransientFailure() throw peerCaptor.getAllValues().stream().map(Peer::getNodeId).collect(Collectors.toList())); } + @Test + public void configNodeWithoutCalendarDurationCapabilityIsRejectedWhileCalendarCQExists() { + TNodeVersionInfo unsupportedVersion = new TNodeVersionInfo("old", "old"); + TConfigNodeRegisterReq registerReq = + new TConfigNodeRegisterReq().setConfigNodeLocation(firstCandidate); + registerReq.setVersionInfo(unsupportedVersion); + + // The CQ manager is mocked in setUp; make the persisted metadata barrier active for this test. + Mockito.when(cqManager.hasCalendarDurationCQ()).thenReturn(true); + + TConfigNodeRegisterResp registerResp = nodeManager.registerConfigNode(registerReq); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), registerResp.getStatus().getCode()); + Assert.assertEquals(-1, registerResp.getConfigNodeId()); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + nodeManager + .updateConfigNodeIfNecessary(firstCandidate.getConfigNodeId(), unsupportedVersion) + .getCode()); + Mockito.verify(nodeInfo, Mockito.never()).generateNextNodeId(); + } + + @Test + public void dataNodeWithoutCalendarDurationCapabilityIsRejectedWhileCalendarCQExists() { + TNodeVersionInfo unsupportedVersion = new TNodeVersionInfo("old", "old"); + TDataNodeLocation location = + new TDataNodeLocation( + 11, + new TEndPoint("127.0.0.1", 6667), + new TEndPoint("127.0.0.1", 10730), + new TEndPoint("127.0.0.1", 10740), + new TEndPoint("127.0.0.1", 10750), + new TEndPoint("127.0.0.1", 10760)); + TDataNodeConfiguration configuration = new TDataNodeConfiguration().setLocation(location); + TDataNodeRegisterReq registerReq = + new TDataNodeRegisterReq() + .setClusterName("cluster") + .setDataNodeConfiguration(configuration) + .setVersionInfo(unsupportedVersion); + + Mockito.when(cqManager.hasCalendarDurationCQ()).thenReturn(true); + + DataNodeRegisterResp registerResp = + (DataNodeRegisterResp) nodeManager.registerDataNode(registerReq); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), registerResp.getStatus().getCode()); + Mockito.verify(nodeInfo, Mockito.never()).generateNextNodeId(); + + TDataNodeRestartReq restartReq = + new TDataNodeRestartReq() + .setClusterName("cluster") + .setDataNodeConfiguration(configuration) + .setVersionInfo(unsupportedVersion); + Assert.assertEquals( + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + nodeManager.updateDataNodeIfNecessary(restartReq).getStatus().getCode()); + } + @Test public void transferLeaderShouldRedirectToActualLeaderAfterFailedResponse() throws Exception { Mockito.when(consensusManager.getLeaderLocation()) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java index e3ced5b069428..3b2a74634b5bd 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java @@ -26,8 +26,10 @@ import org.apache.iotdb.common.rpc.thrift.TNodeResource; import org.apache.iotdb.commons.exception.StartupException; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.persistence.node.NodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; @@ -38,6 +40,7 @@ import java.io.File; import java.io.IOException; +import java.util.Collections; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -73,6 +76,35 @@ public void testSnapshot() throws TException, IOException { Assert.assertEquals(nodeInfo, nodeInfo1); } + @Test + public void testSnapshotPreservesDurationEncodingCapabilities() throws TException, IOException { + File capabilitySnapshotDir = new File(BASE_OUTPUT_PATH, "snapshot-node-capabilities"); + if (capabilitySnapshotDir.exists()) { + FileUtils.deleteDirectory(capabilitySnapshotDir); + } + capabilitySnapshotDir.mkdirs(); + try { + registerConfigNodes(); + registerDataNodes(); + TNodeVersionInfo versionInfo = + new TNodeVersionInfo("2.0.0", "build") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + nodeInfo.updateVersionInfo(new UpdateVersionInfoPlan(versionInfo, 10000)); + Assert.assertTrue(nodeInfo.processTakeSnapshot(capabilitySnapshotDir)); + + NodeInfo restored = new NodeInfo(); + restored.processLoadSnapshot(capabilitySnapshotDir); + Assert.assertEquals(nodeInfo, restored); + Assert.assertTrue( + restored + .getVersionInfo(10000) + .getSupportedCQDurationEncodingVersions() + .contains((short) 1)); + } finally { + FileUtils.deleteDirectory(capabilitySnapshotDir); + } + } + private void registerConfigNodes() { for (int i = 0; i < 3; i++) { ApplyConfigNodePlan applyConfigNodePlan = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java index 682b478134664..835e21dca9c00 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/AddConfigNodeProcedureTest.java @@ -31,6 +31,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; public class AddConfigNodeProcedureTest { @@ -40,7 +41,8 @@ public void serDeTest() throws IOException { new AddConfigNodeProcedure( new TConfigNodeLocation( 0, new TEndPoint("127.0.0.1", 10710), new TEndPoint("0.0.0.0", 10720)), - new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { @@ -50,4 +52,21 @@ public void serDeTest() throws IOException { Assert.assertEquals(procedure0, ProcedureFactory.getInstance().create(buffer)); } } + + @Test + public void deserializeLegacyPayloadWithoutCapabilities() throws IOException { + AddConfigNodeProcedure procedure0 = + new AddConfigNodeProcedure( + new TConfigNodeLocation( + 0, new TEndPoint("127.0.0.1", 10710), new TEndPoint("0.0.0.0", 10720)), + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + procedure0.serialize(outputStream); + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size() - 1); + Assert.assertEquals(procedure0, ProcedureFactory.getInstance().create(buffer)); + } + } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 6e5e3ba8207a0..f3ea7245a5ee8 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3888,4 +3888,7 @@ private DataNodeQueryMessages() {} "No more DeviceEntry records are available"; public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "Only InMemoryDeviceEntryDataSet supports get inline device entries"; + + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3 = + "CQ calendar duration cannot be created until all cluster nodes support duration encoding version 1"; } diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index c8f41c50f8d44..efea296f6d57b 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -4647,4 +4647,7 @@ private DataNodeQueryMessages() {} "没有更多可用的 DeviceEntry 记录"; public static final String EXCEPTION_ONLY_INMEMORYDEVICEENTRYDATASET_SUPPORTS_GET_INLINE_DEVICE_ENTRIES_07A52CAB = "只有 InMemoryDeviceEntryDataSet 支持获取内存中的设备条目"; + + public static final String MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3 = + "集群所有节点支持 duration encoding version 1 后才能创建 CQ 日历 duration"; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 6d67dc912bb91..f7d921b29df7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -157,6 +157,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetUDFTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetUdfTableReq; import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; import org.apache.iotdb.confignode.rpc.thrift.TPipeConfigTransferReq; import org.apache.iotdb.confignode.rpc.thrift.TPipeConfigTransferResp; import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq; @@ -4186,6 +4187,13 @@ public SettableFuture createContinuousQuery( createContinuousQueryStatement.getEveryDuration().monthDuration != 0 || createContinuousQueryStatement.getStartTimeOffsetDuration().monthDuration != 0 || createContinuousQueryStatement.getEndTimeOffsetDuration().monthDuration != 0; + if (hasCalendarDuration && !allClusterNodesSupportCQDurationEncoding(client.showCluster())) { + future.setException( + new SemanticException( + DataNodeQueryMessages + .MESSAGE_CQ_CALENDAR_DURATION_REQUIRES_ALL_NODES_SUPPORT_AC724DE3)); + return future; + } // The legacy fields are still required on the wire. If any structured component contains a // calendar month, none of the legacy fields has a valid representation and all three use the // zero sentinel. This prevents an old reader from interpreting a mixed request as a partially @@ -4234,6 +4242,35 @@ public SettableFuture createContinuousQuery( return future; } + private boolean allClusterNodesSupportCQDurationEncoding(TShowClusterResp response) { + if (response == null + || response.getConfigNodeList() == null + || response.getDataNodeList() == null + || response.getNodeVersionInfo() == null) { + return false; + } + if (response.getConfigNodeList().isEmpty() && response.getDataNodeList().isEmpty()) { + return false; + } + for (TConfigNodeLocation node : response.getConfigNodeList()) { + if (!supportsCQDurationEncoding(response.getNodeVersionInfo().get(node.getConfigNodeId()))) { + return false; + } + } + for (TDataNodeLocation node : response.getDataNodeList()) { + if (!supportsCQDurationEncoding(response.getNodeVersionInfo().get(node.getDataNodeId()))) { + return false; + } + } + return true; + } + + private boolean supportsCQDurationEncoding(TNodeVersionInfo info) { + return info != null + && info.isSetSupportedCQDurationEncodingVersions() + && info.getSupportedCQDurationEncodingVersions().contains((short) 1); + } + @Override public SettableFuture dropContinuousQuery(final String cqId) { final SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 5ce282db7a701..1e1659c74bd1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -561,7 +561,9 @@ private void sendRegisterRequestToConfigNode(boolean isPreCheck) req.setPreCheck(isPreCheck); req.setDataNodeConfiguration(generateDataNodeConfiguration()); req.setClusterName(config.getClusterName()); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo( + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); TDataNodeRegisterResp dataNodeRegisterResp = null; while (retry > 0) { try (ConfigNodeClient configNodeClient = @@ -737,7 +739,9 @@ private void sendRestartRequestToConfigNode() throws StartupException { req.setClusterName( config.getClusterName() == null ? DEFAULT_CLUSTER_NAME : config.getClusterName()); req.setDataNodeConfiguration(generateDataNodeConfiguration()); - req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO)); + req.setVersionInfo( + new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO) + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1))); req.setClusterId(config.getClusterId()); TDataNodeRestartResp dataNodeRestartResp = null; while (retry > 0) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java new file mode 100644 index 0000000000000..c8065a975ca97 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutorCQTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.executor; + +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class ClusterConfigTaskExecutorCQTest { + + @Test + public void mixedVersionClusterIsRejectedAtDataNodeIngress() throws Exception { + Method method = + ClusterConfigTaskExecutor.class.getDeclaredMethod( + "allClusterNodesSupportCQDurationEncoding", TShowClusterResp.class); + method.setAccessible(true); + ClusterConfigTaskExecutor executor = ClusterConfigTaskExecutor.getInstance(); + + Assert.assertFalse((Boolean) method.invoke(executor, new TShowClusterResp())); + Assert.assertFalse((Boolean) method.invoke(executor, (Object) null)); + + TConfigNodeLocation configNode = + new TConfigNodeLocation( + 1, new TEndPoint("127.0.0.1", 10710), new TEndPoint("127.0.0.1", 10720)); + TDataNodeLocation dataNode = + new TDataNodeLocation( + 11, + new TEndPoint("127.0.0.1", 6667), + new TEndPoint("127.0.0.1", 10730), + new TEndPoint("127.0.0.1", 10740), + new TEndPoint("127.0.0.1", 10750), + new TEndPoint("127.0.0.1", 10760)); + TNodeVersionInfo supported = + new TNodeVersionInfo("2.0.0", "new") + .setSupportedCQDurationEncodingVersions(Collections.singleton((short) 1)); + TNodeVersionInfo unsupported = new TNodeVersionInfo("1.3.0", "old"); + + TShowClusterResp mixed = new TShowClusterResp(); + mixed.setConfigNodeList(Collections.singletonList(configNode)); + mixed.setDataNodeList(Collections.singletonList(dataNode)); + Map versions = new HashMap<>(); + versions.put(configNode.getConfigNodeId(), supported); + versions.put(dataNode.getDataNodeId(), unsupported); + mixed.setNodeVersionInfo(versions); + Assert.assertFalse((Boolean) method.invoke(executor, mixed)); + + versions.put(dataNode.getDataNodeId(), supported); + Assert.assertTrue((Boolean) method.invoke(executor, mixed)); + } +} diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index a74d8dcb312a7..8ce6213a44004 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -653,6 +653,8 @@ struct TGetClusterIdResp { struct TNodeVersionInfo { 1: required string version; 2: required string buildInfo; + // Capabilities are optional so older nodes can still register. Missing means unsupported. + 3: optional set supportedCQDurationEncodingVersions; } struct TNodeActivateInfo { From 8f59c8693ad7d3686a133ce99094269519f83e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:08:55 +0800 Subject: [PATCH 08/10] test: add P0 coverage for calendar CQ ser/de and BOUNDARY 0 semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three critical paths were not covered by existing tests: 1. CreateCQProcedure serialization with calendar fields (everyDuration, boundaryExplicit) — the procedure recovery path via ProcedureFactory was never exercised with a calendar-aware CQ request. 2. UpdateCQLastExecTimePlan ser/de with occurrence index — the new 5-parameter constructor (expectedIndex, targetIndex) had no round-trip test, and the legacy 3-parameter format (pre-calendar) had no forward compatibility test to confirm the optional tail is correctly skipped during Ratis log replay. 3. Explicit BOUNDARY 0 vs omitted BOUNDARY — the revised addendum promised 'BOUNDARY 0 stays Unix epoch; omitted uses local 1970-01-01'. The implementation has 5 occurrences of isBoundaryExplicit() checks; flipping any ! operator would silently shift all BOUNDARY 0 CQs by the zone offset (e.g., 8 hours in Asia/Shanghai), but no test observed the difference. The three new tests: - calendarCqWithOmittedBoundarySurvivesSerializeDeserialize - UpdateCQLastExecTimePlanWithOccurrenceIndexTest - UpdateCQLastExecTimePlanLegacyFormatWithoutIndexTest - testExplicitBoundaryZeroStaysUnixEpochWhileOmittedBoundaryUsesLocalEpoch Mutation testing confirmed each catches the bug it was designed to prevent. Related to #18428 --- .../request/ConfigPhysicalPlanSerDeTest.java | 40 +++++++++++ .../manager/cq/CQCalendarUtilsTest.java | 69 +++++++++++++++++++ .../procedure/impl/CreateCQProcedureTest.java | 49 +++++++++++++ 3 files changed, 158 insertions(+) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 0e8585898031e..16c4c8a76ec3d 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -191,10 +191,14 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Assert; import org.junit.Test; +import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -1866,6 +1870,42 @@ public void UpdateCQLastExecTimePlanTest() throws IOException { Assert.assertEquals(updateCQLastExecTimePlan0, updateCQLastExecTimePlan1); } + @Test + public void UpdateCQLastExecTimePlanWithOccurrenceIndexTest() throws IOException { + UpdateCQLastExecTimePlan plan0 = + new UpdateCQLastExecTimePlan("calendarCq", 1000L, "calendarToken", 5L, 6L); + UpdateCQLastExecTimePlan plan1 = + (UpdateCQLastExecTimePlan) ConfigPhysicalPlan.Factory.create(plan0.serializeToByteBuffer()); + + Assert.assertEquals(plan0, plan1); + Assert.assertTrue(plan1.hasOccurrenceIndex()); + Assert.assertEquals(5L, plan1.getExpectedIndex()); + Assert.assertEquals(6L, plan1.getTargetIndex()); + } + + @Test + public void UpdateCQLastExecTimePlanLegacyFormatWithoutIndexTest() throws IOException { + PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + // Byte layout written by pre-calendar ConfigNodes: type, cqId, executionTime, cqToken, with + // no occurrence-index tail. Ratis log replay must still accept it. + outputStream.writeShort(ConfigPhysicalPlanType.UPDATE_CQ_LAST_EXEC_TIME.getPlanType()); + ReadWriteIOUtils.write("legacyCq", outputStream); + ReadWriteIOUtils.write(2000L, outputStream); + ReadWriteIOUtils.write("legacyToken", outputStream); + + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + UpdateCQLastExecTimePlan plan = + (UpdateCQLastExecTimePlan) ConfigPhysicalPlan.Factory.create(buffer); + + Assert.assertEquals("legacyCq", plan.getCqId()); + Assert.assertEquals(2000L, plan.getExecutionTime()); + Assert.assertEquals("legacyToken", plan.getCqToken()); + Assert.assertFalse(plan.hasOccurrenceIndex()); + } + @Test public void RemoveDataNodePlanTest() throws IOException { List locations = new ArrayList<>(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java index 65d35c750d663..122f0e4fd81c9 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtilsTest.java @@ -25,12 +25,14 @@ import org.apache.tsfile.utils.TimeDuration; import org.junit.Test; +import java.lang.reflect.Field; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class CQCalendarUtilsTest { @@ -333,6 +335,73 @@ public void testDiscardClockRollbackKeepsTheNextOccurrence() { TimeoutPolicy.DISCARD, 2, callbackTime, executionTime, 0, 2, boundary, every, UTC)); } + @Test + public void testExplicitBoundaryZeroStaysUnixEpochWhileOmittedBoundaryUsesLocalEpoch() + throws Exception { + ZoneId shanghai = ZoneId.of("Asia/Shanghai"); + long unixEpoch = epochTimestamp(1970, 1, 1, 0, 0, UTC); + long shanghaiLocalEpoch = epochTimestamp(1970, 1, 1, 0, 0, shanghai); + // The two anchors differ by exactly the zone offset; a flipped explicit-flag check would + // silently shift every occurrence of a BOUNDARY 0 CQ by 8 hours in Asia/Shanghai. + assertEquals( + TimestampPrecisionUtils.currPrecision.convert(8, TimeUnit.HOURS), + unixEpoch - shanghaiLocalEpoch); + + // Explicit BOUNDARY 0: the anchor is the Unix epoch instant, observed in the CQ zone. + CQScheduleTask explicitTask = + new CQScheduleTask(shanghaiMonthlyReq(true), unixEpoch, "token", null, null); + assertEquals(0L, boundaryTimeOf(explicitTask)); + assertEquals(unixEpoch, executionTimeOf(explicitTask)); + + // Omitted BOUNDARY: the anchor is local 1970-01-01 00:00 in the persisted CQ zone. + CQScheduleTask omittedTask = + new CQScheduleTask(shanghaiMonthlyReq(false), shanghaiLocalEpoch, "token", null, null); + assertEquals(shanghaiLocalEpoch, boundaryTimeOf(omittedTask)); + assertEquals(shanghaiLocalEpoch, executionTimeOf(omittedTask)); + assertEquals(CQCalendarUtils.localEpochBoundary(shanghai), boundaryTimeOf(omittedTask)); + + // Cross-wiring the two anchors must be rejected by the occurrence/execution-time check. + assertThrows( + IllegalArgumentException.class, + () -> new CQScheduleTask(shanghaiMonthlyReq(true), shanghaiLocalEpoch, "t", null, null)); + assertThrows( + IllegalArgumentException.class, + () -> new CQScheduleTask(shanghaiMonthlyReq(false), unixEpoch, "t", null, null)); + } + + private static TCreateCQReq shanghaiMonthlyReq(boolean boundaryExplicit) { + TCreateCQReq req = + new TCreateCQReq( + "boundaryCq", + 0, + 0, + 0, + 0, + TimeoutPolicy.BLOCKED.getType(), + "select 1", + "create cq boundaryCq", + "Asia/Shanghai", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(boundaryExplicit); + return req; + } + + private static long boundaryTimeOf(CQScheduleTask task) throws Exception { + Field field = CQScheduleTask.class.getDeclaredField("boundaryTime"); + field.setAccessible(true); + return field.getLong(task); + } + + private static long executionTimeOf(CQScheduleTask task) throws Exception { + Field field = CQScheduleTask.class.getDeclaredField("executionTime"); + field.setAccessible(true); + return field.getLong(task); + } + private static long epochTimestamp( int year, int month, int day, int hour, int minute, ZoneId zone) { Instant instant = ZonedDateTime.of(year, month, day, hour, minute, 0, 0, zone).toInstant(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java index 024eaf25ceb6c..9e35935d59554 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java @@ -140,4 +140,53 @@ public void serializeDeserializeTest() { executor.shutdown(); } } + + @Test + public void calendarCqWithOmittedBoundarySurvivesSerializeDeserialize() { + PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + TCreateCQReq req = + new TCreateCQReq( + "calendarCq", + 0, + 0, + 0, + 0, + (byte) 0, + "select s1 into root.backup.d1(s1) from root.sg.d1", + "create cq calendarCq", + "Asia/Shanghai", + "root"); + req.setDurationEncodingVersion((short) 1); + req.setEveryDuration(new TCQDuration(1, 0)); + req.setStartOffsetDuration(new TCQDuration(1, 0)); + req.setEndOffsetDuration(new TCQDuration(0, 0)); + req.setBoundaryExplicit(false); + + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + CreateCQProcedure procedure1 = new CreateCQProcedure(req, executor); + + CQManager cqManager = Mockito.mock(CQManager.class); + Mockito.when(cqManager.getExecutor()).thenReturn(executor); + ConfigManager configManager = Mockito.mock(ConfigManager.class); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); + ConfigNode configNode = new ConfigNode(); + configNode.setConfigManager(configManager); + + try { + procedure1.serialize(outputStream); + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + + CreateCQProcedure procedure2 = + (CreateCQProcedure) ProcedureFactory.getInstance().create(buffer); + assertEquals(procedure1, procedure2); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + executor.shutdown(); + } + } } From 126245effa707db69d5645fcc33f483518e22598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:55:32 +0800 Subject: [PATCH 09/10] refactor: eliminate duplication and FQN pollution in CQ calendar code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: - TCQDuration→TimeDuration conversion (Math.toIntExact) repeated 7 times - Boundary resolution (isSetBoundaryExplicit && !isBoundaryExplicit) repeated 5 times - Precision scaling ("us"→1000, "ns"→1000000) repeated 3 times - FQN pollution: 43 occurrences of org.apache.tsfile.utils.TimeDuration, java.time.ZoneId, org.apache.iotdb.common.rpc.thrift.T* in main code After: - New CQDurationUtils with toTimeDuration(), resolveBoundary(), currentTimeInPrecision() - FQN reduced to 11 (1 intentional in CQDurationUtils, 10 unavoidable in lambdas/locals) - Proper imports added: ZoneId, TConfigNodeLocation, TDataNodeConfiguration, Map Other fixes: - CQScheduleTask.run() wrapped with try-catch to prevent silent task death on assertion failure - CreateContinuousQueryStatement.semanticCheck() reordered: positive-value check moved before lower-bound arithmetic (defensive programming) - Typo: "deliberate not" → "deliberately not" in DataNodeDateTimeUtils All 197 tests (175 ConfigNode + 22 DataNode) pass. Related to #18428 --- .../manager/cq/CQDurationUtils.java | 76 ++++++++++++ .../confignode/manager/cq/CQManager.java | 13 ++- .../confignode/manager/cq/CQScheduleTask.java | 46 ++++---- .../confignode/persistence/cq/CQInfo.java | 109 +++++++----------- .../procedure/impl/cq/CreateCQProcedure.java | 32 ++--- .../CreateContinuousQueryStatement.java | 9 +- .../iotdb/db/utils/DataNodeDateTimeUtils.java | 4 +- 7 files changed, 163 insertions(+), 126 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQDurationUtils.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQDurationUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQDurationUtils.java new file mode 100644 index 0000000000000..b451c92279fe3 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQDurationUtils.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.cq; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; + +import org.apache.tsfile.utils.TimeDuration; + +import java.time.ZoneId; + +/** Shared utilities for converting CQ Thrift types to structured durations and boundaries. */ +public final class CQDurationUtils { + + private CQDurationUtils() {} + + /** + * Converts a Thrift TCQDuration to a structured TimeDuration, falling back to a legacy fixed + * duration if the request predates calendar-aware encoding. + */ + public static TimeDuration toTimeDuration(TCreateCQReq req, TCQDuration d, long legacyFixed) { + if (req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1 && d != null) { + return new TimeDuration(Math.toIntExact(d.getMonthPart()), d.getNonMonthDuration()); + } + return new TimeDuration(0, legacyFixed); + } + + /** + * Resolves the calendar anchor for a CQ. If the request explicitly sets BOUNDARY, that value is + * used as-is; if omitted and the CQ is calendar-aware, the anchor becomes the local epoch + * (1970-01-01 00:00 in the persisted zone). + */ + public static long resolveBoundary(TCreateCQReq req, ZoneId zone, TimeDuration everyDuration) { + if (everyDuration.monthDuration != 0 + && req.isSetBoundaryExplicit() + && !req.isBoundaryExplicit()) { + return CQCalendarUtils.localEpochBoundary(zone); + } + return req.boundaryTime; + } + + /** + * Scales the current system time to the configured timestamp precision, producing a long suitable + * for CQ scheduling arithmetic. + */ + public static long currentTimeInPrecision() { + String precision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); + long multiplier; + if ("ns".equals(precision)) { + multiplier = 1_000_000L; + } else if ("us".equals(precision)) { + multiplier = 1_000L; + } else { + multiplier = 1L; + } + return System.currentTimeMillis() * multiplier; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java index 0159cc3af3be3..eba28d39b01f7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java @@ -19,6 +19,8 @@ package org.apache.iotdb.confignode.manager.cq; +import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; @@ -45,6 +47,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; @@ -219,7 +222,7 @@ public boolean hasCalendarDurationCQ() { } private boolean allClusterNodesSupportDurationEncodingV1() { - java.util.Map versionInfo = + Map versionInfo = configManager.getNodeManager().getNodeVersionInfo(); if (versionInfo == null || versionInfo.isEmpty()) { return false; @@ -227,20 +230,20 @@ private boolean allClusterNodesSupportDurationEncodingV1() { boolean hasRegisteredNode = false; // Check every registered node explicitly. A missing heartbeat/version entry must not allow a // calendar CQ to be created during a rolling upgrade. - List configNodes = + List configNodes = configManager.getNodeManager().getRegisteredConfigNodes(); - List dataNodes = + List dataNodes = configManager.getNodeManager().getRegisteredDataNodes(); if (configNodes == null || dataNodes == null) { return false; } - for (org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation node : configNodes) { + for (TConfigNodeLocation node : configNodes) { hasRegisteredNode = true; if (!supportsDurationEncodingV1(versionInfo.get(node.getConfigNodeId()))) { return false; } } - for (org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration node : dataNodes) { + for (TDataNodeConfiguration node : dataNodes) { hasRegisteredNode = true; if (!supportsDurationEncodingV1(versionInfo.get(node.getLocation().getDataNodeId()))) { return false; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java index c5b473df9045b..793c9eebc5bea 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java @@ -122,23 +122,18 @@ public CQScheduleTask( configManager, firstExecutionTime); this.everyDuration = - req.isSetEveryDuration() - ? new TimeDuration( - Math.toIntExact(req.getEveryDuration().getMonthPart()), - req.getEveryDuration().getNonMonthDuration()) - : new TimeDuration(0, req.everyInterval); + CQDurationUtils.toTimeDuration( + req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval); this.startDuration = - req.isSetStartOffsetDuration() - ? new TimeDuration( - Math.toIntExact(req.getStartOffsetDuration().getMonthPart()), - req.getStartOffsetDuration().getNonMonthDuration()) - : new TimeDuration(0, req.startTimeOffset); + CQDurationUtils.toTimeDuration( + req, + req.isSetStartOffsetDuration() ? req.getStartOffsetDuration() : null, + req.startTimeOffset); this.endDuration = - req.isSetEndOffsetDuration() - ? new TimeDuration( - Math.toIntExact(req.getEndOffsetDuration().getMonthPart()), - req.getEndOffsetDuration().getNonMonthDuration()) - : new TimeDuration(0, req.endTimeOffset); + CQDurationUtils.toTimeDuration( + req, + req.isSetEndOffsetDuration() ? req.getEndOffsetDuration() : null, + req.endTimeOffset); this.calendarAware = everyDuration.monthDuration != 0 || startDuration.monthDuration != 0 @@ -151,9 +146,7 @@ public CQScheduleTask( if (calendarAware) { this.retryWaitTimeInMS = calculateRetryWaitTime(everyDuration); } - if (scheduleCalendarAware && req.isSetBoundaryExplicit() && !req.isBoundaryExplicit()) { - this.boundaryTime = CQCalendarUtils.localEpochBoundary(scheduleZone); - } + this.boundaryTime = CQDurationUtils.resolveBoundary(req, scheduleZone, everyDuration); // The procedure has already selected the first occurrence. Recomputing it from wall clock // time here introduces a race around a calendar boundary and can skip an occurrence. if (req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1) { @@ -216,10 +209,8 @@ public CQScheduleTask( CQCalendarUtils.occurrence(boundaryTime, everyDuration, index, scheduleZone); } if (entry.getNextOccurrenceIndex() >= 0) { + // boundaryTime was already resolved above for calendar EVERY; reuse it unchanged. this.occurrenceIndex = entry.getNextOccurrenceIndex(); - if (!entry.isBoundaryExplicit() && scheduleCalendarAware) { - this.boundaryTime = CQCalendarUtils.localEpochBoundary(scheduleZone); - } this.executionTime = occurrenceAt(this.occurrenceIndex); } } @@ -263,7 +254,7 @@ public CQScheduleTask( } public static long getFirstExecutionTime(long boundaryTime, long everyInterval) { - long now = System.currentTimeMillis() * FACTOR; + long now = CQDurationUtils.currentTimeInPrecision(); return getFirstExecutionTime(boundaryTime, everyInterval, now); } @@ -283,6 +274,17 @@ public static long getFirstExecutionTime( @Override public void run() { + try { + runMayThrow(); + } catch (Throwable t) { + LOGGER.error(ManagerMessages.EXECUTE_CQ_FAILED, cqId, t); + if (needSubmit()) { + submitSelf(retryWaitTimeInMS, TimeUnit.MILLISECONDS); + } + } + } + + private void runMayThrow() { long currentOccurrenceIndex = occurrenceIndex; if (cancelled.get()) { return; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java index 269e9d38f6790..d2a3926efafb1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java @@ -32,7 +32,7 @@ import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; -import org.apache.iotdb.confignode.rpc.thrift.TCQDuration; +import org.apache.iotdb.confignode.manager.cq.CQDurationUtils; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.rpc.TSStatusCode; @@ -51,6 +51,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; +import java.time.ZoneId; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -100,51 +101,37 @@ public TSStatus addCQ(AddCQPlan plan) { res.message = String.format("CQ %s has already been created.", cqId); } else { TCreateCQReq req = plan.getReq(); + boolean versioned = + req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1; TimeDuration everyDuration = - req.isSetDurationEncodingVersion() - && req.getDurationEncodingVersion() == 1 - && req.isSetEveryDuration() - ? new TimeDuration( - Math.toIntExact(req.getEveryDuration().getMonthPart()), - req.getEveryDuration().getNonMonthDuration()) - : new TimeDuration(0, req.everyInterval); + CQDurationUtils.toTimeDuration( + req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval); + boolean calendarAware = + versioned + && (req.getEveryDuration().getMonthPart() != 0 + || req.getStartOffsetDuration().getMonthPart() != 0 + || req.getEndOffsetDuration().getMonthPart() != 0); + // Fixed-only CQs keep the zone opaque so legacy non-canonical zone strings still load. + ZoneId zone = calendarAware ? ZoneId.of(req.zoneId) : null; + long boundary = CQDurationUtils.resolveBoundary(req, zone, everyDuration); + long firstExecutionTime = plan.getFirstExecutionTime(); + long lastExecutionTime; if (everyDuration.monthDuration != 0) { - java.time.ZoneId zone = java.time.ZoneId.of(plan.getReq().zoneId); - long boundary = - req.isSetBoundaryExplicit() && !req.isBoundaryExplicit() - ? CQCalendarUtils.localEpochBoundary(zone) - : req.boundaryTime; long index = CQCalendarUtils.firstOccurrenceIndex( - boundary, everyDuration, plan.getFirstExecutionTime(), zone); + boundary, everyDuration, firstExecutionTime, zone); lastExecutionTime = CQCalendarUtils.occurrence(boundary, everyDuration, index - 1, zone); } else { // Version 1 may carry zero legacy fields when another component is calendar-aware. Use // the structured fixed duration to keep the persisted previous occurrence accurate. - lastExecutionTime = plan.getFirstExecutionTime() - everyDuration.nonMonthDuration; - } - long nextOccurrenceIndex = -1; - if (req.isSetDurationEncodingVersion() && req.getDurationEncodingVersion() == 1) { - TimeDuration duration = - new TimeDuration( - Math.toIntExact(req.getEveryDuration().getMonthPart()), - req.getEveryDuration().getNonMonthDuration()); - boolean calendarAware = - req.getEveryDuration().getMonthPart() != 0 - || req.getStartOffsetDuration().getMonthPart() != 0 - || req.getEndOffsetDuration().getMonthPart() != 0; - java.time.ZoneId zone = calendarAware ? java.time.ZoneId.of(req.zoneId) : null; - long boundary = - duration.monthDuration != 0 - && req.isSetBoundaryExplicit() - && !req.isBoundaryExplicit() - ? CQCalendarUtils.localEpochBoundary(zone) - : req.boundaryTime; - nextOccurrenceIndex = - CQCalendarUtils.firstOccurrenceIndex( - boundary, duration, plan.getFirstExecutionTime(), zone); + lastExecutionTime = firstExecutionTime - everyDuration.nonMonthDuration; } + long nextOccurrenceIndex = + versioned + ? CQCalendarUtils.firstOccurrenceIndex( + boundary, everyDuration, firstExecutionTime, zone) + : -1; CQEntry cqEntry = new CQEntry(plan.getReq(), plan.getCqToken(), lastExecutionTime, nextOccurrenceIndex); cqMap.put(cqId, cqEntry); @@ -422,9 +409,9 @@ public static class CQEntry { private final String zoneId; private final String username; - private org.apache.tsfile.utils.TimeDuration everyDuration; - private org.apache.tsfile.utils.TimeDuration startTimeOffsetDuration; - private org.apache.tsfile.utils.TimeDuration endTimeOffsetDuration; + private TimeDuration everyDuration; + private TimeDuration startTimeOffsetDuration; + private TimeDuration endTimeOffsetDuration; private boolean boundaryExplicit; private CQState state; @@ -449,13 +436,13 @@ private CQEntry( cqToken, req.zoneId, req.username, - durationFromReq( + CQDurationUtils.toTimeDuration( req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval), - durationFromReq( + CQDurationUtils.toTimeDuration( req, req.isSetStartOffsetDuration() ? req.getStartOffsetDuration() : null, req.startTimeOffset), - durationFromReq( + CQDurationUtils.toTimeDuration( req, req.isSetEndOffsetDuration() ? req.getEndOffsetDuration() : null, req.endTimeOffset), @@ -500,9 +487,9 @@ private CQEntry( String cqToken, String zoneId, String username, - org.apache.tsfile.utils.TimeDuration everyDuration, - org.apache.tsfile.utils.TimeDuration startTimeOffsetDuration, - org.apache.tsfile.utils.TimeDuration endTimeOffsetDuration, + TimeDuration everyDuration, + TimeDuration startTimeOffsetDuration, + TimeDuration endTimeOffsetDuration, boolean boundaryExplicit, CQState state, long lastExecutionTime, @@ -581,9 +568,9 @@ private static CQEntry deserializeLegacy(InputStream stream) throws IOException cqToken, zoneId, username, - new org.apache.tsfile.utils.TimeDuration(0, everyInterval), - new org.apache.tsfile.utils.TimeDuration(0, startTimeOffset), - new org.apache.tsfile.utils.TimeDuration(0, endTimeOffset), + new TimeDuration(0, everyInterval), + new TimeDuration(0, startTimeOffset), + new TimeDuration(0, endTimeOffset), false, state, lastExecutionTime, @@ -592,14 +579,11 @@ private static CQEntry deserializeLegacy(InputStream stream) throws IOException private void deserializeExtension(InputStream stream) throws IOException { everyDuration = - new org.apache.tsfile.utils.TimeDuration( - ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + new TimeDuration(ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); startTimeOffsetDuration = - new org.apache.tsfile.utils.TimeDuration( - ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + new TimeDuration(ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); endTimeOffsetDuration = - new org.apache.tsfile.utils.TimeDuration( - ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); + new TimeDuration(ReadWriteIOUtils.readInt(stream), ReadWriteIOUtils.readLong(stream)); boundaryExplicit = ReadWriteIOUtils.readBool(stream); nextOccurrenceIndex = ReadWriteIOUtils.readLong(stream); } @@ -671,15 +655,15 @@ public String getUsername() { return username; } - public org.apache.tsfile.utils.TimeDuration getEveryDuration() { + public TimeDuration getEveryDuration() { return everyDuration; } - public org.apache.tsfile.utils.TimeDuration getStartTimeOffsetDuration() { + public TimeDuration getStartTimeOffsetDuration() { return startTimeOffsetDuration; } - public org.apache.tsfile.utils.TimeDuration getEndTimeOffsetDuration() { + public TimeDuration getEndTimeOffsetDuration() { return endTimeOffsetDuration; } @@ -693,17 +677,6 @@ public boolean hasCalendarDuration() { || endTimeOffsetDuration.monthDuration != 0; } - private static org.apache.tsfile.utils.TimeDuration durationFromReq( - TCreateCQReq req, TCQDuration d, long legacy) { - if (req.isSetDurationEncodingVersion() - && req.getDurationEncodingVersion() == 1 - && d != null) { - return new org.apache.tsfile.utils.TimeDuration( - Math.toIntExact(d.getMonthPart()), d.getNonMonthDuration()); - } - return new org.apache.tsfile.utils.TimeDuration(0, legacy); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java index a22b9c9148610..cf0a5f218a079 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java @@ -29,6 +29,7 @@ import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.manager.cq.CQCalendarUtils; +import org.apache.iotdb.confignode.manager.cq.CQDurationUtils; import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; import org.apache.iotdb.confignode.persistence.cq.CQInfo; @@ -49,6 +50,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.ZoneId; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -85,32 +87,12 @@ public CreateCQProcedure(TCreateCQReq req, ScheduledExecutorService executor) { this.cqToken = generateCQToken(); this.executor = executor; TimeDuration everyDuration = - req.isSetEveryDuration() - ? new TimeDuration( - Math.toIntExact(req.getEveryDuration().getMonthPart()), - req.getEveryDuration().getNonMonthDuration()) - : new TimeDuration(0, req.everyInterval); + CQDurationUtils.toTimeDuration( + req, req.isSetEveryDuration() ? req.getEveryDuration() : null, req.everyInterval); if (everyDuration.monthDuration != 0) { - java.time.ZoneId zone = java.time.ZoneId.of(req.zoneId); - long boundary = - req.isSetBoundaryExplicit() && !req.isBoundaryExplicit() - ? CQCalendarUtils.localEpochBoundary(zone) - : req.boundaryTime; - long now = - System.currentTimeMillis() - * ("ns" - .equals( - org.apache.iotdb.commons.conf.CommonDescriptor.getInstance() - .getConfig() - .getTimestampPrecision()) - ? 1_000_000L - : "us" - .equals( - org.apache.iotdb.commons.conf.CommonDescriptor.getInstance() - .getConfig() - .getTimestampPrecision()) - ? 1_000L - : 1L); + ZoneId zone = ZoneId.of(req.zoneId); + long boundary = CQDurationUtils.resolveBoundary(req, zone, everyDuration); + long now = CQDurationUtils.currentTimeInPrecision(); long index = CQCalendarUtils.firstOccurrenceIndex(boundary, everyDuration, now, zone); this.firstExecutionTime = CQCalendarUtils.occurrence(boundary, everyDuration, index, zone); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java index b10fc9ac89ab5..ae4cd3f176872 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CreateContinuousQueryStatement.java @@ -262,6 +262,11 @@ public R accept(StatementVisitor visitor, C context) { } public void semanticCheck() { + // Positivity first: the lower-bound arithmetic below assumes a non-degenerate EVERY. + if (!isPositive(everyDuration)) { + throw new SemanticException( + DataNodeQueryMessages.EXCEPTION_CQ_EVERY_INTERVAL_MUST_BE_POSITIVE_26259019); + } long minimumEvery = IoTDBDescriptor.getInstance().getConfig().getContinuousQueryMinimumEveryInterval(); long minimumElapsed = @@ -290,10 +295,6 @@ public void semanticCheck() { formatDuration(everyDuration), minimumEvery)); } - if (!isPositive(everyDuration)) { - throw new SemanticException( - DataNodeQueryMessages.EXCEPTION_CQ_EVERY_INTERVAL_MUST_BE_POSITIVE_26259019); - } if (!isPositive(startTimeOffsetDuration)) { throw new SemanticException(DataNodeQueryMessages.CQ_THE_START_TIME_OFFSET_SHOULD_BE_GREATER); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java index a5222ab7256da..2d76c6fa651a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DataNodeDateTimeUtils.java @@ -49,8 +49,8 @@ public class DataNodeDateTimeUtils { Pattern.compile("(\\d+)(y|mo|w|d|h|ms|us|ns|m|s)", Pattern.CASE_INSENSITIVE); /** - * Parses the CQ duration grammar while retaining calendar months. Full aliases are deliberate not - * accepted here; CQ uses the same mo/y abbreviations as Tree SQL. + * Parses the CQ duration grammar while retaining calendar months. Full aliases are deliberately + * not accepted here; CQ uses the same mo/y abbreviations as Tree SQL. */ public static TimeDuration constructTimeDurationForCQ(String duration) { if (duration == null || duration.isEmpty()) { From 585afdea765c0d432b31f8c3dea542b916b126f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=87=E5=A1=9E=E5=A4=A7=E5=98=B4=E5=A5=BD=E5=B8=A5?= <66861267+DaZuiZui@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:33:47 +0800 Subject: [PATCH 10/10] fix: address review findings on calendar CQ intervals - IoTDBCQExecIT: the calendar-month execution test returned silently when setup overran the scheduled first execution, producing a vacuous pass. Fail explicitly instead and widen the scheduling margin to 30s. - ManagerMessages: rename EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_ EXECUTION_TIME suffix from 7E4B91A2 to B2DE4B0F so it matches the md5 of the English value, per the i18n naming convention (en/zh in sync). - CQCalendarUtils: drop unused nextAfter(); firstOccurrenceIndex covers it. --- .../java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java | 5 +++-- .../org/apache/iotdb/confignode/i18n/ManagerMessages.java | 2 +- .../org/apache/iotdb/confignode/i18n/ManagerMessages.java | 2 +- .../iotdb/confignode/manager/cq/CQCalendarUtils.java | 8 -------- .../iotdb/confignode/manager/cq/CQScheduleTask.java | 4 ++-- 5 files changed, 7 insertions(+), 14 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java index 6476c9cc1fce1..4bdaa2a81bbcc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/cq/IoTDBCQExecIT.java @@ -61,7 +61,7 @@ public void testCalendarMonthCQExecutionUsesNaturalMonthWindow() { Statement statement = connection.createStatement()) { connection.setClientInfo("time_zone", "UTC"); long now = System.currentTimeMillis(); - long firstExecutionTime = now + 10_000; + long firstExecutionTime = now + 30_000; ZoneId utc = ZoneId.of("UTC"); long calendarStart = ZonedDateTime.ofInstant(Instant.ofEpochMilli(firstExecutionTime), utc) @@ -98,8 +98,9 @@ public void testCalendarMonthCQExecutionUsesNaturalMonthWindow() { + "END"); if (System.currentTimeMillis() > firstExecutionTime) { + // Do not return silently: a vacuous pass would hide a regression in the calendar window. statement.execute("DROP CQ cq_calendar_month"); - return; + fail("test setup exceeded the scheduled first execution time; increase the margin"); } long targetTime = firstExecutionTime + 10_000; diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 885dc8339f75c..c8abaf5cb43e0 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -737,7 +737,7 @@ private ManagerMessages() {} "Invalid CQ occurrence index transition"; public static final String EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035 = "Negative CQ snapshot entry count: %d"; - public static final String EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_7E4B91A2 = + public static final String EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_B2DE4B0F = "CQ occurrence index does not match the scheduled execution time"; public static final String EXCEPTION_CQ_RANGE_END_MUST_BE_GREATER_THAN_START_3C91E8B4 = "CQ RANGE end time must be greater than start time"; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index b56336a27fc73..f6c8173991b51 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -716,7 +716,7 @@ private ManagerMessages() {} "无效的 CQ occurrence index 转换"; public static final String EXCEPTION_NEGATIVE_CQ_SNAPSHOT_ENTRY_COUNT_ARG_38750035 = "CQ snapshot 条目数量不能为负数：%d"; - public static final String EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_7E4B91A2 = + public static final String EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_B2DE4B0F = "CQ occurrence index 与计划执行时间不匹配"; public static final String EXCEPTION_CQ_RANGE_END_MUST_BE_GREATER_THAN_START_3C91E8B4 = "CQ RANGE 结束时间必须大于开始时间"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java index 820fec0514534..2071f7565a89f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQCalendarUtils.java @@ -88,14 +88,6 @@ public static long firstOccurrenceIndex( return low; } - public static long nextAfter(long boundary, TimeDuration every, long previous, ZoneId zone) { - long index = firstOccurrenceIndex(boundary, every, previous, zone); - if (occurrence(boundary, every, index, zone) <= previous) { - index = Math.addExact(index, 1); - } - return occurrence(boundary, every, index, zone); - } - private static long fromInstant(Instant instant) { String precision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java index 793c9eebc5bea..86c6e26edded6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java @@ -156,7 +156,7 @@ public CQScheduleTask( long expectedExecution = occurrenceAt(this.occurrenceIndex); if (expectedExecution != firstExecutionTime) { throw new IllegalArgumentException( - ManagerMessages.EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_7E4B91A2); + ManagerMessages.EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_B2DE4B0F); } } } @@ -301,7 +301,7 @@ private void runMayThrow() { boundaryTime, everyDuration, currentOccurrenceIndex, scheduleZone); if (expectedExecution != executionTime) { throw new IllegalStateException( - ManagerMessages.EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_7E4B91A2); + ManagerMessages.EXCEPTION_CQ_OCCURRENCE_INDEX_DOES_NOT_MATCH_EXECUTION_TIME_B2DE4B0F); } startTime = calculateCalendarRangeEndpoint(startDuration, currentOccurrenceIndex); endTime = calculateCalendarRangeEndpoint(endDuration, currentOccurrenceIndex);