diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupDistributionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupDistributionIT.java new file mode 100644 index 0000000000000..980f07b2d53c1 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupDistributionIT.java @@ -0,0 +1,1471 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchemaResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE_BINARY; +import static org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils.generatePatternTreeBuffer; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBProactiveRegionGroupDistributionIT { + + private static final String AUTO_DATABASE = "root.auto"; + private static final String PROACTIVE_DATABASE = "root.proactive"; + private static final String CUSTOM_DATABASE = "root.custom"; + // 1000 possible hash slots, but the six devices deliberately activate only slots 0 through 5. + // Each device has one measurement; a new time partition adds no new series slot. + private static final int SERIES_SLOT_NUM = 1000; + private static final int DEVICE_COUNT = 6; + // Group counts below are logical groups: G groups correspond to 3 * G replica rows. + private static final int REPLICATION_FACTOR = 3; + private static final long TIME_PARTITION_INTERVAL = 10; + private static final BKDRHashExecutor PARTITION_EXECUTOR = new BKDRHashExecutor(SERIES_SLOT_NUM); + + /** + * Clean up the test cluster; for example, stop one ConfigNode and three DataNodes after a 1C3D + * case. + */ + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + /** + * Verify six distinct slots on 1C3D: PROACTIVE uses three groups per type with two slots each; + * AUTO uses one Schema group and two Data groups. + */ + @Test + public void testSixDevicesOnThreeDataNodes() throws Exception { + // Two databases: cap = 3 groups per type per database. PROACTIVE: 6 slots / 3 groups = 2. + checkSparseDeviceDistribution(3); + } + + /** + * Verify six distinct slots on 1C6D: PROACTIVE uses six groups per type with one slot each, + * placing 18 replicas as three per node. + */ + @Test + public void testSixDevicesOnSixDataNodes() throws Exception { + // Cap = 6; PROACTIVE uses 6 groups per type, each owning one of the six slots. + checkSparseDeviceDistribution(6); + } + + /** + * Verify sparse allocation on 1C9D: six active slots create only six groups despite a cap of + * nine, placing 18 replicas as two per node. + */ + @Test + public void testSixDevicesOnNineDataNodes() throws Exception { + // Cap = 9, but only 6 slots are active: PROACTIVE stops at 6 groups per type. + // Each type has 6 * 3 = 18 replicas, balanced across 9 DataNodes: 2 replicas per node. + checkSparseDeviceDistribution(9); + } + + /** + * Verify AUTO-to-PROACTIVE redistribution: two groups with three slots each become six groups + * with one slot each. Each old group retains one slot and each of four new groups receives one. + * Also verify historical mappings and SQL values. + */ + @Test + public void testAutoToProactiveHotReloadRedistributesSixSlots() throws Exception { + // Step 1: one database on 1C3D; AUTO starts with minimums of 1 schema group / 2 data groups. + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // Expect successful creation (200): Schema/Data minimums 1/2 are valid, and three nodes + // support three replicas. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(AUTO_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2))); + // Cap per type = ceil(6 groups per node * 3 nodes / (1 database * 3 replicas)) = 6. + // Expect both caps to be 6: ceil(6*3/(1 database*3 replicas))=6. + assertMaximum(client, AUTO_DATABASE, DEVICE_COUNT); + // Creates 6 devices / measurements / series slots and writes t=0 and t=10. + // Result: 1 schema group, 2 data groups, 6 schema partitions and 6 * 2 = 12 data partitions. + checkSixDevices(client, statement, AUTO_DATABASE, 3, false); + List devices = generateDeviceNamesForSlots(AUTO_DATABASE, DEVICE_COUNT); + Set oldGroups = + // Expect 2 Data groups and 6 replicas: AUTO has reached its minimum; six sparse slots + // do not trigger further growth. + assertRegionGroupsAndReplicas(client, AUTO_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet(); + Map>> original = + readDataPartitions(client, AUTO_DATABASE, devices, 0); + List originalAssignments = groupsAtTime(original, 0); + oldGroups.forEach( + // Expect 3 slots per old group: six distinct slots are evenly split between two AUTO + // groups, 6/2=3. + group -> Assert.assertEquals(3, Collections.frequency(originalAssignments, group))); + + // Step 2: t=20 raises the data-partition count to 6 * 3 = 18, still only 6 series slots. + // AUTO keeps 2 data groups with 3 slots each; verify the same slot-to-group routing as t=0. + writeAtTime(statement, devices, 2 * TIME_PARTITION_INTERVAL, 200); + // Expect the same two group IDs: adding only a time partition does not make AUTO extend + // or replace groups. + Assert.assertEquals( + oldGroups, + // Expect 2 Data groups: t=20 adds no series slots, so the active-slot count remains + // 6. + assertRegionGroupsAndReplicas(client, AUTO_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet()); + Map>> beforeReload = + readDataPartitions(client, AUTO_DATABASE, devices, 2 * TIME_PARTITION_INTERVAL); + original.forEach( + (slot, times) -> + // Expect the same owner at t=20 and t=0: AUTO has not extended, so the routing + // stays unchanged. + Assert.assertEquals( + times.get(new TTimePartitionSlot(0)), + beforeReload.get(slot).get(new TTimePartitionSlot(2 * TIME_PARTITION_INTERVAL)))); + + // Step 3: hot-reload only the data policy. Schema stays AUTO; no node is restarted. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + // Reload alone leaves 2 data groups. The first missing partition at t=30 triggers extension. + // Expect the original two IDs: reload alone creates no groups; a later missing-partition + // request triggers extension. + Assert.assertEquals( + oldGroups, + // Expect 2 Data groups: only the policy has changed; no new time partition has been + // written. + assertRegionGroupsAndReplicas(client, AUTO_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet()); + writeAtTime(statement, devices, 3 * TIME_PARTITION_INTERVAL, 300); + // Target = min(6 active slots, cap 6) = 6: keep the old 2 groups and create 6 - 2 = 4. + // t=0/10/20/30 now give 24 data partitions in total, still derived from 6 series slots. + Set allGroups = + // Expect 6 Data groups and 18 replicas: PROACTIVE targets min(6 active slots, cap + // 6)=6. + assertRegionGroupsAndReplicas( + client, AUTO_DATABASE, TConsensusGroupType.DataRegion, DEVICE_COUNT) + .keySet(); + // Expect true: all six groups must include the two old groups; extension must not delete + // or replace them. + Assert.assertTrue(allGroups.containsAll(oldGroups)); + Set newGroups = new HashSet<>(allGroups); + newGroups.removeAll(oldGroups); + // Expect 4 new groups: target 6 minus the existing 2 gives 6-2=4. + Assert.assertEquals(4, newGroups.size()); + Map>> afterReload = + readDataPartitions(client, AUTO_DATABASE, devices, 3 * TIME_PARTITION_INTERVAL); + // Expect all six groups to be used, with one slot each: 6 slots / 6 groups=1. + assertSlotDistribution(allGroups, groupsAtTime(afterReload, 3 * TIME_PARTITION_INTERVAL)); + + // Step 4: verify ownership, not just the final group count. At rebalance, + // PROACTIVE retains mu = max(1, floor(6 active slots / 6 groups)) = 1 slot per old group; + // AUTO would retain 166. + // Each old group unassigns 3 - 1 = 2 slots: 2 * 2 = 4 slots activate four distinct new + // groups. + // Which slot is retained is randomized, so compare its original owner rather than its ID. + Map retainedByOldGroup = new HashMap<>(); + Set reassignedGroups = new HashSet<>(); + original.forEach( + (slot, times) -> { + TConsensusGroupId previous = times.get(new TTimePartitionSlot(0)).get(0); + TConsensusGroupId current = + afterReload + .get(slot) + .get(new TTimePartitionSlot(3 * TIME_PARTITION_INTERVAL)) + .get(0); + if (oldGroups.contains(current)) { + // Expect a retained slot to keep its previous owner: an old group may retain only + // its own slots. + Assert.assertEquals(previous, current); + retainedByOldGroup.merge(current, 1, Integer::sum); + } else { + // Expect true: each reassigned slot uses a distinct new group, giving one slot to + // each of four new groups. + Assert.assertTrue(reassignedGroups.add(current)); + } + }); + // Expect exactly the original two groups to retain slots: both old groups must retain + // one. + Assert.assertEquals(oldGroups, retainedByOldGroup.keySet()); + // Expect 1 retained slot per old group: PROACTIVE uses mu=max(1,floor(6/6))=1. + retainedByOldGroup.values().forEach(count -> Assert.assertEquals(1, count.intValue())); + // Expect reassignment to use exactly the four new groups, without leaving any unused or + // including old groups. + Assert.assertEquals(newGroups, reassignedGroups); + + // Step 5: repeat the reload and write t=40: 6 data groups, 6 slots, 6 * 5 = 30 partitions. + // Verify stable future routing, unchanged historical mappings, and all five values per + // device. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + writeAtTime(statement, devices, 4 * TIME_PARTITION_INTERVAL, 400); + Map>> nextPartitions = + readDataPartitions(client, AUTO_DATABASE, devices, 4 * TIME_PARTITION_INTERVAL); + afterReload.forEach( + (slot, times) -> + // Expect identical routing at t=40 and t=30: reapplying the same policy preserves + // the balanced mapping. + Assert.assertEquals( + times.get(new TTimePartitionSlot(3 * TIME_PARTITION_INTERVAL)), + nextPartitions + .get(slot) + .get(new TTimePartitionSlot(4 * TIME_PARTITION_INTERVAL)))); + // Expect the six historical t=0 mappings to match the snapshot: rebalancing affects + // future allocation only. + Assert.assertEquals(original, readDataPartitions(client, AUTO_DATABASE, devices, 0)); + // Expect the six t=20 mappings to remain unchanged, preserving the last time partition + // before reload. + Assert.assertEquals( + beforeReload, + readDataPartitions(client, AUTO_DATABASE, devices, 2 * TIME_PARTITION_INTERVAL)); + // Expect the same six group IDs after another reload and time partition, with no further + // group creation. + Assert.assertEquals( + allGroups, + // Expect 6 Data groups: there are still six active series slots, and the database cap + // has been reached. + assertRegionGroupsAndReplicas( + client, AUTO_DATABASE, TConsensusGroupType.DataRegion, DEVICE_COUNT) + .keySet()); + // Expect 1 Schema group and 3 replicas: only the data policy changed, with no new schema + // slots. + assertRegionGroupsAndReplicas(client, AUTO_DATABASE, TConsensusGroupType.SchemaRegion, 1); + // Expect both caps to remain 6: node count, database count, replication and quotas are + // unchanged. + assertMaximum(client, AUTO_DATABASE, DEVICE_COUNT); + for (int time = 0; time <= 4; time++) { + // Expect one row per device at t=0/10/20/30/40, with slot+0/100/200/300/400 from the + // five writes. + assertValuesAtTime(statement, devices, time * TIME_PARTITION_INTERVAL, time * 100); + } + } + } + + /** + * Verify switching a low CUSTOM cap to PROACTIVE: refresh the cap from 2 to 6 and use six groups + * with one slot each for the next time partition. + */ + @Test + public void testCustomBelowProactiveHotReloadRedistributesSixSlots() throws Exception { + checkCustomToResourcePolicy("PROACTIVE"); + } + + /** + * Verify CUSTOM-to-AUTO cap refresh: increase the cap from 2 to 6 while the six sparse slots + * remain in the original two groups. + */ + @Test + public void testCustomToAutoHotReloadRefreshesMaximum() throws Exception { + checkCustomToResourcePolicy("AUTO"); + } + + /** + * Compare switching CUSTOM to resource-based policies. Starting with two groups of three slots, + * PROACTIVE uses six groups of one slot, while AUTO keeps two groups of three. Both caps must + * refresh from 2 to 6. + */ + private void checkCustomToResourcePolicy(String policy) throws Exception { + // Step 1: 1C3D / one database gives a resource cap of 6, but CUSTOM explicitly limits data to + // 2. + // At t=0, six devices activate six slots: 1 schema group, 2 data groups, 3 data slots per + // group. + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + // Expect successful creation (200): CUSTOM cap 2 equals the Data minimum, and three nodes + // support three replicas. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(CUSTOM_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2) + .setMaxDataRegionGroupNum(2))); + // Expect Schema cap 6 and Data cap 2: Schema uses ceil(6*3/(1*3)); Data uses the explicit + // CUSTOM cap. + assertMaximum(client, CUSTOM_DATABASE, 6, 2); + List devices = createSixDevices(statement, CUSTOM_DATABASE); + Set oldGroups = + // Expect 2 Data groups and 6 replicas: CUSTOM allocates directly to its configured + // cap of 2. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet(); + Map original = + readDataAssignments(client, CUSTOM_DATABASE, devices, 0); + oldGroups.forEach( + // Expect 3 slots per group: six active slots are evenly split between two CUSTOM + // groups. + group -> Assert.assertEquals(3, Collections.frequency(original.values(), group))); + + // Step 2: expect the data cap to refresh from 2 to 6 when leaving CUSTOM; groups remain at 2. + // Change only the policy: no quota edits, database creation or node registration may + // accidentally refresh the CUSTOM cap before the next partition request. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='" + policy + "'"); + // Expect the same two IDs: the policy switch refreshes the cap but has not yet triggered + // partition allocation. + Assert.assertEquals( + oldGroups, + // Expect 2 Data groups immediately after reload: t=10 has not been written, so + // extension has not run. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 2) + .keySet()); + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + // Step 3: t=10 adds six data partitions (12 total), but still only six active series slots. + // PROACTIVE grows to min(6 slots, cap 6)=6 groups; AUTO stays at its minimum of 2. + Set currentGroups = + // Expect 6 groups for PROACTIVE and 2 for AUTO: only PROACTIVE grows for the six + // active slots. + assertRegionGroupsAndReplicas( + client, + CUSTOM_DATABASE, + TConsensusGroupType.DataRegion, + "PROACTIVE".equals(policy) ? DEVICE_COUNT : 2) + .keySet(); + // Expect both caps to be 6: leaving CUSTOM recalculates the Data cap from resources + // instead of retaining 2. + assertMaximum(client, CUSTOM_DATABASE, 6); + Map current = + readDataAssignments(client, CUSTOM_DATABASE, devices, TIME_PARTITION_INTERVAL); + if ("PROACTIVE".equals(policy)) { + // Retain mu=max(1,6/6)=1 original slot in each old group; reassign 2*(3-1)=4 to new groups. + // Expect one slot in each of six groups: PROACTIVE must actually use the newly created + // groups. + assertSlotDistribution(currentGroups, new ArrayList<>(current.values())); + Set newGroups = new HashSet<>(currentGroups); + newGroups.removeAll(oldGroups); + // Expect 4 new groups: PROACTIVE target 6 minus the original 2 CUSTOM groups. + Assert.assertEquals(4, newGroups.size()); + Map retained = new HashMap<>(); + Set reassigned = new HashSet<>(); + current.forEach( + (slot, group) -> { + if (oldGroups.contains(group)) { + // Expect a retained slot to keep its original owner, without swapping slots + // between old groups. + Assert.assertEquals(original.get(slot), group); + retained.merge(group, 1, Integer::sum); + } else { + // Expect true: the four unassigned slots must each enter a different new group. + Assert.assertTrue(reassigned.add(group)); + } + }); + // Expect both original groups to retain slots; their ID set must match the pre-switch + // snapshot. + Assert.assertEquals(oldGroups, retained.keySet()); + // Expect one retained slot per old group: mu=max(1,6/6)=1; the other four slots enter + // new groups. + retained.values().forEach(count -> Assert.assertEquals(1, count.intValue())); + // Expect exactly the four new groups to receive reassigned slots, verifying that every + // new group is used. + Assert.assertEquals(newGroups, reassigned); + } else { + // AUTO has the same resource cap as PROACTIVE, but six sparse slots still need two groups. + // Expect AUTO routing to remain unchanged: raising the cap from 2 to 6 does not require + // more groups for sparse load. + Assert.assertEquals(original, current); + } + // Step 4: data-policy reload leaves schema at 1 group and preserves t=0 mappings and values. + // Expect the six t=0 mappings to match the snapshot: neither target policy may rewrite + // historical partitions. + Assert.assertEquals(original, readDataAssignments(client, CUSTOM_DATABASE, devices, 0)); + // Expect 1 Schema group and 3 replicas: only the data policy changed; schema load and + // minimum are unchanged. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 1); + // Expect one row per device at t=0 with values 0..5, verifying that pre-switch data + // remains readable. + assertValuesAtTime(statement, devices, 0, 0); + // Expect one row per device at t=10 with values 100..105, verifying the new time + // partition. + assertValuesAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + } + } + + /** + * Verify that existing CUSTOM groups exceeding the PROACTIVE target are preserved. For example, + * eight groups with six active slots and a resource cap of six retain all eight groups and + * existing slot owners. + */ + @Test + public void testCustomAboveProactiveHotReloadPreservesGroups() throws Exception { + // Step 1: CUSTOM creates 8 data groups although six active slots only use six of them. + // There are 8*3=24 data replicas; two logical groups initially have no series slots. + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + // Expect successful creation (200): CUSTOM permits an explicit Data cap of 8 above the + // minimum of 2. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(CUSTOM_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2) + .setMaxDataRegionGroupNum(8))); + List devices = createSixDevices(statement, CUSTOM_DATABASE); + Set originalGroups = + // Expect 8 Data groups and 24 replicas: CUSTOM creates up to its cap even with only + // six active slots. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 8) + .keySet(); + Map original = + readDataAssignments(client, CUSTOM_DATABASE, devices, 0); + // Expect 6 used groups: each slot occupies a distinct group, leaving two of the eight + // groups unused. + Assert.assertEquals(DEVICE_COUNT, new HashSet<>(original.values()).size()); + + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + // Step 2: recalculated data cap=max(minimum 2, resource quota 6, existing groups 8)=8. + // Switching to PROACTIVE does not shrink to the six-group active-slot target. + // Expect Schema cap 6 and Data cap 8: max(minimum 2, resource cap 6, existing groups 8)=8 + // prevents shrinking. + assertMaximum(client, CUSTOM_DATABASE, 6, 8); + // Expect all eight IDs to remain: switching to PROACTIVE must not delete groups to match + // six active slots. + Assert.assertEquals( + originalGroups, + // Expect 8 Data groups and 24 replicas: a policy reload must not shrink existing + // groups. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 8) + .keySet()); + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + // Step 3: t=10 gives 12 data partitions; all 8 group IDs and all six slot owners stay + // unchanged. + // Expect the same eight IDs after t=10: existing groups already outnumber the six active + // slots. + Assert.assertEquals( + originalGroups, + // Expect 8 Data groups: a new time partition adds no active series slots and does not + // trigger shrinking. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 8) + .keySet()); + // Expect t=10 routing to match t=0: without new groups, assigned slots keep their + // original owners. + Assert.assertEquals( + original, readDataAssignments(client, CUSTOM_DATABASE, devices, TIME_PARTITION_INTERVAL)); + // Expect the six historical t=0 mappings to remain unchanged: switching policies does not + // migrate partitions. + Assert.assertEquals(original, readDataAssignments(client, CUSTOM_DATABASE, devices, 0)); + // Expect 1 Schema group: only the data policy changed; schema load and minimum are + // unchanged. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 1); + // Expect one row per device at t=0 with values 0..5, confirming historical reads after + // retaining eight groups. + assertValuesAtTime(statement, devices, 0, 0); + // Expect one row per device at t=10 with values 100..105, verifying writes through the + // retained routing. + assertValuesAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + } + } + + /** + * Verify that entering PROACTIVE balances slots even when CUSTOM has already created enough + * groups. Existing time partitions and repeated policy reloads retain their assignments. + */ + @Test + public void testExpandedCustomGroupsRebalanceOnProactiveSwitchWithoutFurtherGrowth() + throws Exception { + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + assertSuccess( + client.setDatabase( + new TDatabaseSchema(CUSTOM_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2) + .setMaxDataRegionGroupNum(2))); + List devices = createSixDevices(statement, CUSTOM_DATABASE); + Map original = + readDataAssignments(client, CUSTOM_DATABASE, devices, 0); + Assert.assertEquals(2, new HashSet<>(original.values()).size()); + + statement.execute("ALTER DATABASE " + CUSTOM_DATABASE + " WITH MAX_DATA_REGION_GROUP_NUM=6"); + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + Set expandedGroups = + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 6) + .keySet(); + // CUSTOM keeps all six slots on the original two groups despite creating four more groups. + Assert.assertEquals( + original, readDataAssignments(client, CUSTOM_DATABASE, devices, TIME_PARTITION_INTERVAL)); + + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + writeAtTime(statement, devices, 2 * TIME_PARTITION_INTERVAL, 200); + Assert.assertEquals( + expandedGroups, + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 6) + .keySet()); + Map rebalanced = + readDataAssignments(client, CUSTOM_DATABASE, devices, 2 * TIME_PARTITION_INTERVAL); + assertSlotDistribution(expandedGroups, new ArrayList<>(rebalanced.values())); + + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + writeAtTime(statement, devices, 3 * TIME_PARTITION_INTERVAL, 300); + Assert.assertEquals( + rebalanced, + readDataAssignments(client, CUSTOM_DATABASE, devices, 3 * TIME_PARTITION_INTERVAL)); + Assert.assertEquals(original, readDataAssignments(client, CUSTOM_DATABASE, devices, 0)); + Assert.assertEquals( + original, readDataAssignments(client, CUSTOM_DATABASE, devices, TIME_PARTITION_INTERVAL)); + for (int time = 0; time <= 3; time++) { + assertValuesAtTime(statement, devices, time * TIME_PARTITION_INTERVAL, time * 100); + } + } + } + + /** + * Verify PROACTIVE-to-CUSTOM growth to the cap: grow from six to nine groups, then to ten after + * raising the cap. The six slots keep their owners; also verify rejection of a lower cap and SQL + * values. + */ + @Test + public void testProactiveToCustomHotReloadAllocatesMaximum() throws Exception { + // Step 1: quota 9 per node gives cap=ceil(9*3/(1*3))=9; PROACTIVE activates only 6 groups. + // At t=0, six distinct slots each own one group: 6 data partitions and 6*3=18 data replicas. + initAutoCluster(3, 9); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + // Expect successful creation (200): minimums 1/2 are valid, and three nodes support three + // replicas. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(PROACTIVE_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2))); + // Expect both caps to be 9: the explicit per-node quota is 9, so ceil(9*3/(1*3))=9. + assertMaximum(client, PROACTIVE_DATABASE, 9); + List devices = createSixDevices(statement, PROACTIVE_DATABASE); + Set oldGroups = + // Expect 6 Data groups and 18 replicas: PROACTIVE creates groups for six active + // slots, below the cap of 9. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, DEVICE_COUNT) + .keySet(); + Map original = + readDataAssignments(client, PROACTIVE_DATABASE, devices, 0); + // Expect one slot in each of six groups: six active slots are evenly assigned to six + // PROACTIVE groups. + assertSlotDistribution(oldGroups, new ArrayList<>(original.values())); + + // Step 2: CUSTOM inherits the saved cap of 9; changing the policy alone leaves 6 groups. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + // Expect both caps to remain 9: CUSTOM uses the database's current cap rather than + // resetting to a default minimum. + assertMaximum(client, PROACTIVE_DATABASE, 9); + // Expect the original six IDs: changing the policy alone does not create the three + // additional groups. + Assert.assertEquals( + oldGroups, + // Expect 6 Data groups: the missing t=10 partition has not yet been requested, so + // CUSTOM has not extended. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, DEVICE_COUNT) + .keySet()); + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + Set customGroups = + // Expect 9 Data groups and 27 replicas: the first missing t=10 partition makes CUSTOM + // fill the cap of 9. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, 9) + .keySet(); + // Expect true: the nine groups must include the original six, adding exactly 9-6=3 + // groups. + Assert.assertTrue(customGroups.containsAll(oldGroups)); + // Step 3: t=10 triggers 9-6=3 new groups (27 replicas total), giving 12 data partitions. + // CUSTOM retains floor(1000/9)=111 slots per group, so each old group's single slot stays. + // All six slots keep their original groups, leaving the three new groups unused for now. + // Expect all six slots to retain their owners: CUSTOM's threshold floor(1000/9)=111 + // exceeds one slot per old group. + Assert.assertEquals( + original, + readDataAssignments(client, PROACTIVE_DATABASE, devices, TIME_PARTITION_INTERVAL)); + + // Step 4: repeating CUSTOM and writing t=20 preserves 9 groups / 6 slots / the same owners. + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='CUSTOM'"); + writeAtTime(statement, devices, 2 * TIME_PARTITION_INTERVAL, 200); + // Expect the same nine IDs: reapplying CUSTOM finds both the cap and current group count + // already satisfied. + Assert.assertEquals( + customGroups, + // Expect 9 Data groups: t=20 only adds a time partition without raising the cap. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, 9) + .keySet()); + // Expect t=20 routing to match the original six groups: one slot per group remains below + // threshold 111. + Assert.assertEquals( + original, + readDataAssignments(client, PROACTIVE_DATABASE, devices, 2 * TIME_PARTITION_INTERVAL)); + + // Step 5: explicitly raise the CUSTOM cap to 10; t=30 creates one more group (30 replicas). + // mu=1000/10=100 still retains all six slot owners; 6 slots * 4 times = 24 data partitions. + statement.execute( + "ALTER DATABASE " + PROACTIVE_DATABASE + " WITH MAX_DATA_REGION_GROUP_NUM=10"); + writeAtTime(statement, devices, 3 * TIME_PARTITION_INTERVAL, 300); + // Expect true: extending the cap from 9 to 10 must preserve all nine existing groups. + Assert.assertTrue( + // Expect 10 Data groups and 30 replicas: allocation at t=30 fills the new explicit + // CUSTOM cap. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, 10) + .keySet() + .containsAll(customGroups)); + // Expect unchanged routing: the new threshold 1000/10=100 still exceeds one slot per old + // group. + Assert.assertEquals( + original, + readDataAssignments(client, PROACTIVE_DATABASE, devices, 3 * TIME_PARTITION_INTERVAL)); + // Step 6: lowering the cap to 2 must fail; check 10 groups remain and all four times are + // readable. + // Expect SQLException: requested cap 2 is below the current cap and existing group count + // of 10. + Assert.assertThrows( + SQLException.class, + () -> + statement.execute( + "ALTER DATABASE " + PROACTIVE_DATABASE + " WITH MAX_DATA_REGION_GROUP_NUM=2")); + // Expect Schema cap 9 and Data cap 10: only the Data cap increased, and the rejected + // decrease changes nothing. + assertMaximum(client, PROACTIVE_DATABASE, 9, 10); + // Expect 10 Data groups and 30 replicas: rejecting the lower cap must not delete existing + // groups. + assertRegionGroupsAndReplicas(client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, 10); + // Expect the six t=0 mappings to stay unchanged through growth to 9/10 groups and the + // rejected decrease. + Assert.assertEquals(original, readDataAssignments(client, PROACTIVE_DATABASE, devices, 0)); + for (int time = 0; time <= 3; time++) { + // Expect one row per device at t=0/10/20/30, with slot+0/100/200/300 from the four + // writes. + assertValuesAtTime(statement, devices, time * TIME_PARTITION_INTERVAL, time * 100); + } + } + } + + /** + * Verify independent Schema policy reload. Six CUSTOM slots initially share one group; after + * switching to PROACTIVE, new slot 6 triggers growth to cap 6. Existing schema mappings and the + * two Data groups remain unchanged. + */ + @Test + public void testSchemaCustomToProactiveHotReloadRefreshesMaximum() throws Exception { + // Step 1: schema CUSTOM cap=1, data AUTO cap=6; six schema slots all belong to one schema + // group. + initAutoCluster(3); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET CONFIGURATION 'schema_region_group_extension_policy'='CUSTOM'"); + // Expect successful creation (200): CUSTOM Schema cap 1 equals its minimum; Data minimum + // 2 is valid. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(CUSTOM_DATABASE) + .setMinSchemaRegionGroupNum(1) + .setMaxSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2))); + // Expect Schema cap 1 and Data cap 6: the former is explicit CUSTOM; the latter uses + // ceil(6*3/(1*3)). + assertMaximum(client, CUSTOM_DATABASE, 1, 6); + List devices = createSixDevices(statement, CUSTOM_DATABASE); + TSchemaPartitionTableResp original = client.getSchemaPartitionTable(schemaRequest(devices)); + // Expect successful query (200): schema partitions for all six existing measurements must + // be readable. + assertSuccess(original.getStatus()); + Set oldGroups = + // Expect 1 Schema group and 3 replicas: CUSTOM cap 1 places all six schema slots in + // that group. + assertRegionGroupsAndReplicas( + client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 1) + .keySet(); + statement.execute("SET CONFIGURATION 'schema_region_group_extension_policy'='PROACTIVE'"); + // Expect the original Schema group ID: reloading the policy has not yet requested a new + // schema slot. + Assert.assertEquals( + oldGroups, + // Expect 1 Schema group: extension waits for the seventh device's new slot. + assertRegionGroupsAndReplicas( + client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 1) + .keySet()); + // Step 2: reload only the schema policy, then create a seventh device in a seventh slot. + // Target=min(7 schema slots, cap 6)=6 groups: add 5 groups, while the original six slots stay + // put. + // The new device has no data write, so data stays at 6 active slots and 2 groups. + String newDevice = findDeviceInSlot(CUSTOM_DATABASE, DEVICE_COUNT); + statement.execute("CREATE TIMESERIES " + newDevice + ".s WITH DATATYPE=INT32, ENCODING=RLE"); + // Expect true: the six Schema groups must include the original group, preserving its + // existing mappings. + Assert.assertTrue( + // Expect 6 Schema groups and 18 replicas: the new device raises active slots to 7, so + // min(7,cap 6)=6. + assertRegionGroupsAndReplicas( + client, CUSTOM_DATABASE, TConsensusGroupType.SchemaRegion, 6) + .keySet() + .containsAll(oldGroups)); + // Expect both caps to be 6: Schema resumes resource-based calculation; the Data cap stays + // unchanged. + assertMaximum(client, CUSTOM_DATABASE, 6); + TSchemaPartitionTableResp restored = client.getSchemaPartitionTable(schemaRequest(devices)); + // Step 3: verify the original schema mappings and SQL values survive the schema-only change. + // Expect successful query (200): all six original devices' schema partitions remain + // readable after extension. + assertSuccess(restored.getStatus()); + // Expect unchanged mappings for the original six schema slots: extension assigns new + // slots without moving old ones. + Assert.assertEquals(original.getSchemaPartitionTable(), restored.getSchemaPartitionTable()); + // Expect 2 Data groups and 6 replicas: Data stays AUTO, and no data was written to the + // seventh device. + assertRegionGroupsAndReplicas(client, CUSTOM_DATABASE, TConsensusGroupType.DataRegion, 2); + // Expect one t=0 row per original device with values 0..5: Schema reload must preserve + // existing reads. + assertValuesAtTime(statement, devices, 0, 0); + } + } + + /** + * Initialize AUTO with an explicit per-node quota of 6. For example, dataNodeCount=3 starts 1C3D + * with three replicas per Region type and 1000 series slots. + */ + private void initAutoCluster(int dataNodeCount) throws Exception { + initAutoCluster(dataNodeCount, 6); + } + + /** + * Initialize AUTO with a specified node quota. For example, three nodes, regionPerDataNode=9 and + * one database with three replicas yield ceil(9*3/3)=9 groups. The quota is not a simulated CPU + * count. + */ + private void initAutoCluster(int dataNodeCount, int regionPerDataNode) throws Exception { + // Shared setup: 1 ConfigNode, N DataNodes, 3 replicas, INHERIT, and 10 ms time partitions. + // Per-type cap depends on regionPerDataNode (normally 6), node count, databases and replicas. + // A nonzero Data quota is used directly without the 0.5 CPU multiplier; quota 6 matches + // automatic sizing with 12 reported cores per node. + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS) + .setSchemaReplicationFactor(REPLICATION_FACTOR) + .setDataReplicationFactor(REPLICATION_FACTOR) + .setSchemaRegionGroupExtensionPolicy("AUTO") + .setDataRegionGroupExtensionPolicy("AUTO") + .setDataPartitionAllocationStrategy("INHERIT") + .setSchemaRegionPerDataNode(regionPerDataNode) + .setDataRegionPerDataNode(regionPerDataNode) + .setSeriesSlotNum(SERIES_SLOT_NUM) + .setSeriesPartitionExecutorClass(BKDRHashExecutor.class.getName()) + .setTimePartitionInterval(TIME_PARTITION_INTERVAL); + EnvFactory.getEnv().initClusterEnvironment(1, dataNodeCount); + } + + /** + * Run the two-database distribution, reload, cap and recovery scenarios. For example, on 3DN each + * cap is 3: AUTO uses two Data groups of three slots, then PROACTIVE uses three groups of two + * while preserving historical mappings. + */ + private void checkSparseDeviceDistribution(int dataNodeCount) throws Exception { + initAutoCluster(dataNodeCount); + Map>> + originalAutoPartitions; + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // Step 1: create both databases before allocating any regions, ensuring the same cap for + // each. + // Cap per type = ceil(6 groups per node * N nodes / (2 databases * 3 replicas)) = N. + // Thus the 3D / 6D / 9D cases have caps of 3 / 6 / 9; minimums are schema=1 and data=2. + for (String database : Arrays.asList(AUTO_DATABASE, PROACTIVE_DATABASE)) { + // Expect both creations to succeed (200): minimums 1/2 are valid, and 3/6/9DN all + // support three replicas. + assertSuccess( + client.setDatabase( + new TDatabaseSchema(database) + .setMinSchemaRegionGroupNum(1) + .setMinDataRegionGroupNum(2))); + } + // Expect both root.auto caps to be 3/6/9 on 3/6/9DN: ceil(6*N/(2 databases*3 + // replicas))=N. + assertMaximum(client, AUTO_DATABASE, dataNodeCount); + // Expect root.proactive caps to be 3/6/9 as well: both databases share the same resource + // calculation. + assertMaximum(client, PROACTIVE_DATABASE, dataNodeCount); + + // Step 2: AUTO database, t=0/10: 6 slots, 1 schema group, 2 data groups, 12 data partitions. + checkSixDevices(client, statement, AUTO_DATABASE, dataNodeCount, false); + + // Step 3: switch both policies, then populate the second database with the same workload. + // K = min(6, N) gives 3 / 6 / 6 groups per type for 3D / 6D / 9D, respectively. + statement.execute("SET CONFIGURATION 'schema_region_group_extension_policy'='PROACTIVE'"); + statement.execute("SET CONFIGURATION 'data_region_group_extension_policy'='PROACTIVE'"); + // Expect caps to remain 3/6/9: AUTO and PROACTIVE share the resource formula; nodes and + // database count are unchanged. + assertMaximum(client, PROACTIVE_DATABASE, dataNodeCount); + checkSixDevices(client, statement, PROACTIVE_DATABASE, dataNodeCount, true); + + // Step 4: write the existing AUTO database's six devices at t=20 after the global switch. + // Its data groups grow 2 -> K (3 / 6 / 6); six slots spread as 2 / 1 / 1 slots per group. + // Its schema group stays at 1 because these devices need no new schema partitions. + // Verify all K data groups are used and t=0 keeps its original group IDs (no data migration). + List autoDevices = generateDeviceNamesForSlots(AUTO_DATABASE, DEVICE_COUNT); + originalAutoPartitions = readDataPartitions(client, AUTO_DATABASE, autoDevices, 0); + writeAtTime(statement, autoDevices, 2 * TIME_PARTITION_INTERVAL, 200); + // Expect 3/6/6 Data groups on 3/6/9DN, with 2/1/1 slots per group: target=min(6,N). + // Expect replicas on all 3/6/9 nodes, with 3/3/2 per node: divide 9/18/18 total replicas + // by node count. + assertNewTimeDistribution( + client, AUTO_DATABASE, autoDevices, 2 * TIME_PARTITION_INTERVAL, dataNodeCount); + // Expect one t=20 row per device with slot+200 (200..205): identical writes on 3/6/9DN. + assertValuesAtTime(statement, autoDevices, 2 * TIME_PARTITION_INTERVAL, 200); + // Expect the six t=0 mappings to match the snapshot: growing to three or six groups must + // preserve history. + Assert.assertEquals( + originalAutoPartitions, readDataPartitions(client, AUTO_DATABASE, autoDevices, 0)); + + // Step 5: add distinct slots starting at 6 to the PROACTIVE database only. + // The 3D / 6D / 9D cases add 2 / 2 / 5 slots, giving 8 / 8 / 11 active slots in total. + // Both region types reach/stay at their caps of 3 / 6 / 9 despite exceeding the cap in slots. + List extraDevices = new ArrayList<>(); + for (int slot = DEVICE_COUNT; slot < Math.max(DEVICE_COUNT, dataNodeCount) + 2; slot++) { + extraDevices.add(findDeviceInSlot(PROACTIVE_DATABASE, slot)); + } + TSchemaPartitionTableResp schemaResponse = + client.getOrCreateSchemaPartitionTable(schemaRequest(extraDevices)); + // Expect schema allocation to succeed (200): assign the extra distinct slots, reusing + // groups at the cap. + assertSuccess(schemaResponse.getStatus()); + TDataPartitionTableResp dataResponse = + client.getOrCreateDataPartitionTable(dataRequest(PROACTIVE_DATABASE, extraDevices, 0)); + // Expect data allocation to succeed (200): once capped, new slots must still be assigned + // to existing groups. + assertSuccess(dataResponse.getStatus()); + // Expect 2/2/5 series slots on 3/6/9DN: this request contains only the extra devices, not + // the original six. + Assert.assertEquals( + extraDevices.size(), dataResponse.getDataPartitionTable().get(PROACTIVE_DATABASE).size()); + // Expect 3/6/9 Schema groups: 8/8/11 active slots exceed the respective caps of 3/6/9. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.SchemaRegion, dataNodeCount); + // Expect 3/6/9 Data groups: the cap prevents growing to match all 8/8/11 active slots. + assertRegionGroupsAndReplicas( + client, PROACTIVE_DATABASE, TConsensusGroupType.DataRegion, dataNodeCount); + } + + // Step 6: restart the ConfigNode and rebuild its in-memory allot map from persisted partitions. + // In root.auto, t=30 must still use K=3/6/6 data groups for the same 6 slots (24 partitions + // total). + // Wait for all K * 3 replicas to become Running, then check routing, SQL values and t=0 + // history. + EnvFactory.getEnv().shutdownConfigNode(0); + EnvFactory.getEnv().startConfigNode(0); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection(); + Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // Expect all replicas of 3/6/6 groups to recover on 3/6/9DN, yielding 9/18/18 Running + // records. + awaitRunningDataRegions(client, AUTO_DATABASE, Math.min(DEVICE_COUNT, dataNodeCount)); + List autoDevices = generateDeviceNamesForSlots(AUTO_DATABASE, DEVICE_COUNT); + writeAtTime(statement, autoDevices, 3 * TIME_PARTITION_INTERVAL, 300); + // Expect 3/6/6 Data groups with 2/1/1 slots each after restart: root.auto still has six + // active slots. + // Expect replicas on all 3/6/9 nodes, with 3/3/2 per node: divide 9/18/18 total replicas + // by node count. + assertNewTimeDistribution( + client, AUTO_DATABASE, autoDevices, 3 * TIME_PARTITION_INTERVAL, dataNodeCount); + // Expect one t=30 row per device with slot+300 (300..305), verifying routing and reads + // after recovery. + assertValuesAtTime(statement, autoDevices, 3 * TIME_PARTITION_INTERVAL, 300); + // Expect the six t=0 mappings to remain unchanged: rebuilding the allocation map must + // preserve persisted history. + Assert.assertEquals( + originalAutoPartitions, readDataPartitions(client, AUTO_DATABASE, autoDevices, 0)); + } + } + + /** + * Create six devices in distinct slots one by one and write t=0 and t=10. Verify groups, + * replicas, routing and SQL values. For example, on 3DN PROACTIVE ends with three groups of two + * slots; AUTO Data uses two groups of three. + */ + private void checkSixDevices( + SyncConfigNodeIServiceClient client, + Statement statement, + String database, + int dataNodeCount, + boolean proactive) + throws Exception { + // Insert devices one at a time so each iteration activates exactly one distinct series slot. + // At iteration i (1..6): AUTO schema=1, data=min(i,2); PROACTIVE schema=data=min(i,N). + List devices = new ArrayList<>(); + for (int slot = 0; slot < DEVICE_COUNT; slot++) { + String device = findDeviceInSlot(database, slot); + devices.add(device); + statement.execute("CREATE TIMESERIES " + device + ".s WITH DATATYPE=INT32, ENCODING=RLE"); + statement.execute("INSERT INTO " + device + "(time,s) VALUES (0," + slot + ")"); + int expectedSchemaGroups = proactive ? Math.min(slot + 1, dataNodeCount) : 1; + int expectedDataGroups = + proactive ? Math.min(slot + 1, dataNodeCount) : Math.min(slot + 1, 2); + // Let i=slot+1. Expect one AUTO Schema group: its minimum is 1 and six slots do not + // trigger proportional growth. + // Expect PROACTIVE counts 1,2,3,3,3,3 on 3DN and 1,2,3,4,5,6 on 6/9DN: min(i,N). + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.SchemaRegion, expectedSchemaGroups); + // Expect AUTO Data counts 1,2,2,2,2,2: successive requests gradually reach the minimum of + // 2. + // Expect PROACTIVE counts 1,2,3,3,3,3 on 3DN and 1,2,3,4,5,6 on 6/9DN, bounded by active + // slots and the cap. + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.DataRegion, expectedDataGroups); + } + + // After t=0: 6 measurements, 6 schema slots and 6 (series slot, time slot) data partitions. + // For each type, G logical groups produce G * 3 replica rows on distinct nodes within a group. + int expectedSchemaGroups = proactive ? Math.min(DEVICE_COUNT, dataNodeCount) : 1; + int expectedDataGroups = proactive ? Math.min(DEVICE_COUNT, dataNodeCount) : 2; + Map> schemaReplicas = + // Expect Schema counts: AUTO 1 on all 3/6/9DN cases; PROACTIVE 3/6/6, or min(6 + // slots,N). + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.SchemaRegion, expectedSchemaGroups); + Map> dataReplicas = + // Expect Data counts: AUTO 2 on all 3/6/9DN cases; PROACTIVE 3/6/6 after activating all + // six slots. + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.DataRegion, expectedDataGroups); + // Expect Schema replicas to cover 3/3/3 nodes for AUTO (one three-replica group), and 3/6/9 + // for PROACTIVE. + // Expect 3/3/2 replicas of this type per node for PROACTIVE: 9/18/18 replicas divided by + // 3/6/9 nodes. + assertReplicaDistribution( + schemaReplicas, proactive ? dataNodeCount : REPLICATION_FACTOR, proactive); + // Expect Data replicas to cover 3/6/6 nodes for AUTO (six replicas across two groups), and + // 3/6/9 for PROACTIVE. + // Expect 3/3/2 replicas per node for PROACTIVE; AUTO checks node coverage here without + // requiring balance. + assertReplicaDistribution( + dataReplicas, + proactive ? dataNodeCount : Math.min(dataNodeCount, 2 * REPLICATION_FACTOR), + proactive); + + // Check actual partition owners as well as created groups: no expected group may be unused. + // AUTO data distribution is 3+3; PROACTIVE is 2+2+2 (3D), or six groups with one slot (6D/9D). + TSchemaPartitionTableResp schemaResponse = + client.getSchemaPartitionTable(schemaRequest(devices)); + // Expect successful schema query (200): all six measurements exist on every 3/6/9DN + // configuration. + assertSuccess(schemaResponse.getStatus()); + Map schemaPartitions = + schemaResponse.getSchemaPartitionTable().get(database); + // Expect exactly 6 schema slots: hash-selected device names avoid collisions regardless of + // node count. + Assert.assertEquals(DEVICE_COUNT, schemaPartitions.size()); + // Expect Schema slots: AUTO has one group of 6; PROACTIVE has three groups of 2 on 3DN or + // six of 1 on 6/9DN. + assertSlotDistribution(schemaReplicas.keySet(), new ArrayList<>(schemaPartitions.values())); + Map>> firstPartitions = + readDataPartitions(client, database, devices, 0); + // Expect Data slots: AUTO has two groups of 3; PROACTIVE has three groups of 2 on 3DN or + // six of 1 on 6/9DN. + assertSlotDistribution(dataReplicas.keySet(), groupsAtTime(firstPartitions, 0)); + + // At t=10: 6 series slots * 2 time slots = 12 data partitions; schema partitions stay at 6. + // No extra groups are needed here; verify unchanged replica placement and per-slot routing. + writeAtTime(statement, devices, TIME_PARTITION_INTERVAL, 100); + // Expect identical Schema groups and replica placements to t=0: writing t=10 adds no + // devices or schema slots. + Assert.assertEquals( + schemaReplicas, + // Expect Schema counts AUTO=1 or PROACTIVE=3/6/6 on 3/6/9DN: schema partitions are + // unchanged. + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.SchemaRegion, expectedSchemaGroups)); + // Expect identical Data groups and replica placements to t=0: the same six series slots + // need no further growth. + Assert.assertEquals( + dataReplicas, + // Expect Data counts AUTO=2 or PROACTIVE=3/6/6 on 3/6/9DN: only the time slot is new. + assertRegionGroupsAndReplicas( + client, database, TConsensusGroupType.DataRegion, expectedDataGroups)); + Map>> secondPartitions = + readDataPartitions(client, database, devices, TIME_PARTITION_INTERVAL); + firstPartitions.forEach( + (slot, times) -> + // Expect each slot's owner at t=10 to match t=0: no new group triggers rebalancing, + // so INHERIT keeps its mapping. + Assert.assertEquals( + times.get(new TTimePartitionSlot(0)), + secondPartitions.get(slot).get(new TTimePartitionSlot(TIME_PARTITION_INTERVAL)))); + + // Finally verify SQL reads: exactly two rows per device, values slotId and slotId + 100. + for (int slot = 0; slot < DEVICE_COUNT; slot++) { + try (ResultSet rows = statement.executeQuery("SELECT s FROM " + devices.get(slot))) { + // Expect true: each device has a t=0 row on all 3/6/9DN configurations. + Assert.assertTrue(rows.next()); + // Expect first timestamp 0: default ascending time order returns the earliest write + // first. + Assert.assertEquals(0, rows.getLong(1)); + // Expect first value slot (0..5), matching this device's INSERT at t=0. + Assert.assertEquals(slot, rows.getInt(2)); + // Expect true: each device also has a t=10 write, so a second row must exist. + Assert.assertTrue(rows.next()); + // Expect second timestamp 10: TIME_PARTITION_INTERVAL=10 places the second write in the + // next partition. + Assert.assertEquals(TIME_PARTITION_INTERVAL, rows.getLong(1)); + // Expect second value slot+100 (100..105), matching the INSERT at t=10. + Assert.assertEquals(slot + 100, rows.getInt(2)); + // Expect false: only t=0 and t=10 were written, so no third or duplicate row may exist. + Assert.assertFalse(rows.next()); + } + } + } + + /** + * Select device names with the real BKDR hash; this only generates names, without creating + * measurements or partitions or mocking the executor. + * + *

For database=root.auto and slotCount=3, return three names whose indices i hash to slots i: + * 0, 1 and 2. Their d suffixes need not be 0, 1 and 2. + * + * @param database database containing the device paths + * @param slotCount number of distinct series slots to cover, from 0 to SERIES_SLOT_NUM; target + * slots are 0 through slotCount-1 + * @return names ordered by target slot, for subsequent SQL/RPC calls to allocate actual + * partitions + */ + private static List generateDeviceNamesForSlots(String database, int slotCount) { + List devices = new ArrayList<>(); + for (int slot = 0; slot < slotCount; slot++) { + devices.add(findDeviceInSlot(database, slot)); + } + return devices; + } + + /** + * Create six devices with one INT32 measurement s each and write initial data. For example, the + * devices hash to slots 0..5 and receive values 0..5 at t=0. + */ + private static List createSixDevices(Statement statement, String database) + throws Exception { + List devices = generateDeviceNamesForSlots(database, DEVICE_COUNT); + for (String device : devices) { + statement.execute("CREATE TIMESERIES " + device + ".s WITH DATATYPE=INT32, ENCODING=RLE"); + } + writeAtTime(statement, devices, 0, 0); + return devices; + } + + /** + * Read the unique logical owner of each series slot at a given time partition. For example, six + * devices at t=10 yield six slot-to-DataRegionGroup mappings; three replicas do not count as + * three groups. + */ + private static Map readDataAssignments( + SyncConfigNodeIServiceClient client, String database, List devices, long time) + throws Exception { + Map assignments = new HashMap<>(); + readDataPartitions(client, database, devices, time) + .forEach( + (slot, times) -> { + List groups = times.get(new TTimePartitionSlot(time)); + // Expect one logical Data group per (series slot,time slot), not three replica + // records. + Assert.assertEquals(1, groups.size()); + assignments.put(slot, groups.get(0)); + }); + return assignments; + } + + /** + * Write one value per device, using its list index plus valueOffset. For example, six devices + * with time=10 and valueOffset=100 receive values 100..105. + */ + private static void writeAtTime( + Statement statement, List devices, long time, int valueOffset) throws Exception { + for (int slot = 0; slot < devices.size(); slot++) { + statement.execute( + "INSERT INTO " + + devices.get(slot) + + "(time,s) VALUES (" + + time + + "," + + (slot + valueOffset) + + ")"); + } + } + + /** + * Verify exactly one row per device at the specified time, including timestamp and value. For + * example, six devices with time=20 and valueOffset=200 must return values 200..205; callers + * describe each scenario. + */ + private static void assertValuesAtTime( + Statement statement, List devices, long time, int valueOffset) throws Exception { + for (int slot = 0; slot < devices.size(); slot++) { + try (ResultSet rows = + statement.executeQuery("SELECT s FROM " + devices.get(slot) + " WHERE time=" + time)) { + // Expect true: filtering by the exact time must return the row written for each device, + // regardless of node count. + Assert.assertTrue(rows.next()); + // Expect the supplied timestamp: WHERE time selects the exact write batch requested by + // the caller. + Assert.assertEquals(time, rows.getLong(1)); + // Expect device index slot + valueOffset, matching the writeAtTime rule. + Assert.assertEquals(slot + valueOffset, rows.getInt(2)); + // Expect false: one value was written per device at this time, so no second row may + // exist. + Assert.assertFalse(rows.next()); + } + } + } + + /** + * Check a new time partition in the two-database PROACTIVE scenario, assuming six distinct slots + * and a per-database cap equal to dataNodeCount. For example, 3DN uses three groups of two slots + * and nine replicas across three nodes. + */ + private static void assertNewTimeDistribution( + SyncConfigNodeIServiceClient client, + String database, + List devices, + long time, + int dataNodeCount) + throws Exception { + Map> replicas = + // Expect min(DEVICE_COUNT, dataNodeCount) Data groups; callers list the concrete counts + // for each cluster size. + assertRegionGroupsAndReplicas( + client, + database, + TConsensusGroupType.DataRegion, + Math.min(DEVICE_COUNT, dataNodeCount)); + // Expect replicas on dataNodeCount nodes, with counts differing by at most one. + assertReplicaDistribution(replicas, dataNodeCount, true); + // Expect this time partition to use every expected group, with slot counts differing by at + // most one. + assertSlotDistribution( + replicas.keySet(), groupsAtTime(readDataPartitions(client, database, devices, time), time)); + } + + /** + * Wait for all expected Data replicas in the database to become Running. For example, + * expectedRegionGroupCount=6 with three replicas requires 18 Running records; query at most 30 + * times. + */ + private static void awaitRunningDataRegions( + SyncConfigNodeIServiceClient client, String database, int expectedRegionGroupCount) + throws Exception { + TShowRegionResp response = null; + for (int retry = 0; retry < 30; retry++) { + response = + client.showRegion( + new TShowRegionReq().setConsensusGroupType(TConsensusGroupType.DataRegion)); + // Expect successful SHOW REGIONS (200): statuses may change during recovery, but + // readiness requires a valid query. + assertSuccess(response.getStatus()); + long runningReplicas = + response.getRegionInfoList().stream() + .filter(region -> database.equals(region.getDatabase())) + .filter(region -> RegionStatus.Running.getStatus().equals(region.getStatus())) + .count(); + if (runningReplicas == (long) expectedRegionGroupCount * REPLICATION_FACTOR) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + // Expect expectedRegionGroupCount * REPLICATION_FACTOR Running replicas within 30 attempts. + Assert.fail( + "DataRegion replicas did not become Running after ConfigNode recovery: " + response); + } + + /** + * Enumerate names and use the real BKDR hash to find the target slot. For example, slot=2 returns + * a root.auto.dN that hashes to 2, without assuming d2 hashes to slot 2. + */ + private static String findDeviceInSlot(String database, int slot) { + // Control hashing instead of assuming d0...d5 occupy different slots; measurements share a + // slot. + for (int candidate = 0; candidate < SERIES_SLOT_NUM * 100; candidate++) { + String device = database + ".d" + candidate; + if (PARTITION_EXECUTOR.getSeriesPartitionSlot(device).getSlotId() == slot) { + return device; + } + } + throw new AssertionError("No device found in series slot " + slot); + } + + /** + * Build a schema partition request for measurement s of each device. For example, root.auto.d8 + * becomes root.auto.d8.s; this method only constructs the request. + */ + private static TSchemaPartitionReq schemaRequest(List devices) throws Exception { + return new TSchemaPartitionReq( + generatePatternTreeBuffer( + devices.stream().map(device -> device + ".s").toArray(String[]::new))); + } + + /** + * Build a request for the database, series slots and time partition start. For example, two + * devices in distinct slots and time=10 produce two entries requesting t=10; time must be a + * partition start. + */ + private static TDataPartitionReq dataRequest(String database, List devices, long time) { + Map slots = new HashMap<>(); + for (String device : devices) { + slots.put( + PARTITION_EXECUTOR.getSeriesPartitionSlot(device), + new TTimeSlotList(Collections.singletonList(new TTimePartitionSlot(time)), false, false)); + } + return new TDataPartitionReq(Collections.singletonMap(database, slots)); + } + + /** + * Query allocated data partitions, requiring devices in distinct series slots. For example, six + * distinct slots at time=10 must return six slot entries; callers verify their owners. + */ + private static Map>> + readDataPartitions( + SyncConfigNodeIServiceClient client, String database, List devices, long time) + throws Exception { + TDataPartitionTableResp response = + client.getDataPartitionTable(dataRequest(database, devices, time)); + // Expect successful partition query (200): the requested partitions must already have been + // allocated. + assertSuccess(response.getStatus()); + Map>> partitions = + response.getDataPartitionTable().get(database); + // Expect devices.size() distinct slots: callers must provide devices in different slots + // with existing partitions. + Assert.assertEquals(devices.size(), partitions.size()); + return partitions; + } + + /** + * Extract group IDs at the requested time, retaining duplicates for slot counting. For example, + * six slots owned by A and B may produce three A entries and three B entries. + */ + private static List groupsAtTime( + Map>> partitions, + long time) { + List groups = new ArrayList<>(); + partitions + .values() + .forEach( + times -> { + List regionIds = times.get(new TTimePartitionSlot(time)); + // Expect one group ID per slot/time: this checks logical ownership; the three + // replicas are checked separately. + Assert.assertEquals(1, regionIds.size()); + groups.add(regionIds.get(0)); + }); + return groups; + } + + /** + * Verify that every expected group is used and slot counts differ by at most one. For example, + * expected groups A/B with assignments A,A,A,B,B,B pass; an unused B fails. + */ + private static void assertSlotDistribution( + Set groups, List assigned) { + // One assignment per series slot at a fixed time: all groups must be used, counts differ by + // <=1. + Map counts = new HashMap<>(); + assigned.forEach(group -> counts.merge(group, 1, Integer::sum)); + // Expect exactly the caller's group set to be used: every expected group must receive a + // slot. + Assert.assertEquals(groups, counts.keySet()); + // Expect slot counts to differ by at most one; callers describe the concrete slot and group + // counts. + Assert.assertTrue( + counts.toString(), + Collections.max(counts.values()) - Collections.min(counts.values()) <= 1); + } + + /** + * Check the number of distinct DataNodes hosting replicas and optionally require counts to differ + * by at most one. For example, six groups with three replicas balanced across nine nodes yield + * two replicas per node. + * + * @param replicas group IDs mapped to the DataNode IDs hosting their replicas + * @param expectedHostingDataNodeCount expected number of distinct DataNodes actually hosting + * these replicas + * @param requireBalancedReplicaCounts whether to require balanced replica counts among hosting + * nodes + */ + private static void assertReplicaDistribution( + Map> replicas, + int expectedHostingDataNodeCount, + boolean requireBalancedReplicaCounts) { + Map counts = new HashMap<>(); + replicas.values().forEach(nodes -> nodes.forEach(node -> counts.merge(node, 1, Integer::sum))); + // Expect expectedHostingDataNodeCount distinct DataNodes to host these replicas. + Assert.assertEquals(counts.toString(), expectedHostingDataNodeCount, counts.size()); + if (requireBalancedReplicaCounts) { + // Expect replica counts among hosting nodes to differ by at most one, only when balance + // is requested. + Assert.assertTrue( + counts.toString(), + Collections.max(counts.values()) - Collections.min(counts.values()) <= 1); + } + } + + /** + * Check logical group count, total replica rows and distinct DataNodes per group for one database + * and Region type. + * + *

For root.auto, DataRegion and expectedRegionGroupCount=2, expect two logical groups, 2*3=6 + * replica rows and three distinct DataNodes per group. Slot allocation and balance across groups + * are checked separately. + * + * @param database database whose Regions are counted + * @param regionType SchemaRegion or DataRegion to check + * @param expectedRegionGroupCount expected number of created logical groups for this database and + * type, not nodes or slots + * @return group IDs mapped to the DataNode IDs hosting their replicas + */ + private static Map> assertRegionGroupsAndReplicas( + SyncConfigNodeIServiceClient client, + String database, + TConsensusGroupType regionType, + int expectedRegionGroupCount) + throws Exception { + // SHOW REGIONS returns replicas, not logical groups: deduplicate IDs before checking G and 3*G. + TShowRegionResp response = + client.showRegion(new TShowRegionReq().setConsensusGroupType(regionType)); + // Expect successful SHOW REGIONS (200) before counting replicas by database, type and group + // ID. + assertSuccess(response.getStatus()); + Map> replicas = new HashMap<>(); + int replicaCount = 0; + for (TRegionInfo region : response.getRegionInfoList()) { + if (database.equals(region.getDatabase())) { + replicas + .computeIfAbsent(region.getConsensusGroupId(), ignored -> new HashSet<>()) + .add(region.getDataNodeId()); + replicaCount++; + } + } + // Expect expectedRegionGroupCount distinct groups for the specified database and Region + // type. + Assert.assertEquals(database + " " + regionType, expectedRegionGroupCount, replicas.size()); + // Expect expectedRegionGroupCount * REPLICATION_FACTOR replica rows. + Assert.assertEquals(expectedRegionGroupCount * REPLICATION_FACTOR, replicaCount); + // Expect each group's replicas on REPLICATION_FACTOR distinct DataNodes, preventing + // colocated replicas of one group. + replicas.values().forEach(nodes -> Assert.assertEquals(REPLICATION_FACTOR, nodes.size())); + return replicas; + } + + /** + * Check that both database group caps equal the same expected value. For example, 6 means both + * Schema and Data caps are 6, without requiring six groups to have been created. + */ + private static void assertMaximum( + SyncConfigNodeIServiceClient client, String database, int expectedRegionGroupLimit) + throws Exception { + // Expect both Schema and Data caps to equal the caller's expectedRegionGroupLimit. + assertMaximum(client, database, expectedRegionGroupLimit, expectedRegionGroupLimit); + } + + /** + * Check database group caps independently. For example, expectedSchemaRegionGroupLimit=6 and + * expectedDataRegionGroupLimit=2 require Schema cap 6 and Data cap 2; actual group counts are + * checked separately. + */ + private static void assertMaximum( + SyncConfigNodeIServiceClient client, + String database, + int expectedSchemaRegionGroupLimit, + int expectedDataRegionGroupLimit) + throws Exception { + TDatabaseSchemaResp response = + client.getMatchedDatabaseSchemas( + new TGetDatabaseReq(Arrays.asList(database.split("\\.")), ALL_MATCH_SCOPE_BINARY)); + // Expect successful database metadata query (200) before checking both group caps. + assertSuccess(response.getStatus()); + TDatabaseSchema schema = response.getDatabaseSchemaMap().get(database); + // Expect Schema cap expectedSchemaRegionGroupLimit; a cap is not the number of groups + // already created. + Assert.assertEquals(expectedSchemaRegionGroupLimit, schema.getMaxSchemaRegionGroupNum()); + // Expect Data cap expectedDataRegionGroupLimit, checked independently of the Schema cap. + Assert.assertEquals(expectedDataRegionGroupLimit, schema.getMaxDataRegionGroupNum()); + } + + /** + * Check RPC success; for example, normal database creation returns SUCCESS_STATUS (200). Callers + * assert expected failures separately. + */ + private static void assertSuccess(TSStatus status) { + // Expect SUCCESS_STATUS (200) for normal database creation, allocation and queries; failure + // cases use assertThrows. + Assert.assertEquals( + status.getMessage(), TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupExtensionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupExtensionIT.java new file mode 100644 index 0000000000000..1f2238d29f32b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBProactiveRegionGroupExtensionIT.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.it.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.cluster.RegionStatus; +import org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor; +import org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchemaResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionReq; +import org.apache.iotdb.confignode.rpc.thrift.TSchemaPartitionTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE_BINARY; +import static org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils.generatePatternTreeBuffer; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBProactiveRegionGroupExtensionIT { + + private static final String DATABASE = "root.proactive"; + private static final int SERIES_SLOT_NUM = 32; + private static final int MAX_REGION_GROUP_NUM = 4; + private static final long TIME_PARTITION_INTERVAL = 10; + private static final BKDRHashExecutor PARTITION_EXECUTOR = new BKDRHashExecutor(SERIES_SLOT_NUM); + + @After + public void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testMinimumGrowthCountsOnlyDistinctPendingSlots() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS) + .setDataRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS) + .setSchemaReplicationFactor(1) + .setDataReplicationFactor(1) + .setSchemaRegionGroupExtensionPolicy("PROACTIVE") + .setDataRegionGroupExtensionPolicy("PROACTIVE") + .setSchemaRegionPerDataNode(4) + .setDataRegionPerDataNode(4) + .setTimePartitionInterval(TIME_PARTITION_INTERVAL); + EnvFactory.getEnv().initClusterEnvironment(1, 1); + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + assertSuccess( + client.setDatabase( + new TDatabaseSchema(DATABASE) + .setMinSchemaRegionGroupNum(4) + .setMinDataRegionGroupNum(4))); + Map> schemaRequest = new HashMap<>(); + schemaRequest.put( + DATABASE, Arrays.asList(new TSeriesPartitionSlot(0), new TSeriesPartitionSlot(0))); + assertSuccess(client.getOrCreateSchemaPartitionTableWithSlots(schemaRequest).getStatus()); + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 1, 1); + + createDataPartitions(client, 0, 2, 0, 1); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, 1); + // Slots 0 and 1 already have this time partition; only slot 2 contributes to minimum growth. + createDataPartitions(client, 0, 3, 0, 1); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 3, 1); + createDataPartitions(client, 0, 3, 0, 1); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 3, 1); + // A genuinely new time partition for an existing series slot still grows toward the minimum. + createDataPartitions(client, 2, 3, 1, 2); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 4, 1); + } + } + + @Test + public void testSingleConfigNodeAndDataNode() throws Exception { + checkProactiveExtension( + 1, 1, ConsensusFactory.SIMPLE_CONSENSUS, ConsensusFactory.SIMPLE_CONSENSUS); + } + + @Test + public void testTwoConfigNodesAndDataNodes() throws Exception { + checkProactiveExtension( + 2, 2, ConsensusFactory.RATIS_CONSENSUS, ConsensusFactory.RATIS_CONSENSUS); + } + + @Test + public void testThreeConfigNodesAndDataNodes() throws Exception { + checkProactiveExtension( + 3, 3, ConsensusFactory.RATIS_CONSENSUS, ConsensusFactory.RATIS_CONSENSUS); + } + + @Test + public void testSingleConfigNodeAndThreeDataNodes() throws Exception { + checkProactiveExtension(1, 3, ConsensusFactory.RATIS_CONSENSUS, ConsensusFactory.IOT_CONSENSUS); + } + + @Test + public void testThreeConfigNodesAndSingleDataNode() throws Exception { + checkProactiveExtension( + 3, 1, ConsensusFactory.SIMPLE_CONSENSUS, ConsensusFactory.SIMPLE_CONSENSUS); + } + + private void checkProactiveExtension( + int configNodeCount, int dataNodeCount, String schemaConsensus, String dataConsensus) + throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setConfigNodeConsensusProtocolClass( + configNodeCount == 1 && dataNodeCount == 1 + ? ConsensusFactory.SIMPLE_CONSENSUS + : ConsensusFactory.RATIS_CONSENSUS) + .setSchemaRegionConsensusProtocolClass(schemaConsensus) + .setDataRegionConsensusProtocolClass(dataConsensus) + .setSchemaReplicationFactor(dataNodeCount) + .setDataReplicationFactor(dataNodeCount) + .setSchemaRegionGroupExtensionPolicy("PROACTIVE") + .setDataRegionGroupExtensionPolicy("PROACTIVE") + .setSchemaRegionPerDataNode(MAX_REGION_GROUP_NUM) + .setDataRegionPerDataNode(MAX_REGION_GROUP_NUM) + .setSeriesSlotNum(SERIES_SLOT_NUM) + .setSeriesPartitionExecutorClass(BKDRHashExecutor.class.getName()) + .setTimePartitionInterval(TIME_PARTITION_INTERVAL); + EnvFactory.getEnv().initClusterEnvironment(configNodeCount, dataNodeCount); + + String[] paths = new String[MAX_REGION_GROUP_NUM + 2]; + for (int slot = 0; slot < paths.length; slot++) { + paths[slot] = findPathInSlot(slot, null); + } + + Set schemaGroups; + Set dataGroups; + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + assertSuccess( + client.setDatabase( + new TDatabaseSchema(DATABASE) + .setMinSchemaRegionGroupNum(2) + .setMinDataRegionGroupNum(2))); + assertMaxRegionGroupNum(client); + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 0, dataNodeCount); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 0, dataNodeCount); + + // A minimum of two is reached gradually, adding one group per pending schema slot. + for (int slot = 0; slot < 3; slot++) { + createSchemaPartitions(client, paths[slot]); + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, slot + 1, dataNodeCount); + } + // Measurements, repeated requests and another device hashing to the same slot do not grow. + createSchemaPartitions(client, paths[0], paths[0] + "2", findPathInSlot(0, paths[0])); + schemaGroups = assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 3, dataNodeCount); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 0, dataNodeCount); + + // Activating schema slots must not count as activating data slots. + createDataPartitions(client, 0, 1, 0, 1); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 1, dataNodeCount); + createDataPartitions(client, 0, 1, 0, 1); + // Already assigned partitions do not trigger growth toward the minimum. + assertRegionGroups(client, TConsensusGroupType.DataRegion, 1, dataNodeCount); + createDataPartitions(client, 0, 1, 1, 4); + // New times for the existing series slot complete the minimum, just as with AUTO. + assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount); + createDataPartitions(client, 0, 2, 4, 5); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount); + // New time partitions for two existing slots are still only two activated series slots. + createDataPartitions(client, 0, 2, 5, 8); + dataGroups = assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount); + } + + int oldLeader = EnvFactory.getEnv().getLeaderConfigNodeIndex(); + EnvFactory.getEnv().shutdownConfigNode(oldLeader); + if (configNodeCount < 3) { + // A two-member ConfigNode group needs both members for quorum. + EnvFactory.getEnv().startConfigNode(oldLeader); + } else { + Assert.assertNotEquals(oldLeader, EnvFactory.getEnv().getLeaderConfigNodeIndex()); + } + + try (SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { + // Restart / leader election preserves the groups, counts and resource-derived cap. + assertMaxRegionGroupNum(client); + Assert.assertEquals( + schemaGroups, + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 3, dataNodeCount)); + Assert.assertEquals( + dataGroups, assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount)); + // Leader discovery precedes the heartbeat statistics that make recovered groups available. + awaitRunningRegions(client, (schemaGroups.size() + dataGroups.size()) * dataNodeCount); + createSchemaPartitions(client, paths[0], paths[1], paths[2]); + createDataPartitions(client, 0, 2, 8, 9); + assertRegionGroups(client, TConsensusGroupType.SchemaRegion, 3, dataNodeCount); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 2, dataNodeCount); + + // Batched requests mix existing and new slots, then reach and stay at AUTO's same cap. + createDataPartitions(client, 0, 3, 9, 10); + assertRegionGroups(client, TConsensusGroupType.DataRegion, 3, dataNodeCount); + createSchemaPartitions(client, paths); + createDataPartitions(client, 0, paths.length, 10, 11); + assertRegionGroups( + client, TConsensusGroupType.SchemaRegion, MAX_REGION_GROUP_NUM, dataNodeCount); + assertRegionGroups( + client, TConsensusGroupType.DataRegion, MAX_REGION_GROUP_NUM, dataNodeCount); + createSchemaPartitions(client, findPathInSlot(paths.length, null)); + createDataPartitions(client, 0, paths.length + 1, 11, 12); + assertRegionGroups( + client, TConsensusGroupType.SchemaRegion, MAX_REGION_GROUP_NUM, dataNodeCount); + assertRegionGroups( + client, TConsensusGroupType.DataRegion, MAX_REGION_GROUP_NUM, dataNodeCount); + } + } + + private static void awaitRunningRegions( + SyncConfigNodeIServiceClient client, int expectedReplicaCount) throws Exception { + TShowRegionResp response = null; + for (int retry = 0; retry < 30; retry++) { + response = client.showRegion(new TShowRegionReq()); + assertSuccess(response.getStatus()); + int replicaCount = 0; + int runningReplicaCount = 0; + for (TRegionInfo region : response.getRegionInfoList()) { + if (DATABASE.equals(region.getDatabase())) { + replicaCount++; + if (RegionStatus.Running.getStatus().equals(region.getStatus())) { + runningReplicaCount++; + } + } + } + if (replicaCount == expectedReplicaCount && runningReplicaCount == expectedReplicaCount) { + return; + } + TimeUnit.SECONDS.sleep(1); + } + Assert.fail("Regions did not become Running after ConfigNode recovery: " + response); + } + + private static String findPathInSlot(int slot, String excludedPath) { + for (int device = 0; device < SERIES_SLOT_NUM * 100; device++) { + String devicePath = DATABASE + ".d" + device; + String path = devicePath + ".s"; + if (!path.equals(excludedPath) + && PARTITION_EXECUTOR.getSeriesPartitionSlot(devicePath).getSlotId() == slot) { + return path; + } + } + throw new AssertionError("No device found in series slot " + slot); + } + + private static void createSchemaPartitions(SyncConfigNodeIServiceClient client, String... paths) + throws Exception { + TSchemaPartitionTableResp response = + client.getOrCreateSchemaPartitionTable( + new TSchemaPartitionReq(generatePatternTreeBuffer(paths))); + assertSuccess(response.getStatus()); + Set expectedSlots = new HashSet<>(); + for (String path : paths) { + expectedSlots.add( + PARTITION_EXECUTOR.getSeriesPartitionSlot(path.substring(0, path.lastIndexOf('.')))); + } + Assert.assertEquals(expectedSlots, response.getSchemaPartitionTable().get(DATABASE).keySet()); + } + + private static void createDataPartitions( + SyncConfigNodeIServiceClient client, int slotStart, int slotEnd, int timeStart, int timeEnd) + throws Exception { + TDataPartitionTableResp response = + client.getOrCreateDataPartitionTable( + new TDataPartitionReq( + ConfigNodeTestUtils.constructPartitionSlotsMap( + DATABASE, slotStart, slotEnd, timeStart, timeEnd, TIME_PARTITION_INTERVAL))); + assertSuccess(response.getStatus()); + ConfigNodeTestUtils.checkDataPartitionTable( + DATABASE, + slotStart, + slotEnd, + timeStart, + timeEnd, + TIME_PARTITION_INTERVAL, + response.getDataPartitionTable()); + } + + private static void assertMaxRegionGroupNum(SyncConfigNodeIServiceClient client) + throws Exception { + TDatabaseSchemaResp response = + client.getMatchedDatabaseSchemas( + new TGetDatabaseReq(Arrays.asList("root", "proactive"), ALL_MATCH_SCOPE_BINARY)); + assertSuccess(response.getStatus()); + TDatabaseSchema schema = response.getDatabaseSchemaMap().get(DATABASE); + // One database: per-node quota * DataNode count / replication factor = 4, as with AUTO. + Assert.assertEquals(MAX_REGION_GROUP_NUM, schema.getMaxSchemaRegionGroupNum()); + Assert.assertEquals(MAX_REGION_GROUP_NUM, schema.getMaxDataRegionGroupNum()); + } + + private static Set assertRegionGroups( + SyncConfigNodeIServiceClient client, + TConsensusGroupType type, + int expectedGroupCount, + int replicationFactor) + throws Exception { + TShowRegionResp response = client.showRegion(new TShowRegionReq().setConsensusGroupType(type)); + assertSuccess(response.getStatus()); + Map> replicas = new HashMap<>(); + int replicaCount = 0; + for (TRegionInfo region : response.getRegionInfoList()) { + if (DATABASE.equals(region.getDatabase())) { + replicas + .computeIfAbsent(region.getConsensusGroupId(), ignored -> new HashSet<>()) + .add(region.getDataNodeId()); + replicaCount++; + } + } + Assert.assertEquals(type.toString(), expectedGroupCount, replicas.size()); + Assert.assertEquals(expectedGroupCount * replicationFactor, replicaCount); + replicas.values().forEach(nodes -> Assert.assertEquals(replicationFactor, nodes.size())); + return replicas.keySet(); + } + + private static void assertSuccess(TSStatus status) { + Assert.assertEquals( + status.getMessage(), TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java index 8c816f61fcda7..a9b0fbea52d9e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java @@ -1047,7 +1047,13 @@ public void testShowDevicesWithLimit() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { try { - List exps = Arrays.asList("root.sg1.d0,false", "root.sg1.d1,false"); + List exps = + new ArrayList<>( + Arrays.asList( + "root.sg1.d0,false", + "root.sg1.d1,false", + "root.sg1.d2,false", + "root.sg1.d3,false")); statement.execute("INSERT INTO root.sg1.d0(timestamp, s1) VALUES (5, 5)"); statement.execute("INSERT INTO root.sg1.d1(timestamp, s2) VALUES (5, 5)"); @@ -1057,8 +1063,8 @@ public void testShowDevicesWithLimit() throws SQLException { int count = 0; try (ResultSet resultSet = statement.executeQuery("show devices limit 2")) { while (resultSet.next()) { - Assert.assertEquals( - exps.get(count), resultSet.getString(1) + "," + resultSet.getString(2)); + String device = resultSet.getString(1) + "," + resultSet.getString(2); + Assert.assertTrue(device, exps.remove(device)); ++count; } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java index 7243ca5736657..1b293f75865c3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiTAGsWithAttributesTableIT.java @@ -1380,14 +1380,22 @@ public void maxByMinByExtremeTest() { tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); expectedHeader = buildHeaders(30); + // Exclude l4 so both blob extrema are unique; tied rows may arrive in either order. sql = - "select max_by(time,blob),max_by(device,blob),max_by(level,blob),max_by(attr1,blob),max_by(attr2,blob),max_by(num,blob),max_by(bignum,blob),max_by(floatnum,blob),max_by(str,blob),max_by(bool,blob),max_by(date,blob),max_by(blob,blob),max_by(ts,blob),max_by(stringv,blob),max_by(doubleNum,blob),min_by(time,blob),min_by(device,blob),min_by(level,blob),min_by(attr1,blob),min_by(attr2,blob),min_by(num,blob),min_by(bignum,blob),min_by(floatnum,blob),min_by(str,blob),min_by(bool,blob),min_by(date,blob),min_by(blob,blob),min_by(ts,blob),min_by(stringv,blob),min_by(doubleNum,blob) from table0"; + "select max_by(time,blob),max_by(device,blob),max_by(level,blob),max_by(attr1,blob),max_by(attr2,blob),max_by(num,blob),max_by(bignum,blob),max_by(floatnum,blob),max_by(str,blob),max_by(bool,blob),max_by(date,blob),max_by(blob,blob),max_by(ts,blob),max_by(stringv,blob),max_by(doubleNum,blob),min_by(time,blob),min_by(device,blob),min_by(level,blob),min_by(attr1,blob),min_by(attr2,blob),min_by(num,blob),min_by(bignum,blob),min_by(floatnum,blob),min_by(str,blob),min_by(bool,blob),min_by(date,blob),min_by(blob,blob),min_by(ts,blob),min_by(stringv,blob),min_by(doubleNum,blob) from table0 where level != 'l4'"; retArray = new String[] { "1971-01-01T00:00:10.000Z,d1,l5,null,null,7,2147983648,213.112,lemon,true,null,0x108dcd63,2024-09-25T06:15:35.000Z,null,null,1970-01-01T00:00:00.020Z,d1,l2,yy,zz,2,2147483648,434.12,pineapple,true,null,0x108dcd62,2024-09-24T06:15:35.000Z,null,6666.8,", }; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + // Retain the unfiltered ties: either associated value is valid, but the extrema are fixed. + expectedHeader = buildHeaders(4); + sql = + "select max_by(num,blob) in (7,9),min_by(num,blob) in (2,5),max_by(blob,blob),min_by(blob,blob) from table0"; + retArray = new String[] {"true,true,0x108dcd63,0x108dcd62,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + expectedHeader = buildHeaders(3); sql = "select extreme(num),extreme(bignum),extreme(floatnum) from table0"; retArray = new String[] {"15,3147483648,4654.231,"}; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java index ccc267e1ccd5f..d3dc0323ee819 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java @@ -259,7 +259,7 @@ public void testPartitionByAndOrderByWithNulls() { "2021-01-01T09:20:00.000Z,null,null,2,", }; tableResultSetEqualTest( - "SELECT *, count(value) OVER (PARTITION BY device ORDER BY value) AS cnt FROM demo2 ORDER BY device", + "SELECT *, count(value) OVER (PARTITION BY device ORDER BY value) AS cnt FROM demo2 ORDER BY device,value,time", expectedHeader, retArray, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java index 3dc7386a71ae1..1d7933b30292c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryIT.java @@ -175,15 +175,16 @@ public void test() throws Exception { "select current from view1 where time > 1", "select current from table1 where time > 1" + (aligned ? "" : " and current is not null"), true); + // FILL runs before an outer ORDER BY, so give both sources the same ordered input. compareQueryResults( session, - "select * from view1 fill method linear", - "select * from table1 fill method linear", + "select * from (select * from view1 order by time, battery) fill method linear", + "select * from (select * from table1 order by time, battery) fill method linear", true); compareQueryResults( session, - "select * from view1 fill method previous", - "select * from table1 fill method previous", + "select * from (select * from view1 order by time, battery) fill method previous", + "select * from (select * from table1 order by time, battery) fill method previous", true); compareQueryResults( session, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 16a183ecf1ed6..c5facfb371ece 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -95,34 +95,82 @@ public class ConfigNodeConfig { private String dataPartitionAllocationStrategy = "INHERIT"; - /** The policy of extension SchemaRegionGroup for each Database. */ + /** + * The policy of extending SchemaRegionGroups for each Database: CUSTOM, AUTO, or PROACTIVE. + * + *

CUSTOM suits known workloads that need manual resource allocation, creating the configured + * target number of groups per Database when schema partitions are first allocated. AUTO suits + * most routine workloads (roughly 80% as an approximate planning guideline), gradually meeting + * the configured minimum and expanding with slot occupancy up to the resource-based maximum to + * balance parallelism and group management overhead. PROACTIVE complements AUTO for workloads + * such as very few devices with many measurements and a high load, creating groups earlier for + * more parallelism at the cost of additional group management overhead. + * + *

AUTO and PROACTIVE use the same per-Database maximum calculation, including resource sharing + * across Databases; their group counts match when both reach the same maximum. PROACTIVE targets + * one group per distinct schema series slot, not per measurement, and grows toward the configured + * minimum incrementally like AUTO. When all groups of this type are disabled, it may add one + * group within the maximum. Switching policies does not remove existing groups. + */ private volatile RegionGroupExtensionPolicy schemaRegionGroupExtensionPolicy = - RegionGroupExtensionPolicy.AUTO; + RegionGroupExtensionPolicy.PROACTIVE; /** * When set schema_region_group_extension_policy=CUSTOM, this parameter is the default number of - * SchemaRegionGroups for each Database. When set schema_region_group_extension_policy=AUTO, this - * parameter is the default minimal number of SchemaRegionGroups for each Database. + * SchemaRegionGroups for each Database. For AUTO and PROACTIVE, this parameter is the default + * minimum number of SchemaRegionGroups and a lower bound for the per-Database maximum. Both + * policies grow toward this minimum incrementally, adding at most the number of series slots in + * the pending request to satisfy the minimum. PROACTIVE also grows with the number of active + * series slots. */ private volatile int defaultSchemaRegionGroupNumPerDatabase = 1; - /** The maximum number of SchemaRegions expected to be managed by each DataNode. */ + /** + * The expected number of SchemaRegions per DataNode, used to calculate the same per-Database + * maximum for AUTO and PROACTIVE. This is not a hard limit on node or cluster totals: + * per-Database minimums, rounding, and existing groups can raise the total above this resource + * estimate. + */ private volatile int schemaRegionPerDataNode = 1; - /** The policy of extension DataRegionGroup for each Database. */ + /** + * The policy of extending DataRegionGroups for each Database: CUSTOM, AUTO, or PROACTIVE. + * + *

CUSTOM suits known workloads that need manual resource allocation, creating the configured + * target number of groups per Database when data partitions are first allocated. AUTO suits most + * routine workloads (roughly 80% as an approximate planning guideline), gradually meeting the + * configured minimum and expanding with slot occupancy up to the resource-based maximum to + * balance parallelism and group management overhead. PROACTIVE complements AUTO for workloads + * such as very few devices with many measurements and a high load, creating groups earlier for + * more parallelism at the cost of additional group management overhead. + * + *

AUTO and PROACTIVE use the same per-Database maximum calculation, including resource sharing + * across Databases; their group counts match when both reach the same maximum. PROACTIVE targets + * one group per distinct data series slot, not per measurement, and grows toward the configured + * minimum incrementally like AUTO. New time partitions in an existing slot can also trigger this + * growth toward the minimum. When all groups of this type are disabled, it may add one group + * within the maximum. Its allocation policy table balances active slots so new time partitions + * can use new groups; existing time-partition assignments are retained. Switching policies does + * not remove existing groups. + */ private volatile RegionGroupExtensionPolicy dataRegionGroupExtensionPolicy = - RegionGroupExtensionPolicy.AUTO; + RegionGroupExtensionPolicy.PROACTIVE; /** * When set data_region_group_extension_policy=CUSTOM, this parameter is the default number of - * DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this - * parameter is the default minimal number of DataRegionGroups for each Database. + * DataRegionGroups for each Database. For AUTO and PROACTIVE, this parameter is the default + * minimum number of DataRegionGroups and a lower bound for the per-Database maximum. Both + * policies grow toward this minimum incrementally, adding at most the number of series slots in + * the pending request to satisfy the minimum. PROACTIVE also grows with the number of active + * series slots. */ private volatile int defaultDataRegionGroupNumPerDatabase = 2; /** - * The maximum number of DataRegions expected to be managed by each DataNode. Set to 0 means that - * each dataNode automatically has the number of CPU cores / 2 regions. + * The expected number of DataRegions per DataNode, used to calculate the same per-Database + * maximum for AUTO and PROACTIVE. Set to 0 to use half the CPU core count as the expected number + * for each DataNode. This is not a hard limit on node or cluster totals: per-Database minimums, + * rounding, and existing groups can raise the total above this resource estimate. */ private volatile int dataRegionPerDataNode = 0; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterQuotaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterQuotaManager.java index d53890f75c341..7af050dacd4f8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterQuotaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterQuotaManager.java @@ -272,11 +272,10 @@ public void updateSpaceQuotaUsage() { .forEach( schemaRegionId -> { if (deviceNum.containsKey(schemaRegionId)) { - deviceCount.addAndGet(deviceCount.get() + deviceNum.get(schemaRegionId)); + deviceCount.addAndGet(deviceNum.get(schemaRegionId)); } if (timeSeriesNum.containsKey(schemaRegionId)) { - timeSeriesCount.addAndGet( - timeSeriesCount.get() + timeSeriesNum.get(schemaRegionId)); + timeSeriesCount.addAndGet(timeSeriesNum.get(schemaRegionId)); } }); quotaInfo.getSpaceQuotaUsage().get(entry.getKey()).setDeviceNum(deviceCount.get()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 5041b77136b9a..5fe329af8d48a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -125,6 +125,7 @@ import org.apache.iotdb.confignode.manager.node.NodeMetrics; import org.apache.iotdb.confignode.manager.partition.PartitionManager; import org.apache.iotdb.confignode.manager.partition.PartitionMetrics; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent; import org.apache.iotdb.confignode.manager.pipe.coordinator.PipeManager; import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; @@ -1836,6 +1837,10 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { long previousHeartbeatIntervalInMs = CONF.getHeartbeatIntervalInMs(); int previousSchemaRegionPerDataNode = CONF.getSchemaRegionPerDataNode(); int previousDataRegionPerDataNode = CONF.getDataRegionPerDataNode(); + RegionGroupExtensionPolicy previousSchemaRegionGroupExtensionPolicy = + CONF.getSchemaRegionGroupExtensionPolicy(); + RegionGroupExtensionPolicy previousDataRegionGroupExtensionPolicy = + CONF.getDataRegionGroupExtensionPolicy(); boolean wasTopologyProbingEnabled = CONF.isEnableTopologyProbing(); int previousProcedureCompletedCleanInterval = CONF.getProcedureCompletedCleanInterval(); int previousProcedureCompletedEvictTTL = CONF.getProcedureCompletedEvictTTL(); @@ -1866,8 +1871,11 @@ public TSStatus setConfiguration(TSetConfigurationReq req) { return tsStatus; } handleHeartbeatIntervalHotReload(previousHeartbeatIntervalInMs); - handleRegionPerDataNodeHotReload( - previousSchemaRegionPerDataNode, previousDataRegionPerDataNode); + handleRegionGroupConfigHotReload( + previousSchemaRegionPerDataNode, + previousDataRegionPerDataNode, + previousSchemaRegionGroupExtensionPolicy, + previousDataRegionGroupExtensionPolicy); handleTopologyProbingHotReload(wasTopologyProbingEnabled); handleProcedureCleanerHotReload( previousProcedureCompletedCleanInterval, previousProcedureCompletedEvictTTL); @@ -1914,16 +1922,31 @@ private void handleHeartbeatIntervalHotReload(long previousHeartbeatIntervalInMs getRetryFailedTasksThread().reloadHeartbeatInterval(); } - private void handleRegionPerDataNodeHotReload( - int previousSchemaRegionPerDataNode, int previousDataRegionPerDataNode) { + private void handleRegionGroupConfigHotReload( + int previousSchemaRegionPerDataNode, + int previousDataRegionPerDataNode, + RegionGroupExtensionPolicy previousSchemaRegionGroupExtensionPolicy, + RegionGroupExtensionPolicy previousDataRegionGroupExtensionPolicy) { if (previousSchemaRegionPerDataNode == CONF.getSchemaRegionPerDataNode() - && previousDataRegionPerDataNode == CONF.getDataRegionPerDataNode()) { + && previousDataRegionPerDataNode == CONF.getDataRegionPerDataNode() + && previousSchemaRegionGroupExtensionPolicy == CONF.getSchemaRegionGroupExtensionPolicy() + && previousDataRegionGroupExtensionPolicy == CONF.getDataRegionGroupExtensionPolicy()) { return; } if (!getConsensusManager().isLeader()) { return; } + // Leaving CUSTOM must replace its stored cap with the resource-derived cap, even when + // per-node quotas are unchanged. This preserves existing groups and skips types still CUSTOM. getClusterSchemaManager().adjustMaxRegionGroupNum(); + if (previousDataRegionGroupExtensionPolicy != RegionGroupExtensionPolicy.PROACTIVE + && CONF.getDataRegionGroupExtensionPolicy() == RegionGroupExtensionPolicy.PROACTIVE) { + // Existing groups may already meet the new target, so no creation procedure would rebalance + // their slot assignments. Refresh the policy table for future time partitions on this leader. + getClusterSchemaManager() + .getDatabaseNames(null) + .forEach(database -> getLoadManager().reBalanceDataPartitionPolicy(database)); + } } private void handleTopologyProbingHotReload(boolean wasEnabled) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTable.java index 2cce461478c49..bc80ef62014c0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTable.java @@ -25,6 +25,7 @@ import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.i18n.ManagerMessages; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.locks.ReentrantLock; public class DataPartitionPolicyTable { @@ -111,7 +113,7 @@ public void reBalanceDataPartitionPolicy(List dataRegionGroup } Collections.shuffle(seriesPartitionSlots); - int mu = SERIES_SLOT_NUM / dataRegionGroups.size(); + int mu = getRetainedSlotLimit(dataAllotMap, dataRegionGroups.size()); for (TSeriesPartitionSlot seriesPartitionSlot : seriesPartitionSlots) { if (!dataAllotMap.containsKey(seriesPartitionSlot)) { // Skip unallocated SeriesPartitionSlot @@ -141,7 +143,7 @@ public void setDataAllotMap(Map dataAll } dataAllotTableLock.lock(); try { - int mu = SERIES_SLOT_NUM / seriesPartitionSlotCounter.size(); + int mu = getRetainedSlotLimit(dataAllotMap, seriesPartitionSlotCounter.size()); dataAllotMap.forEach( (seriesPartitionSlot, regionGroupId) -> { if (regionGroupId != null && seriesPartitionSlotCounter.get(regionGroupId) < mu) { @@ -158,6 +160,18 @@ public void setDataAllotMap(Map dataAll } } + private int getRetainedSlotLimit( + Map assignments, int regionGroupCount) { + if (CONF.getDataRegionGroupExtensionPolicy() == RegionGroupExtensionPolicy.PROACTIVE) { + // Empty DataPartition entries have no last group and must not inflate the recovery limit. + long activeSlotCount = assignments.values().stream().filter(Objects::nonNull).count(); + // New groups are created before the pending slots are activated. Keep at least one slot + // per existing group so that incremental growth does not discard balanced assignments. + return Math.max(1, (int) (activeSlotCount / regionGroupCount)); + } + return SERIES_SLOT_NUM / regionGroupCount; + } + public void logDataAllotTable(String database) { seriesPartitionSlotCounter .keySet() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index 2ee0bcdb60bbc..249a4f9bd0e05 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -109,6 +109,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -283,15 +284,9 @@ public SchemaPartitionResp getOrCreateSchemaPartition(final GetOrCreateSchemaPar // Here we ensure that each StorageGroup has at least one SchemaRegion. // And if some StorageGroups own too many slots, extend SchemaRegion for them. - // Map - final Map unassignedSchemaPartitionSlotsCountMap = new ConcurrentHashMap<>(); - unassignedSchemaPartitionSlotsMap.forEach( - (storageGroup, unassignedSchemaPartitionSlots) -> - unassignedSchemaPartitionSlotsCountMap.put( - storageGroup, unassignedSchemaPartitionSlots.size())); TSStatus status = extendRegionGroupIfNecessary( - unassignedSchemaPartitionSlotsCountMap, TConsensusGroupType.SchemaRegion); + unassignedSchemaPartitionSlotsMap, TConsensusGroupType.SchemaRegion); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // Return an error code if Region extension failed resp.setStatus(status); @@ -438,15 +433,20 @@ public DataPartitionResp getOrCreateDataPartition(final GetOrCreateDataPartition // Here we ensure that each StorageGroup has at least one DataRegion. // And if some StorageGroups own too many slots, extend DataRegion for them. - // Map - Map unassignedDataPartitionSlotsCountMap = new ConcurrentHashMap<>(); + // Keep the series slots so PROACTIVE can distinguish new slots from new time partitions. + Map> unassignedDataSeriesPartitionSlotsMap = + new HashMap<>(); unassignedDataPartitionSlotsMap.forEach( (storageGroup, unassignedDataPartitionSlots) -> - unassignedDataPartitionSlotsCountMap.put( - storageGroup, unassignedDataPartitionSlots.size())); + unassignedDataSeriesPartitionSlotsMap.put( + storageGroup, + unassignedDataPartitionSlots.entrySet().stream() + .filter(entry -> !entry.getValue().getTimePartitionSlots().isEmpty()) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()))); TSStatus status = extendRegionGroupIfNecessary( - unassignedDataPartitionSlotsCountMap, TConsensusGroupType.DataRegion); + unassignedDataSeriesPartitionSlotsMap, TConsensusGroupType.DataRegion); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { // Return an error code if Region extension failed resp.setStatus(status); @@ -599,13 +599,13 @@ private TSStatus consensusWritePartitionResult(ConfigPhysicalPlan plan) { /** * Allocate more RegionGroup to the specified StorageGroups if necessary. * - * @param unassignedPartitionSlotsCountMap Map + * @param unassignedPartitionSlotsMap Map * @param consensusGroupType SchemaRegion or DataRegion * @return SUCCESS_STATUS when RegionGroup extension successful; NOT_ENOUGH_DATA_NODE when there * are not enough DataNodes; STORAGE_GROUP_NOT_EXIST when some StorageGroups don't exist */ - private TSStatus extendRegionGroupIfNecessary( - final Map unassignedPartitionSlotsCountMap, + TSStatus extendRegionGroupIfNecessary( + final Map> unassignedPartitionSlotsMap, final TConsensusGroupType consensusGroupType) { final TSStatus result = new TSStatus(); @@ -613,23 +613,29 @@ private TSStatus extendRegionGroupIfNecessary( try { if (TConsensusGroupType.SchemaRegion.equals(consensusGroupType)) { switch (CONF.getSchemaRegionGroupExtensionPolicy()) { + case PROACTIVE: + return proactiveExtendRegionGroupIfNecessary( + unassignedPartitionSlotsMap, consensusGroupType); case CUSTOM: return customExtendRegionGroupIfNecessary( - unassignedPartitionSlotsCountMap, consensusGroupType); + unassignedPartitionSlotsMap, consensusGroupType); case AUTO: default: return autoExtendRegionGroupIfNecessary( - unassignedPartitionSlotsCountMap, consensusGroupType); + unassignedPartitionSlotsMap, consensusGroupType); } } else { switch (CONF.getDataRegionGroupExtensionPolicy()) { + case PROACTIVE: + return proactiveExtendRegionGroupIfNecessary( + unassignedPartitionSlotsMap, consensusGroupType); case CUSTOM: return customExtendRegionGroupIfNecessary( - unassignedPartitionSlotsCountMap, consensusGroupType); + unassignedPartitionSlotsMap, consensusGroupType); case AUTO: default: return autoExtendRegionGroupIfNecessary( - unassignedPartitionSlotsCountMap, consensusGroupType); + unassignedPartitionSlotsMap, consensusGroupType); } } } catch (NotEnoughDataNodeException e) { @@ -645,16 +651,62 @@ private TSStatus extendRegionGroupIfNecessary( return result; } + private TSStatus proactiveExtendRegionGroupIfNecessary( + final Map> unassignedPartitionSlotsMap, + final TConsensusGroupType consensusGroupType) + throws DatabaseNotExistsException, NotEnoughDataNodeException { + final Map allotmentMap = new HashMap<>(); + for (Map.Entry> entry : + unassignedPartitionSlotsMap.entrySet()) { + final String database = entry.getKey(); + final int maxRegionGroupCount = + getClusterSchemaManager().getMaxRegionGroupNum(database, consensusGroupType); + final int minRegionGroupCount = + getClusterSchemaManager().getMinRegionGroupNum(database, consensusGroupType); + final int allocatedRegionGroupCount = + partitionInfo.getRegionGroupCount(database, consensusGroupType); + // As with AUTO, grow toward the minimum by at most the number of pending series slots. + // An existing data slot with new time partitions also contributes to this incremental growth. + final int minimumRegionGroupTarget = + Math.min( + minRegionGroupCount, + allocatedRegionGroupCount + new HashSet<>(entry.getValue()).size()); + final int targetRegionGroupCount = + Math.min( + maxRegionGroupCount, + Math.max( + minimumRegionGroupTarget, + partitionInfo.getSeriesPartitionSlotsCount( + database, consensusGroupType, entry.getValue()))); + // Also allocate one group per active series slot, up to the same maximum as AUTO. + // Existing groups are retained when the maximum decreases or the policy changes. + if (allocatedRegionGroupCount < targetRegionGroupCount) { + allotmentMap.put(database, targetRegionGroupCount - allocatedRegionGroupCount); + } else if (!entry.getValue().isEmpty() + && allocatedRegionGroupCount > 0 + && allocatedRegionGroupCount < maxRegionGroupCount + && partitionInfo.getAllRegionGroupIds(database, consensusGroupType).stream() + .allMatch( + regionGroupId -> + RegionGroupStatus.Disabled.equals( + getLoadManager().getRegionGroupStatus(regionGroupId)))) { + // As with AUTO, preserve availability when all groups are disabled. This may exceed + // the active-slot target, but not the database's maximum for this region type. + allotmentMap.put(database, 1); + } + } + return generateAndAllocateRegionGroups(allotmentMap, consensusGroupType); + } + private TSStatus customExtendRegionGroupIfNecessary( - final Map unassignedPartitionSlotsCountMap, + final Map> unassignedPartitionSlotsMap, final TConsensusGroupType consensusGroupType) throws DatabaseNotExistsException, NotEnoughDataNodeException { // Map final Map allotmentMap = new ConcurrentHashMap<>(); - for (final Map.Entry entry : unassignedPartitionSlotsCountMap.entrySet()) { - final String database = entry.getKey(); + for (final String database : unassignedPartitionSlotsMap.keySet()) { final int maxRegionGroupNum = getClusterSchemaManager().getMaxRegionGroupNum(database, consensusGroupType); final int allocatedRegionGroupCount = @@ -670,16 +722,17 @@ private TSStatus customExtendRegionGroupIfNecessary( } private TSStatus autoExtendRegionGroupIfNecessary( - final Map unassignedPartitionSlotsCountMap, + final Map> unassignedPartitionSlotsMap, final TConsensusGroupType consensusGroupType) throws NotEnoughDataNodeException, DatabaseNotExistsException { // Map final Map allotmentMap = new ConcurrentHashMap<>(); - for (Map.Entry entry : unassignedPartitionSlotsCountMap.entrySet()) { + for (Map.Entry> entry : + unassignedPartitionSlotsMap.entrySet()) { final String database = entry.getKey(); - final int unassignedPartitionSlotsCount = entry.getValue(); + final int unassignedPartitionSlotsCount = entry.getValue().size(); float allocatedRegionGroupCount = partitionInfo.getRegionGroupCount(database, consensusGroupType); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/RegionGroupExtensionPolicy.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/RegionGroupExtensionPolicy.java index 5f69a5d3525a5..fca087933ddab 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/RegionGroupExtensionPolicy.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/RegionGroupExtensionPolicy.java @@ -25,7 +25,9 @@ public enum RegionGroupExtensionPolicy { CUSTOM("CUSTOM"), - AUTO("AUTO"); + AUTO("AUTO"), + + PROACTIVE("PROACTIVE"); private final String policy; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java index 1ed35210afcd2..d3528c40ef835 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java @@ -46,6 +46,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -267,6 +268,21 @@ public int getAssignedSeriesPartitionSlotsCount() { dataPartitionTable.getDataPartitionMap().size()); } + /** Count distinct series slots after allocation, independently for schema and data partitions. */ + public int getSeriesPartitionSlotsCount( + TConsensusGroupType type, Collection unassignedSlots) { + final Set assignedSlots = + type == TConsensusGroupType.SchemaRegion + ? schemaPartitionTable.getSchemaPartitionMap().keySet() + : dataPartitionTable.getDataPartitionMap().entrySet().stream() + .filter(entry -> !entry.getValue().getSeriesPartitionMap().isEmpty()) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + final Set newSlots = new HashSet<>(unassignedSlots); + newSlots.removeAll(assignedSlots); + return assignedSlots.size() + newSlots.size(); + } + /** * Thread-safely get SchemaPartition within the specific StorageGroup. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index 71fef7c21d017..7459510f4a23c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -89,6 +89,7 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -911,6 +912,17 @@ public int getAssignedSeriesPartitionSlotsCount(String database) { .orElse(0); } + /** Count the series slots of one region type, including those in the pending allocation. */ + public int getSeriesPartitionSlotsCount( + String database, TConsensusGroupType type, Collection unassignedSlots) + throws DatabaseNotExistsException { + final DatabasePartitionTable partitionTable = databasePartitionTables.get(database); + if (partitionTable == null || !partitionTable.isNotPreDeleted()) { + throw new DatabaseNotExistsException(database); + } + return partitionTable.getSeriesPartitionSlotsCount(type, unassignedSlots); + } + /** * Only leader use this interface. * diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/conf/RegionGroupExtensionPolicyConfigTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/conf/RegionGroupExtensionPolicyConfigTest.java new file mode 100644 index 0000000000000..dbda9eb9c35e8 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/conf/RegionGroupExtensionPolicyConfigTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.conf; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.TrimProperties; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class RegionGroupExtensionPolicyConfigTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testParsePolicies() throws Exception { + for (RegionGroupExtensionPolicy policy : RegionGroupExtensionPolicy.values()) { + assertEquals(policy, RegionGroupExtensionPolicy.parse(policy.getPolicy())); + } + assertEquals( + RegionGroupExtensionPolicy.PROACTIVE, RegionGroupExtensionPolicy.parse("PROACTIVE")); + assertThrows(IOException.class, () -> RegionGroupExtensionPolicy.parse("UNKNOWN")); + } + + @Test + public void testProactiveStartupAndHotReloadForBothTypes() throws Exception { + String originalConf = System.getProperty(ConfigNodeConstant.CONFIGNODE_CONF); + File confDir = temporaryFolder.newFolder(); + Files.writeString( + confDir.toPath().resolve(CommonConfig.SYSTEM_CONFIG_NAME), + "cn_seed_config_node=127.0.0.1:10710\n" + + "schema_region_group_extension_policy=PROACTIVE\n" + + "data_region_group_extension_policy=PROACTIVE\n", + StandardCharsets.UTF_8); + System.setProperty(ConfigNodeConstant.CONFIGNODE_CONF, confDir.getAbsolutePath()); + try { + Constructor constructor = + ConfigNodeDescriptor.class.getDeclaredConstructor(); + constructor.setAccessible(true); + ConfigNodeDescriptor descriptor = constructor.newInstance(); + ConfigNodeConfig conf = descriptor.getConf(); + assertEquals( + RegionGroupExtensionPolicy.PROACTIVE, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.PROACTIVE, conf.getDataRegionGroupExtensionPolicy()); + + TrimProperties properties = new TrimProperties(); + properties.setProperty("schema_region_group_extension_policy", "CUSTOM"); + properties.setProperty("data_region_group_extension_policy", "AUTO"); + descriptor.loadHotModifiedProps(properties); + assertEquals(RegionGroupExtensionPolicy.CUSTOM, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.AUTO, conf.getDataRegionGroupExtensionPolicy()); + + properties.setProperty("schema_region_group_extension_policy", "PROACTIVE"); + properties.setProperty("data_region_group_extension_policy", "PROACTIVE"); + descriptor.loadHotModifiedProps(properties); + assertEquals( + RegionGroupExtensionPolicy.PROACTIVE, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.PROACTIVE, conf.getDataRegionGroupExtensionPolicy()); + + TrimProperties schemaOnly = new TrimProperties(); + schemaOnly.setProperty("schema_region_group_extension_policy", "AUTO"); + descriptor.loadHotModifiedProps(schemaOnly); + assertEquals(RegionGroupExtensionPolicy.AUTO, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.PROACTIVE, conf.getDataRegionGroupExtensionPolicy()); + + TrimProperties dataOnly = new TrimProperties(); + dataOnly.setProperty("data_region_group_extension_policy", "CUSTOM"); + descriptor.loadHotModifiedProps(dataOnly); + assertEquals(RegionGroupExtensionPolicy.AUTO, conf.getSchemaRegionGroupExtensionPolicy()); + assertEquals(RegionGroupExtensionPolicy.CUSTOM, conf.getDataRegionGroupExtensionPolicy()); + } finally { + if (originalConf == null) { + System.clearProperty(ConfigNodeConstant.CONFIGNODE_CONF); + } else { + System.setProperty(ConfigNodeConstant.CONFIGNODE_CONF, originalConf); + } + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerRegionGroupPolicyReloadTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerRegionGroupPolicyReloadTest.java new file mode 100644 index 0000000000000..8b8e4bd5da917 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerRegionGroupPolicyReloadTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.manager; + +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Method; +import java.util.Arrays; + +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +public class ConfigManagerRegionGroupPolicyReloadTest { + private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); + private RegionGroupExtensionPolicy originalSchemaPolicy; + private RegionGroupExtensionPolicy originalDataPolicy; + private ConfigManager manager; + private ConsensusManager consensusManager; + private ClusterSchemaManager schemaManager; + private LoadManager loadManager; + + @Before + public void setUp() { + originalSchemaPolicy = CONF.getSchemaRegionGroupExtensionPolicy(); + originalDataPolicy = CONF.getDataRegionGroupExtensionPolicy(); + CONF.setSchemaRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.AUTO); + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + manager = mock(ConfigManager.class); + consensusManager = mock(ConsensusManager.class); + schemaManager = mock(ClusterSchemaManager.class); + loadManager = mock(LoadManager.class); + when(manager.getConsensusManager()).thenReturn(consensusManager); + when(manager.getClusterSchemaManager()).thenReturn(schemaManager); + when(manager.getLoadManager()).thenReturn(loadManager); + when(consensusManager.isLeader()).thenReturn(true); + when(schemaManager.getDatabaseNames(null)).thenReturn(Arrays.asList("root.one", "root.two")); + } + + @After + public void tearDown() { + CONF.setSchemaRegionGroupExtensionPolicy(originalSchemaPolicy); + CONF.setDataRegionGroupExtensionPolicy(originalDataPolicy); + } + + @Test + public void testEnteringProactiveRebalancesEveryDatabase() throws Exception { + for (RegionGroupExtensionPolicy previous : + Arrays.asList(RegionGroupExtensionPolicy.AUTO, RegionGroupExtensionPolicy.CUSTOM)) { + clearInvocations(schemaManager, loadManager); + reload(RegionGroupExtensionPolicy.AUTO, previous, CONF.getDataRegionPerDataNode()); + verify(schemaManager).adjustMaxRegionGroupNum(); + verify(loadManager).reBalanceDataPartitionPolicy("root.one"); + verify(loadManager).reBalanceDataPartitionPolicy("root.two"); + } + } + + @Test + public void testRepeatedPolicyDoesNotRebalance() throws Exception { + reload( + RegionGroupExtensionPolicy.AUTO, + RegionGroupExtensionPolicy.PROACTIVE, + CONF.getDataRegionPerDataNode()); + verifyZeroInteractions(loadManager, schemaManager); + } + + @Test + public void testSchemaAndQuotaChangesDoNotRebalanceData() throws Exception { + reload( + RegionGroupExtensionPolicy.CUSTOM, + RegionGroupExtensionPolicy.PROACTIVE, + CONF.getDataRegionPerDataNode()); + reload( + RegionGroupExtensionPolicy.AUTO, + RegionGroupExtensionPolicy.PROACTIVE, + CONF.getDataRegionPerDataNode() + 1); + verifyZeroInteractions(loadManager); + } + + @Test + public void testLeavingProactiveDoesNotRebalance() throws Exception { + for (RegionGroupExtensionPolicy next : + Arrays.asList(RegionGroupExtensionPolicy.AUTO, RegionGroupExtensionPolicy.CUSTOM)) { + CONF.setDataRegionGroupExtensionPolicy(next); + reload( + RegionGroupExtensionPolicy.AUTO, + RegionGroupExtensionPolicy.PROACTIVE, + CONF.getDataRegionPerDataNode()); + } + verifyZeroInteractions(loadManager); + } + + @Test + public void testFollowerDoesNotRebalance() throws Exception { + when(consensusManager.isLeader()).thenReturn(false); + reload( + RegionGroupExtensionPolicy.AUTO, + RegionGroupExtensionPolicy.CUSTOM, + CONF.getDataRegionPerDataNode()); + verifyZeroInteractions(loadManager, schemaManager); + } + + private void reload( + RegionGroupExtensionPolicy previousSchema, + RegionGroupExtensionPolicy previousData, + int previousDataQuota) + throws Exception { + Method reload = + ConfigManager.class.getDeclaredMethod( + "handleRegionGroupConfigHotReload", + int.class, + int.class, + RegionGroupExtensionPolicy.class, + RegionGroupExtensionPolicy.class); + reload.setAccessible(true); + reload.invoke( + manager, + CONF.getSchemaRegionPerDataNode(), + previousDataQuota, + previousSchema, + previousData); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTableProactiveTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTableProactiveTest.java new file mode 100644 index 0000000000000..9b0c6117c9877 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/partition/DataPartitionPolicyTableProactiveTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.load.balancer.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.manager.partition.RegionGroupExtensionPolicy; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +public class DataPartitionPolicyTableProactiveTest { + + private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); + private RegionGroupExtensionPolicy originalPolicy; + private int originalSlotCount; + + @Before + public void setUp() { + originalPolicy = CONF.getDataRegionGroupExtensionPolicy(); + originalSlotCount = CONF.getSeriesSlotNum(); + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + CONF.setSeriesSlotNum(1000); + } + + @After + public void tearDown() { + CONF.setDataRegionGroupExtensionPolicy(originalPolicy); + CONF.setSeriesSlotNum(originalSlotCount); + } + + @Test + public void testSixExistingSlotsUseNewGroupsAfterSwitchingFromAuto() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.AUTO); + table.reBalanceDataPartitionPolicy(groups(2)); + Assert.assertEquals(2, new HashSet<>(activate(table, 6).values()).size()); + + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + table.reBalanceDataPartitionPolicy(groups(6)); + assertBalanced(groups(6), activate(table, 6)); + } + + @Test + public void testCustomExpansionIsRebalancedWhenEnteringProactive() { + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.CUSTOM); + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(groups(2)); + Map original = activate(table, 6); + table.reBalanceDataPartitionPolicy(groups(6)); + Assert.assertEquals(original, activate(table, 6)); + + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + table.reBalanceDataPartitionPolicy(groups(6)); + assertBalanced(groups(6), activate(table, 6)); + } + + @Test + public void testUnevenSlotCountStillUsesEveryNewGroup() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(groups(2)); + activate(table, 5); + table.reBalanceDataPartitionPolicy(groups(4)); + assertBalanced(groups(4), activate(table, 5)); + } + + @Test + public void testGrowingAheadOfPendingSlotsPreservesExistingAssignments() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + for (int slot = 0; slot < 6; slot++) { + Map existing = activate(table, slot); + table.reBalanceDataPartitionPolicy(groups(slot + 1)); + Assert.assertEquals(existing, activate(table, slot)); + assertBalanced(groups(slot + 1), activate(table, slot + 1)); + } + } + + @Test + public void testLeaderRecoveryRebalancesSparseSlotsAndIgnoresEmptyEntries() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(groups(6)); + Map previous = new HashMap<>(); + for (int slot = 0; slot < 6; slot++) { + previous.put(new TSeriesPartitionSlot(slot), groups(2).get(slot % 2)); + } + // DataPartitionTable can retain empty entries after filtering or time-partition cleanup. + for (int slot = 6; slot < 100; slot++) { + previous.put(new TSeriesPartitionSlot(slot), null); + } + table.setDataAllotMap(previous); + assertBalanced(groups(6), activate(table, 6)); + } + + @Test + public void testLeaderRecoveryPreservesAlreadyBalancedAssignments() { + DataPartitionPolicyTable original = new DataPartitionPolicyTable(); + original.reBalanceDataPartitionPolicy(groups(6)); + Map previous = activate(original, 6); + DataPartitionPolicyTable recovered = new DataPartitionPolicyTable(); + recovered.reBalanceDataPartitionPolicy(groups(6)); + recovered.setDataAllotMap(previous); + Assert.assertEquals(previous, activate(recovered, 6)); + } + + @Test + public void testAutoAndCustomKeepExistingSparseAssignments() { + for (RegionGroupExtensionPolicy policy : + Arrays.asList(RegionGroupExtensionPolicy.AUTO, RegionGroupExtensionPolicy.CUSTOM)) { + CONF.setDataRegionGroupExtensionPolicy(policy); + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(groups(2)); + Map previous = activate(table, 6); + table.reBalanceDataPartitionPolicy(groups(6)); + Assert.assertEquals(previous, activate(table, 6)); + + DataPartitionPolicyTable recovered = new DataPartitionPolicyTable(); + recovered.reBalanceDataPartitionPolicy(groups(6)); + recovered.setDataAllotMap(previous); + Assert.assertEquals(previous, activate(recovered, 6)); + } + } + + @Test + public void testEmptyRecoveryCanActivateFirstSlot() { + DataPartitionPolicyTable table = new DataPartitionPolicyTable(); + table.reBalanceDataPartitionPolicy(Collections.emptyList()); + table.setDataAllotMap(Collections.emptyMap()); + table.reBalanceDataPartitionPolicy(groups(1)); + table.setDataAllotMap(Collections.emptyMap()); + assertBalanced(groups(1), activate(table, 1)); + } + + private static List groups(int count) { + List groups = new ArrayList<>(); + for (int id = 0; id < count; id++) { + groups.add(new TConsensusGroupId(TConsensusGroupType.DataRegion, id)); + } + return groups; + } + + private static Map activate( + DataPartitionPolicyTable table, int slotCount) { + Map assignments = new HashMap<>(); + for (int id = 0; id < slotCount; id++) { + TSeriesPartitionSlot slot = new TSeriesPartitionSlot(id); + assignments.put(slot, table.getRegionGroupIdOrActivateIfNecessary(slot)); + } + return assignments; + } + + private static void assertBalanced( + List groups, Map assignments) { + Map counts = new HashMap<>(); + assignments.values().forEach(group -> counts.merge(group, 1, Integer::sum)); + Assert.assertEquals(new HashSet<>(groups), counts.keySet()); + Assert.assertTrue( + counts.toString(), + Collections.max(counts.values()) - Collections.min(counts.values()) <= 1); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/partition/PartitionManagerProactiveExtensionTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/partition/PartitionManagerProactiveExtensionTest.java new file mode 100644 index 0000000000000..1eb9c7acb203e --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/partition/PartitionManagerProactiveExtensionTest.java @@ -0,0 +1,811 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.manager.partition; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.partition.DataPartitionTable; +import org.apache.iotdb.commons.partition.SchemaPartitionTable; +import org.apache.iotdb.commons.partition.SeriesPartitionTable; +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.partition.GetDataPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.read.partition.GetOrCreateDataPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.PreDeleteDatabasePlan; +import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; +import org.apache.iotdb.confignode.exception.NotEnoughDataNodeException; +import org.apache.iotdb.confignode.manager.IManager; +import org.apache.iotdb.confignode.manager.ProcedureManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; +import org.apache.iotdb.confignode.persistence.partition.PartitionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class PartitionManagerProactiveExtensionTest { + + private static final String DATABASE = "root.proactive"; + private static final String SECOND_DATABASE = "root.other"; + private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf(); + + @Parameterized.Parameters(name = "{0}") + public static Collection parameters() { + return Arrays.asList( + new Object[] {TConsensusGroupType.SchemaRegion}, + new Object[] {TConsensusGroupType.DataRegion}); + } + + private final TConsensusGroupType type; + private PartitionInfo partitionInfo; + private PartitionManager partitionManager; + private ClusterSchemaManager schemaManager; + private LoadManager loadManager; + private ProcedureManager procedureManager; + private ConsensusManager consensusManager; + private RegionGroupExtensionPolicy originalSchemaPolicy; + private RegionGroupExtensionPolicy originalDataPolicy; + private int originalSeriesSlotNum; + + public PartitionManagerProactiveExtensionTest(TConsensusGroupType type) { + this.type = type; + } + + @Before + public void setUp() throws Exception { + originalSchemaPolicy = CONF.getSchemaRegionGroupExtensionPolicy(); + originalDataPolicy = CONF.getDataRegionGroupExtensionPolicy(); + originalSeriesSlotNum = CONF.getSeriesSlotNum(); + CONF.setSchemaRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + CONF.setDataRegionGroupExtensionPolicy(RegionGroupExtensionPolicy.PROACTIVE); + CONF.setSeriesSlotNum(100); + + partitionInfo = new PartitionInfo(); + for (String database : Arrays.asList(DATABASE, SECOND_DATABASE)) { + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database))); + } + IManager manager = mock(IManager.class); + schemaManager = mock(ClusterSchemaManager.class); + loadManager = mock(LoadManager.class); + procedureManager = mock(ProcedureManager.class); + consensusManager = mock(ConsensusManager.class); + when(manager.getClusterSchemaManager()).thenReturn(schemaManager); + when(manager.getLoadManager()).thenReturn(loadManager); + when(manager.getProcedureManager()).thenReturn(procedureManager); + when(manager.getConsensusManager()).thenReturn(consensusManager); + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(4); + when(schemaManager.getMaxRegionGroupNum(DATABASE, type)).thenReturn(5); + when(schemaManager.getMaxRegionGroupNum(SECOND_DATABASE, type)).thenReturn(3); + when(loadManager.allocateRegionGroups(anyMap(), any(TConsensusGroupType.class))) + .thenAnswer( + invocation -> { + Map allotments = invocation.getArgument(0); + TConsensusGroupType allocatedType = invocation.getArgument(1); + return regionPlan(allotments, allocatedType); + }); + when(procedureManager.createRegionGroups(any(TConsensusGroupType.class), any())) + .thenAnswer( + invocation -> { + partitionInfo.createRegionGroups(invocation.getArgument(1)); + return RpcUtils.SUCCESS_STATUS; + }); + partitionManager = new PartitionManager(manager, partitionInfo); + } + + @After + public void tearDown() { + if (partitionManager != null) { + partitionManager.stopRegionCleaner(); + } + CONF.setSchemaRegionGroupExtensionPolicy(originalSchemaPolicy); + CONF.setDataRegionGroupExtensionPolicy(originalDataPolicy); + CONF.setSeriesSlotNum(originalSeriesSlotNum); + } + + @Test + public void testEachNewSlotCreatesOneRegionUntilMaximum() throws Exception { + for (int slot = 0; slot < 8; slot++) { + assertSuccess(extend(DATABASE, slots(slot))); + // A configured minimum of four is reached gradually, as with AUTO. + assertEquals(Math.min(slot + 1, 5), partitionInfo.getRegionGroupCount(DATABASE, type)); + persistSlots(DATABASE, type, 0, slot); + } + verify(loadManager, never()).allocateRegionGroups(anyMap(), eq(otherType())); + } + + @Test + public void testDuplicatePendingSlotsDoNotInflateMinimumGrowth() throws Exception { + List pending = slots(0, 0); + if (type == TConsensusGroupType.SchemaRegion) { + pending = + partitionInfo + .filterUnassignedSchemaPartitionSlots(Collections.singletonMap(DATABASE, pending)) + .get(DATABASE); + } + assertSuccess(extend(DATABASE, pending)); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + persistSlots(DATABASE, type, 0, 0); + assertSuccess(extend(DATABASE, slots(1, 1))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + @Test + public void testMixedDataRequestOnlyCountsSlotsWithMissingTimePartitions() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + assertSuccess(extend(DATABASE, slots(0, 1))); + persistSlots(DATABASE, type, 0, 0, 1); + configureDataPartitionConsensusAndAllocation(); + clearInvocations(loadManager); + + GetOrCreateDataPartitionPlan request = + new GetOrCreateDataPartitionPlan( + Collections.singletonMap(DATABASE, dataRequest(0, 0, 1, 2))); + assertSuccess(partitionManager.getOrCreateDataPartition(request).getStatus()); + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 1), type); + + clearInvocations(loadManager); + assertSuccess(partitionManager.getOrCreateDataPartition(request).getStatus()); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testSatisfiedDatabaseDoesNotGrowForAnotherDatabasesPendingSlots() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + configureDataPartitionConsensusAndAllocation(); + clearInvocations(loadManager); + + Map> request = new HashMap<>(); + request.put(DATABASE, dataRequest(0, 0)); + request.put(SECOND_DATABASE, dataRequest(0, 1)); + assertSuccess( + partitionManager + .getOrCreateDataPartition(new GetOrCreateDataPartitionPlan(request)) + .getStatus()); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(1, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(SECOND_DATABASE, 1), type); + } + + @Test + public void testNewTimePartitionsStillGrowSingleSlotToMinimum() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + configureDataPartitionConsensusAndAllocation(); + for (int time = 0; time < 6; time++) { + assertSuccess( + partitionManager + .getOrCreateDataPartition( + new GetOrCreateDataPartitionPlan( + Collections.singletonMap(DATABASE, dataRequest(time, 0)))) + .getStatus()); + assertEquals(Math.min(4, time + 1), partitionInfo.getRegionGroupCount(DATABASE, type)); + } + } + + private Map dataRequest(long time, int... ids) { + Map request = new HashMap<>(); + for (TSeriesPartitionSlot slot : slots(ids)) { + request.put( + slot, + new TTimeSlotList(Collections.singletonList(new TTimePartitionSlot(time)), false, false)); + } + return request; + } + + private void configureDataPartitionConsensusAndAllocation() throws Exception { + when(consensusManager.read(any(GetDataPartitionPlan.class))) + .thenAnswer(invocation -> partitionInfo.getDataPartition(invocation.getArgument(0))); + when(consensusManager.confirmLeader()).thenReturn(RpcUtils.SUCCESS_STATUS); + when(consensusManager.write(any(CreateDataPartitionPlan.class))) + .thenAnswer(invocation -> partitionInfo.createDataPartition(invocation.getArgument(0))); + when(loadManager.allocateDataPartition(anyMap())) + .thenAnswer( + invocation -> { + Map> pending = + invocation.getArgument(0); + Map assigned = new HashMap<>(); + for (Map.Entry> database : + pending.entrySet()) { + DataPartitionTable table = new DataPartitionTable(); + TConsensusGroupId region = + partitionInfo.getAllRegionGroupIds(database.getKey(), type).get(0); + database + .getValue() + .forEach( + (slot, times) -> { + SeriesPartitionTable series = new SeriesPartitionTable(); + times + .getTimePartitionSlots() + .forEach(time -> series.putDataPartition(time, region)); + table.getDataPartitionMap().put(slot, series); + }); + assigned.put(database.getKey(), table); + } + return assigned; + }); + } + + @Test + public void testBatchCountsDistinctPersistedAndIncomingSlots() throws Exception { + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + assertSuccess(extend(DATABASE, slots(1))); + persistSlots(DATABASE, type, 0, 1); + clearInvocations(loadManager); + + assertSuccess(extend(DATABASE, slots(1, 2, 2, 3, 3))); + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 2), type); + persistSlots(DATABASE, type, 0, 2, 3); + assertEquals(3, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, type, slots(1, 1))); + + clearInvocations(loadManager); + for (int time = 1; time <= 3; time++) { + assertSuccess(extend(DATABASE, slots(1, 2))); + persistSlots(DATABASE, type, time, 1, 2); + } + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testSchemaAndDataActivationAreIndependent() throws Exception { + partitionInfo.createRegionGroups( + regionPlan(Collections.singletonMap(DATABASE, 1), otherType())); + persistSlots(DATABASE, otherType(), 0, 10, 11, 12, 13); + + assertSuccess(extend(DATABASE, slots(10))); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + persistSlots(DATABASE, type, 0, 10); + assertEquals(1, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, type, slots(10))); + assertEquals(4, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, otherType(), slots(10))); + + assertSuccess(extend(DATABASE, slots(11))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, otherType())); + } + + @Test + public void testBatchUsesEachDatabaseMaximum() throws Exception { + Map> requested = new HashMap<>(); + requested.put(DATABASE, slots(0, 1, 2, 3)); + requested.put(SECOND_DATABASE, slots(0, 1, 2, 3, 4, 5)); + assertSuccess(partitionManager.extendRegionGroupIfNecessary(requested, type)); + assertEquals(4, partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(3, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + Map expected = new HashMap<>(); + expected.put(DATABASE, 4); + expected.put(SECOND_DATABASE, 3); + verify(loadManager).allocateRegionGroups(expected, type); + } + + @Test + public void testNoPreallocationWithoutActivatedSlots() throws Exception { + assertSuccess(partitionManager.extendRegionGroupIfNecessary(Collections.emptyMap(), type)); + assertSuccess(extend(DATABASE, Collections.emptyList())); + assertEquals(0, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testSingleSlotTimePartitionsReachDefaultDataMinimumLikeAuto() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + checkSingleSlotTimePartitionsReachMinimumLikeAuto(2); + } + + @Test + public void testSingleSlotTimePartitionsReachConfiguredDataMinimumLikeAuto() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + checkSingleSlotTimePartitionsReachMinimumLikeAuto(4); + } + + @Test + public void testEachDatabaseReachesItsOwnMinimum() throws Exception { + assumeTrue(type == TConsensusGroupType.DataRegion); + when(schemaManager.getMinRegionGroupNum(SECOND_DATABASE, type)).thenReturn(2); + + for (int time = 0; time < 6; time++) { + Map> pending = new HashMap<>(); + for (String database : Arrays.asList(DATABASE, SECOND_DATABASE)) { + pending.put(database, pendingDataSlots(database, time)); + } + assertSuccess(partitionManager.extendRegionGroupIfNecessary(pending, type)); + assertEquals(Math.min(time + 1, 4), partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(Math.min(time + 1, 2), partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + persistSlots(DATABASE, type, time, 0); + persistSlots(SECOND_DATABASE, type, time, 0); + } + } + + @Test + public void testNoMinimumGrowthWithoutPendingSlots() throws Exception { + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + clearInvocations(loadManager); + + assertSuccess(partitionManager.extendRegionGroupIfNecessary(Collections.emptyMap(), type)); + assertSuccess(extend(DATABASE, Collections.emptyList())); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testExistingRegionsAreNotRemovedOrExceeded() throws Exception { + partitionInfo.createRegionGroups(regionPlan(Collections.singletonMap(DATABASE, 4), type)); + when(schemaManager.getMaxRegionGroupNum(DATABASE, type)).thenReturn(3); + assertSuccess(extend(DATABASE, slots(0))); + assertSuccess(extend(DATABASE, slots(0, 1, 2, 3, 4))); + assertEquals(4, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testAllDisabledRegionsCreateOneReplacement() throws Exception { + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + TConsensusGroupId disabledGroup = partitionInfo.getAllRegionGroupIds(DATABASE, type).get(0); + when(loadManager.getRegionGroupStatus(disabledGroup)).thenReturn(RegionGroupStatus.Disabled); + clearInvocations(loadManager); + + assertSuccess(extend(DATABASE, slots(0))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 1), type); + for (TConsensusGroupId group : partitionInfo.getAllRegionGroupIds(DATABASE, type)) { + if (!disabledGroup.equals(group)) { + when(loadManager.getRegionGroupStatus(group)).thenReturn(RegionGroupStatus.Running); + } + } + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0))); + assertSuccess(extend(DATABASE, slots(1))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testAllDisabledRegionsRespectMaximum() throws Exception { + assertSuccess(extend(DATABASE, slots(0, 1, 2, 3, 4))); + persistSlots(DATABASE, type, 0, 0, 1, 2, 3, 4); + for (TConsensusGroupId group : partitionInfo.getAllRegionGroupIds(DATABASE, type)) { + when(loadManager.getRegionGroupStatus(group)).thenReturn(RegionGroupStatus.Disabled); + } + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0))); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testDisabledStatusOfOtherTypeDoesNotAffectReplacement() throws Exception { + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + partitionInfo.createRegionGroups( + regionPlan(Collections.singletonMap(DATABASE, 1), otherType())); + TConsensusGroupId currentGroup = partitionInfo.getAllRegionGroupIds(DATABASE, type).get(0); + TConsensusGroupId otherGroup = partitionInfo.getAllRegionGroupIds(DATABASE, otherType()).get(0); + when(loadManager.getRegionGroupStatus(currentGroup)).thenReturn(RegionGroupStatus.Running); + when(loadManager.getRegionGroupStatus(otherGroup)).thenReturn(RegionGroupStatus.Disabled); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0))); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + + when(loadManager.getRegionGroupStatus(currentGroup)).thenReturn(RegionGroupStatus.Disabled); + when(loadManager.getRegionGroupStatus(otherGroup)).thenReturn(RegionGroupStatus.Running); + assertSuccess(extend(DATABASE, slots(0))); + assertEquals(2, partitionInfo.getRegionGroupCount(DATABASE, type)); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, otherType())); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 1), type); + } + + @Test + public void testNoReplacementWithoutPendingSlots() throws Exception { + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + TConsensusGroupId group = partitionInfo.getAllRegionGroupIds(DATABASE, type).get(0); + when(loadManager.getRegionGroupStatus(group)).thenReturn(RegionGroupStatus.Disabled); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, Collections.emptyList())); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testCustomStillAllocatesMaximumImmediately() throws Exception { + setPolicy(RegionGroupExtensionPolicy.CUSTOM); + assertSuccess(extend(DATABASE, slots(0))); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(1))); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testAutoKeepsSlowerGrowthAndSwitchCatchesUp() throws Exception { + setPolicy(RegionGroupExtensionPolicy.AUTO); + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + for (int slot = 0; slot < 3; slot++) { + assertSuccess(extend(DATABASE, slots(slot))); + persistSlots(DATABASE, type, 0, slot); + } + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + assertSuccess(extend(DATABASE, slots(3))); + assertEquals(4, partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + @Test + public void testAutoAndProactiveReachSameMaximum() throws Exception { + setPolicy(RegionGroupExtensionPolicy.AUTO); + List allSlots = + IntStream.range(0, CONF.getSeriesSlotNum()) + .mapToObj(TSeriesPartitionSlot::new) + .collect(Collectors.toList()); + assertSuccess(extend(DATABASE, allSlots)); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + when(schemaManager.getMaxRegionGroupNum(SECOND_DATABASE, type)).thenReturn(5); + assertSuccess(extend(SECOND_DATABASE, allSlots)); + assertEquals(5, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + } + + @Test + public void testSixSlotsGrowFasterThanAutoWithOneThousandSlots() throws Exception { + configureSparseSlotComparison(); + int minimum = type == TConsensusGroupType.SchemaRegion ? 1 : 2; + setPolicy(RegionGroupExtensionPolicy.AUTO); + for (int slot = 0; slot < 6; slot++) { + assertSuccess(extend(DATABASE, slots(slot))); + persistSlots(DATABASE, type, 0, slot); + assertEquals(Math.min(slot + 1, minimum), partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + for (int slot = 0; slot < 6; slot++) { + assertSuccess(extend(SECOND_DATABASE, slots(slot))); + persistSlots(SECOND_DATABASE, type, 0, slot); + assertEquals(slot + 1, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + } + // Six active slots are still below the nine-group resource limit. + assertEquals(6, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + clearInvocations(loadManager); + assertSuccess(extend(SECOND_DATABASE, slots(0, 1, 2, 3, 4, 5))); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + + for (int slot = 6; slot < 11; slot++) { + assertSuccess(extend(SECOND_DATABASE, slots(slot))); + persistSlots(SECOND_DATABASE, type, 0, slot); + assertEquals(Math.min(slot + 1, 9), partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + } + } + + @Test + public void testSixSlotBatchGrowsFasterThanAutoWithOneThousandSlots() throws Exception { + configureSparseSlotComparison(); + setPolicy(RegionGroupExtensionPolicy.AUTO); + assertSuccess(extend(DATABASE, slots(0, 1, 2, 3, 4, 5))); + assertEquals( + type == TConsensusGroupType.SchemaRegion ? 1 : 2, + partitionInfo.getRegionGroupCount(DATABASE, type)); + + setPolicy(RegionGroupExtensionPolicy.PROACTIVE); + assertSuccess(extend(SECOND_DATABASE, slots(0, 1, 2, 3, 4, 5))); + assertEquals(6, partitionInfo.getRegionGroupCount(SECOND_DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(SECOND_DATABASE, 6), type); + } + + @Test + public void testFilteredRequestCountsEachSeriesSlotOnce() throws Exception { + when(schemaManager.getMinRegionGroupNum(DATABASE, type)).thenReturn(1); + assertSuccess(extend(DATABASE, slots(0))); + persistSlots(DATABASE, type, 0, 0); + + Collection pendingSlots; + if (type == TConsensusGroupType.SchemaRegion) { + pendingSlots = + partitionInfo + .filterUnassignedSchemaPartitionSlots( + Collections.singletonMap(DATABASE, slots(0, 1, 2))) + .get(DATABASE); + assertEquals(slots(1, 2), pendingSlots); + } else { + Map request = new HashMap<>(); + for (TSeriesPartitionSlot slot : slots(0, 1, 2)) { + request.put( + slot, + new TTimeSlotList(Collections.singletonList(new TTimePartitionSlot(1)), false, false)); + } + pendingSlots = + partitionInfo + .filterUnassignedDataPartitionSlots(Collections.singletonMap(DATABASE, request)) + .get(DATABASE) + .keySet(); + // Filtering inserted empty entries for slots 1 and 2, but only slot 0 is assigned. + assertEquals( + 1, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, type, Collections.emptyList())); + assertEquals(2, partitionInfo.getSeriesPartitionSlotsCount(DATABASE, type, slots(3, 3))); + } + assertSuccess(extend(DATABASE, pendingSlots)); + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + @Test + public void testPreDeletedDatabaseIsReported() throws Exception { + partitionInfo.preDeleteDatabase( + new PreDeleteDatabasePlan(DATABASE, PreDeleteDatabasePlan.PreDeleteType.EXECUTE)); + assertEquals( + TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), extend(DATABASE, slots(0)).getCode()); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testHigherResourceMaximumAllowsFurtherGrowth() throws Exception { + when(schemaManager.getMaxRegionGroupNum(DATABASE, type)).thenReturn(3); + assertSuccess(extend(DATABASE, slots(0, 1, 2, 3, 4, 5))); + persistSlots(DATABASE, type, 0, 0, 1, 2, 3, 4, 5); + assertEquals(3, partitionInfo.getRegionGroupCount(DATABASE, type)); + clearInvocations(loadManager); + + when(schemaManager.getMaxRegionGroupNum(DATABASE, type)).thenReturn(5); + assertSuccess(extend(DATABASE, slots(6))); + assertEquals(5, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 2), type); + } + + @Test + public void testAllocationFailureIsReturnedWithoutPersistingRegions() throws Exception { + doThrow(new NotEnoughDataNodeException(Collections.emptyList(), 3)) + .when(loadManager) + .allocateRegionGroups(anyMap(), eq(type)); + assertEquals( + TSStatusCode.NO_ENOUGH_DATANODE.getStatusCode(), extend(DATABASE, slots(0)).getCode()); + assertEquals(0, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(procedureManager, never()).createRegionGroups(any(), any()); + } + + @Test + public void testProcedureFailureIsReturned() throws Exception { + TSStatus failure = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + doReturn(failure).when(procedureManager).createRegionGroups(eq(type), any()); + assertEquals(failure, extend(DATABASE, slots(0))); + assertEquals(0, partitionInfo.getRegionGroupCount(DATABASE, type)); + } + + @Test + public void testRetryOnlyCreatesRemainingRegionsAfterPartialSuccess() throws Exception { + TSStatus failure = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + doAnswer( + invocation -> { + CreateRegionGroupsPlan requested = invocation.getArgument(1); + CreateRegionGroupsPlan partial = new CreateRegionGroupsPlan(); + partial.addRegionGroup(DATABASE, requested.getRegionGroupMap().get(DATABASE).get(0)); + partitionInfo.createRegionGroups(partial); + return failure; + }) + .doAnswer( + invocation -> { + partitionInfo.createRegionGroups(invocation.getArgument(1)); + return RpcUtils.SUCCESS_STATUS; + }) + .when(procedureManager) + .createRegionGroups(eq(type), any()); + + assertEquals(failure, extend(DATABASE, slots(0, 1, 2))); + assertEquals(1, partitionInfo.getRegionGroupCount(DATABASE, type)); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0, 1, 2))); + // The three pending slots can now complete the configured minimum of four. + assertEquals(4, partitionInfo.getRegionGroupCount(DATABASE, type)); + verify(loadManager).allocateRegionGroups(Collections.singletonMap(DATABASE, 3), type); + persistSlots(DATABASE, type, 0, 0, 1, 2); + clearInvocations(loadManager); + assertSuccess(extend(DATABASE, slots(0, 1, 2))); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + @Test + public void testMissingDatabaseIsReported() throws Exception { + assertEquals( + TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), + extend("root.missing", slots(0)).getCode()); + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + + private TSStatus extend(String database, Collection requested) { + return partitionManager.extendRegionGroupIfNecessary( + Collections.singletonMap(database, requested), type); + } + + private void checkSingleSlotTimePartitionsReachMinimumLikeAuto(int minimum) throws Exception { + for (String database : Arrays.asList(DATABASE, SECOND_DATABASE)) { + when(schemaManager.getMinRegionGroupNum(database, type)).thenReturn(minimum); + when(schemaManager.getMaxRegionGroupNum(database, type)).thenReturn(5); + setPolicy( + database.equals(DATABASE) + ? RegionGroupExtensionPolicy.AUTO + : RegionGroupExtensionPolicy.PROACTIVE); + + for (int request = 0; request < minimum + 2; request++) { + // Several new time partitions of one existing series slot still add at most one group. + int firstTime = request * 3; + Collection pending = + pendingDataSlots(database, firstTime, firstTime + 1, firstTime + 2); + assertEquals(slots(0), pending); + if (request >= minimum) { + clearInvocations(loadManager); + } + assertSuccess(extend(database, pending)); + assertEquals( + Math.min(request + 1, minimum), partitionInfo.getRegionGroupCount(database, type)); + if (request >= minimum) { + verify(loadManager, never()).allocateRegionGroups(anyMap(), any()); + } + for (int time = firstTime; time < firstTime + 3; time++) { + persistSlots(database, type, time, 0); + } + assertEquals( + 1, partitionInfo.getSeriesPartitionSlotsCount(database, type, Collections.emptyList())); + assertEquals( + Collections.emptyList(), + unassignedDataSlots(database, firstTime) + .get(new TSeriesPartitionSlot(0)) + .getTimePartitionSlots()); + } + } + } + + private List pendingDataSlots(String database, int... times) { + return unassignedDataSlots(database, times).keySet().stream().collect(Collectors.toList()); + } + + private Map unassignedDataSlots( + String database, int... times) { + TTimeSlotList timeSlots = + new TTimeSlotList( + Arrays.stream(times).mapToObj(TTimePartitionSlot::new).collect(Collectors.toList()), + false, + false); + return partitionInfo + .filterUnassignedDataPartitionSlots( + Collections.singletonMap( + database, Collections.singletonMap(new TSeriesPartitionSlot(0), timeSlots))) + .get(database); + } + + private void configureSparseSlotComparison() { + CONF.setSeriesSlotNum(1000); + int minimum = type == TConsensusGroupType.SchemaRegion ? 1 : 2; + for (String database : Arrays.asList(DATABASE, SECOND_DATABASE)) { + when(schemaManager.getMinRegionGroupNum(database, type)).thenReturn(minimum); + when(schemaManager.getMaxRegionGroupNum(database, type)).thenReturn(9); + } + } + + private void setPolicy(RegionGroupExtensionPolicy policy) { + if (type == TConsensusGroupType.SchemaRegion) { + CONF.setSchemaRegionGroupExtensionPolicy(policy); + } else { + CONF.setDataRegionGroupExtensionPolicy(policy); + } + } + + private TConsensusGroupType otherType() { + return type == TConsensusGroupType.SchemaRegion + ? TConsensusGroupType.DataRegion + : TConsensusGroupType.SchemaRegion; + } + + private CreateRegionGroupsPlan regionPlan( + Map allotments, TConsensusGroupType allocatedType) { + CreateRegionGroupsPlan plan = new CreateRegionGroupsPlan(); + allotments.forEach( + (database, count) -> { + for (int i = 0; i < count; i++) { + plan.addRegionGroup( + database, + new TRegionReplicaSet( + new TConsensusGroupId(allocatedType, partitionInfo.generateNextRegionGroupId()), + Collections.singletonList(new TDataNodeLocation().setDataNodeId(0)))); + } + }); + return plan; + } + + private void persistSlots( + String database, TConsensusGroupType persistedType, long time, int... slotIds) + throws Exception { + TConsensusGroupId regionId = partitionInfo.getAllRegionGroupIds(database, persistedType).get(0); + if (persistedType == TConsensusGroupType.SchemaRegion) { + SchemaPartitionTable table = new SchemaPartitionTable(); + for (TSeriesPartitionSlot slot : slots(slotIds)) { + table.getSchemaPartitionMap().put(slot, regionId); + } + CreateSchemaPartitionPlan plan = new CreateSchemaPartitionPlan(); + plan.setAssignedSchemaPartition(Collections.singletonMap(database, table)); + partitionInfo.createSchemaPartition(plan); + } else { + DataPartitionTable table = new DataPartitionTable(); + for (TSeriesPartitionSlot slot : slots(slotIds)) { + SeriesPartitionTable seriesTable = new SeriesPartitionTable(); + seriesTable.putDataPartition(new TTimePartitionSlot(time), regionId); + table.getDataPartitionMap().put(slot, seriesTable); + } + CreateDataPartitionPlan plan = new CreateDataPartitionPlan(); + plan.setAssignedDataPartition(Collections.singletonMap(database, table)); + partitionInfo.createDataPartition(plan); + } + } + + private static List slots(int... ids) { + return Arrays.stream(ids).mapToObj(TSeriesPartitionSlot::new).collect(Collectors.toList()); + } + + private static void assertSuccess(TSStatus status) { + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } +} diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 2c6186f6b4dd8..ce806f57541ec 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -678,51 +678,86 @@ series_slot_num=1000 # Datatype: String series_partition_executor_class=org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor -# The policy of extension SchemaRegionGroup for each Database. +# The policy of extending SchemaRegionGroups for each Database. # These policies are currently supported: -# 1. CUSTOM(Each Database will allocate schema_region_group_per_database RegionGroups as soon as created) -# 2. AUTO(Each Database will automatically extend SchemaRegionGroups based on the data it has) +# 1. CUSTOM: For known workloads that need manual resource allocation; creates the configured target +# number of SchemaRegionGroups per Database when schema partitions are first allocated. +# 2. AUTO: For most routine workloads (roughly 80% as an approximate planning guideline); gradually +# meets the configured minimum and expands with slot occupancy up to the resource-based maximum, +# balancing parallelism and RegionGroup management overhead. +# 3. PROACTIVE: Complements AUTO where it does not expand eagerly enough, such as very few devices with +# many measurements and a high load; creates groups earlier for more parallelism at the cost of +# additional RegionGroup management overhead. +# AUTO and PROACTIVE use the same per-Database maximum calculation, including resource sharing across +# Databases. PROACTIVE expands more eagerly within this limit; both have the same group count once +# they reach the same maximum. +# PROACTIVE targets one SchemaRegionGroup per distinct schema series slot, not per measurement, +# and grows toward the configured minimum incrementally like AUTO, without exceeding the maximum. +# If all SchemaRegionGroups are disabled, PROACTIVE may add one group above this target, +# without exceeding the maximum, to allow partition allocation to continue. +# Switching policies does not remove existing RegionGroups. # effectiveMode: hot_reload # Datatype: String -schema_region_group_extension_policy=AUTO +schema_region_group_extension_policy=PROACTIVE # When set schema_region_group_extension_policy=CUSTOM, # this parameter is the default number of SchemaRegionGroups for each Database. -# When set schema_region_group_extension_policy=AUTO, -# this parameter is the default minimal number of SchemaRegionGroups for each Database. +# When set schema_region_group_extension_policy=AUTO or PROACTIVE, +# this parameter is the default minimum number of SchemaRegionGroups and a lower bound for the per-Database maximum. +# Both policies grow toward this minimum incrementally, adding at most the number of pending series slots +# to satisfy the minimum. PROACTIVE also grows with the number of active series slots. # effectiveMode: hot_reload # Datatype: Integer default_schema_region_group_num_per_database=1 -# Only take effect when set schema_region_group_extension_policy=AUTO. -# This parameter is the maximum number of SchemaRegions expected to be managed by each DataNode. -# Notice: Since each Database requires at least one SchemaRegionGroup to manage its schema, -# this parameter doesn't limit the upper bound of cluster SchemaRegions when there are too many Databases. +# Only takes effect when schema_region_group_extension_policy=AUTO or PROACTIVE. +# The expected number of SchemaRegions per DataNode, used to calculate the same per-Database maximum +# for AUTO and PROACTIVE. This is not a hard limit on node or cluster totals: per-Database minimums, +# rounding, and existing groups can raise the total above this resource estimate. # effectiveMode: hot_reload # Datatype: Integer schema_region_per_data_node=1 -# The policy of extension DataRegionGroup for each Database. +# The policy of extending DataRegionGroups for each Database. # These policies are currently supported: -# 1. CUSTOM(Each Database will allocate data_region_group_per_database DataRegionGroups as soon as created) -# 2. AUTO(Each Database will automatically extend DataRegionGroups based on the data it has) +# 1. CUSTOM: For known workloads that need manual resource allocation; creates the configured target +# number of DataRegionGroups per Database when data partitions are first allocated. +# 2. AUTO: For most routine workloads (roughly 80% as an approximate planning guideline); gradually +# meets the configured minimum and expands with slot occupancy up to the resource-based maximum, +# balancing parallelism and RegionGroup management overhead. +# 3. PROACTIVE: Complements AUTO where it does not expand eagerly enough, such as very few devices with +# many measurements and a high load; creates groups earlier for more parallelism at the cost of +# additional RegionGroup management overhead. +# AUTO and PROACTIVE use the same per-Database maximum calculation, including resource sharing across +# Databases. PROACTIVE expands more eagerly within this limit; both have the same group count once +# they reach the same maximum. +# PROACTIVE targets one DataRegionGroup per distinct data series slot, not per measurement, +# and grows toward the configured minimum incrementally like AUTO, without exceeding the maximum. +# New time partitions in an existing slot can also trigger this incremental growth toward the minimum. +# If all DataRegionGroups are disabled, PROACTIVE may add one group above this target, +# without exceeding the maximum, to allow partition allocation to continue. +# PROACTIVE balances active slots in its allocation policy table so new time partitions can use new groups. +# Existing time-partition assignments are retained. +# Switching policies does not remove existing RegionGroups. # effectiveMode: hot_reload # Datatype: String -data_region_group_extension_policy=AUTO +data_region_group_extension_policy=PROACTIVE # When set data_region_group_extension_policy=CUSTOM, # this parameter is the default number of DataRegionGroups for each Database. -# When set data_region_group_extension_policy=AUTO, -# this parameter is the default minimal number of DataRegionGroups for each Database. +# When set data_region_group_extension_policy=AUTO or PROACTIVE, +# this parameter is the default minimum number of DataRegionGroups and a lower bound for the per-Database maximum. +# Both policies grow toward this minimum incrementally, adding at most the number of pending series slots +# to satisfy the minimum. PROACTIVE also grows with the number of active series slots. # effectiveMode: hot_reload # Datatype: Integer default_data_region_group_num_per_database=2 -# Only take effect when set data_region_group_extension_policy=AUTO. -# This parameter is the maximum number of DataRegions expected to be managed by each DataNode. -# Set to 0 means that each node automatically has the number of CPU cores / 2 regions -# Notice: Since each Database requires at least two DataRegionGroups to manage its data, -# this parameter doesn't limit the upper bound of cluster DataRegions when there are too many Databases. +# Only takes effect when data_region_group_extension_policy=AUTO or PROACTIVE. +# The expected number of DataRegions per DataNode, used to calculate the same per-Database maximum +# for AUTO and PROACTIVE. Set to 0 to use half the CPU core count as the expected number for each DataNode. +# This is not a hard limit on node or cluster totals: per-Database minimums, rounding, and existing +# groups can raise the total above this resource estimate. # effectiveMode: hot_reload # Datatype: Integer data_region_per_data_node=0