From 7cbb2cc5a05ead07a3edbbb1e25d28c146bcf043 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati <115860222+sreejasahithi@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:59:50 +0530 Subject: [PATCH 1/3] HDDS-16179. Dry run command for container balancer with profiles --- .../balancer/ContainerBalancerAdvisor.java | 408 ++++++++++++++++++ .../ContainerBalancerConfiguration.java | 103 ++++- .../balancer/ContainerBalancerEstimation.java | 193 +++++++++ .../balancer/ContainerBalancerProfile.java | 40 ++ .../TestContainerBalancerAdvisor.java | 292 +++++++++++++ .../scm/cli/ContainerBalancerCommands.java | 28 +- .../cli/ContainerBalancerConfigOptions.java | 175 ++++++++ .../ContainerBalancerDryRunSubcommand.java | 153 +++++++ .../cli/ContainerBalancerStartSubcommand.java | 92 +--- .../TestContainerBalancerSubCommand.java | 225 +++++++++- 10 files changed, 1631 insertions(+), 78 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerEstimation.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerProfile.java create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java create mode 100644 hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java create mode 100644 hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java new file mode 100644 index 000000000000..629fed5dfe4b --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java @@ -0,0 +1,408 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.balancer; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeUsageInfoProto; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; + +/** + * Orchestrates cluster analysis, estimation and recommendation for container balancer. + */ +public final class ContainerBalancerAdvisor { + + private static final long MIN_DELETE_PHASE_MILLIS = Duration.ofMinutes(9).toMillis(); + private static final String DATANODE_OFFSET_KEY = + "hdds.scm.replication.event.timeout.datanode.offset"; + + private ContainerBalancerAdvisor() { + } + + /** + * Estimates per-iteration size, iterations and duration for one or more balancer profiles. + * + * If profile is unset, returns SLOW, MEDIUM, and FAST. + * If profile is set, returns a result for that profile only. + * Per-profile validation failures are returned with {@link ContainerBalancerEstimation#succeeded()} false + * instead of aborting other profiles. + */ + public static List estimateDryRun(OzoneConfiguration conf, AdvisorRequest request) { + Objects.requireNonNull(conf, "conf"); + Objects.requireNonNull(request, "request"); + List nodes = Objects.requireNonNull(request.nodes, "nodes"); + + ContainerBalancerConfiguration balancerConfig = conf.getObject(ContainerBalancerConfiguration.class); + + double thresholdRatio = request.thresholdPercent != null + ? request.thresholdPercent / 100.0 + : balancerConfig.getThresholdAsRatio(); + Set includeNodes = request.includeNodes != null + ? request.includeNodes + : balancerConfig.getIncludeNodes(); + Set excludeNodes = request.excludeNodes != null + ? request.excludeNodes + : balancerConfig.getExcludeNodes(); + + ContainerBalancerClusterSnapshot snapshot = ContainerBalancerClusterAnalyzer.analyze(nodes, thresholdRatio, + includeNodes, excludeNodes); + validateSnapshotForEstimation(snapshot, conf); + + List profiles = selectProfiles(request); + List estimations = new ArrayList<>(profiles.size()); + for (ContainerBalancerProfile profile : profiles) { + estimations.add(estimateForProfile(conf, request, profile, snapshot, balancerConfig)); + } + return Collections.unmodifiableList(estimations); + } + + private static ContainerBalancerEstimation estimateForProfile(OzoneConfiguration conf, AdvisorRequest request, + ContainerBalancerProfile profile, ContainerBalancerClusterSnapshot snapshot, + ContainerBalancerConfiguration balancerConfig) { + + boolean userProvidedMaxDatanodesPercentage = + request.maxDatanodesPercentageToInvolvePerIteration != null; + int maxDatanodesPercentage = userProvidedMaxDatanodesPercentage + ? request.maxDatanodesPercentageToInvolvePerIteration + : profile.getDatanodesMaxPercentage(balancerConfig); + long maxSizeEnteringTarget = request.maxSizeEnteringTarget != null + ? request.maxSizeEnteringTarget + : profile.getMaxSizeEnteringTarget(balancerConfig); + long maxSizeLeavingSource = request.maxSizeLeavingSource != null + ? request.maxSizeLeavingSource + : profile.getMaxSizeLeavingSource(balancerConfig); + long maxSizeToMovePerIteration = request.maxSizeToMovePerIteration != null + ? request.maxSizeToMovePerIteration + : balancerConfig.getMaxSizeToMovePerIteration(); + long moveTimeoutMillis = request.moveTimeoutMillis != null + ? request.moveTimeoutMillis + : balancerConfig.getMoveTimeout().toMillis(); + long moveReplicationTimeoutMillis = request.moveReplicationTimeoutMillis != null + ? request.moveReplicationTimeoutMillis + : balancerConfig.getMoveReplicationTimeout().toMillis(); + long balancingIntervalMillis = request.balancingIntervalMillis != null + ? request.balancingIntervalMillis + : balancerConfig.getBalancingInterval().toMillis(); + + ContainerBalancerEstimation.Builder builder = ContainerBalancerEstimation.newBuilder() + .setProfile(profile) + .setMaxSizeEnteringTarget(maxSizeEnteringTarget) + .setMaxSizeLeavingSource(maxSizeLeavingSource) + .setMaxSizeToMovePerIteration(maxSizeToMovePerIteration) + .setMoveTimeoutMillis(moveTimeoutMillis) + .setBalancingIntervalMillis(balancingIntervalMillis); + + try { + validateMoveTimeouts(conf, moveReplicationTimeoutMillis, moveTimeoutMillis); + validateResolvedMoveLimits(conf, maxSizeEnteringTarget, maxSizeLeavingSource, maxSizeToMovePerIteration); + + int eligibleDatanodeCount = snapshot.getTotalEligibleDatanodes(); + int maxInvolved = ContainerBalancerConfiguration.computeMaxDatanodesToInvolvePerIteration( + maxDatanodesPercentage / 100d, eligibleDatanodeCount); + + if (!userProvidedMaxDatanodesPercentage && maxInvolved < 2) { + int maxProfileDatanodesPercentage = + balancerConfig.getProfileDatanodesMaxPercentage(ContainerBalancerProfile.FAST); + maxDatanodesPercentage = minimumPercentForAtLeastTwoNodes( + eligibleDatanodeCount, maxDatanodesPercentage, maxProfileDatanodesPercentage); + maxInvolved = ContainerBalancerConfiguration.computeMaxDatanodesToInvolvePerIteration( + maxDatanodesPercentage / 100d, eligibleDatanodeCount); + } + + if (maxInvolved < 2) { + throw new IllegalArgumentException(String.format( + "max-datanodes-percentage-to-involve-per-iteration=%d allows at most %d datanode(s) " + + "per iteration with %d eligible datanode(s), but at least 2 are required for a " + + "source and target datanode pair.", + maxDatanodesPercentage, maxInvolved, eligibleDatanodeCount)); + } + + int[] involved = computeInvolvedDatanodeCounts(snapshot.getSourceCount(), snapshot.getTargetCount(), maxInvolved); + + long bytesToMove = snapshot.getBytesToMove(); + long perIterationBytes = computePerIterationBytes( + bytesToMove, + maxSizeToMovePerIteration, + maxSizeLeavingSource, + maxSizeEnteringTarget, + involved); + long estimatedIterations = computeEstimatedIterations(perIterationBytes, bytesToMove); + long cycleTimeMillis = computeCycleTimeMillis(moveTimeoutMillis, balancingIntervalMillis); + long estimatedDurationMillis = estimatedIterations * cycleTimeMillis; + + return builder + .setMaxDatanodesPercentage(maxDatanodesPercentage) + .setBytesToMove(bytesToMove) + .setPerIterationBytes(perIterationBytes) + .setEstimatedIterations(estimatedIterations) + .setEstimatedDurationMillis(estimatedDurationMillis) + .build(); + } catch (IllegalArgumentException e) { + return builder + .setMaxDatanodesPercentage(maxDatanodesPercentage) + .setFailureMessage(e.getMessage()) + .build(); + } + } + + /** Raises datanode involvement percent until at least two datanodes can be involved. */ + static int minimumPercentForAtLeastTwoNodes( + int eligibleDatanodeCount, int startPercent, int maxPercent) { + for (int percent = startPercent; percent <= maxPercent; percent++) { + if (ContainerBalancerConfiguration.computeMaxDatanodesToInvolvePerIteration( + percent / 100d, eligibleDatanodeCount) >= 2) { + return percent; + } + } + return maxPercent; + } + + /** + * Estimated bytes moved in one iteration: minimum of global cap, source cap, + * target cap, and total bytes to move. + */ + static long computePerIterationBytes( + long bytesToMove, + long maxSizeToMovePerIteration, + long maxSizeLeavingSource, + long maxSizeEnteringTarget, + int[] involved) { + + long fromLeaving = involved[0] * maxSizeLeavingSource; + long fromEntering = involved[1] * maxSizeEnteringTarget; + + return minPositive( + maxSizeToMovePerIteration, + fromLeaving, + fromEntering, + bytesToMove); + } + + static long computeCycleTimeMillis(long moveTimeoutMillis, long balancingIntervalMillis) { + return moveTimeoutMillis + balancingIntervalMillis; + } + + static long computeEstimatedIterations(long perIterationBytes, long bytesToMove) { + if (perIterationBytes <= 0) { + throw new IllegalArgumentException("Per-iteration move size must be positive."); + } + return (long) Math.ceil(bytesToMove / (double) perIterationBytes); + } + + /** + * Estimated source and target datanode counts for one iteration (50/50 split heuristic). + * + * @return {@code [involvedSources, involvedTargets]} + */ + static int[] computeInvolvedDatanodeCounts( + int sourceCount, + int targetCount, + int maxInvolved) { + if (sourceCount <= 0 || targetCount <= 0 || maxInvolved < 2) { + return new int[] {0, 0}; + } + + int sEff = Math.min(sourceCount, (maxInvolved + 1) / 2); + int tEff = Math.min(targetCount, maxInvolved / 2); + return new int[] {sEff, tEff}; + } + + private static long minPositive(long... values) { + long result = Long.MAX_VALUE; + for (long value : values) { + if (value > 0 && value < result) { + result = value; + } + } + return result; + } + + private static void validateSnapshotForEstimation(ContainerBalancerClusterSnapshot snapshot, + OzoneConfiguration conf) { + long containerSizeBytes = (long) conf.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, + StorageUnit.BYTES); + if (snapshot.getSourceCount() < 1) { + throw new IllegalArgumentException("No over-utilized datanodes (sources) found."); + } + if (snapshot.getTargetCount() < 1) { + throw new IllegalArgumentException("No under-utilized datanodes (targets) found."); + } + if (snapshot.getBytesToMove() <= 0) { + throw new IllegalArgumentException("No bytes to move."); + } + if (snapshot.getBytesToMove() < containerSizeBytes) { + throw new IllegalArgumentException( + "Bytes to move (" + snapshot.getBytesToMove() + + ") is less than container size (" + containerSizeBytes + ")."); + } + if (snapshot.getTotalEligibleDatanodes() < 2) { + throw new IllegalArgumentException(String.format( + "Container Balancer found %d eligible datanode(s) but requires at least 2.", + snapshot.getTotalEligibleDatanodes())); + } + } + + private static void validateResolvedMoveLimits(OzoneConfiguration conf, long maxSizeEnteringTarget, + long maxSizeLeavingSource, long maxSizeToMovePerIteration) { + long containerSizeBytes = (long) conf.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, + StorageUnit.BYTES); + + if (maxSizeEnteringTarget <= containerSizeBytes) { + throw new IllegalArgumentException( + "max-size-entering-target must be greater than ozone.scm.container.size (" + + containerSizeBytes + " bytes)."); + } + if (maxSizeLeavingSource <= containerSizeBytes) { + throw new IllegalArgumentException( + "max-size-leaving-source must be greater than ozone.scm.container.size (" + + containerSizeBytes + " bytes)."); + } + if (maxSizeEnteringTarget > maxSizeToMovePerIteration) { + throw new IllegalArgumentException( + "max-size-entering-target must be less than or equal to " + + "max-size-to-move-per-iteration."); + } + if (maxSizeLeavingSource > maxSizeToMovePerIteration) { + throw new IllegalArgumentException( + "max-size-leaving-source must be less than or equal to " + + "max-size-to-move-per-iteration."); + } + } + + private static void validateMoveTimeouts(OzoneConfiguration conf, long moveReplicationTimeoutMillis, + long moveTimeoutMillis) { + if (moveReplicationTimeoutMillis >= moveTimeoutMillis) { + throw new IllegalArgumentException("hdds.container.balancer.move.replication.timeout should " + + "be less than hdds.container.balancer.move.timeout."); + } + long datanodeOffsetMillis = conf.getTimeDuration( + DATANODE_OFFSET_KEY, Duration.ofMinutes(6).toMillis(), TimeUnit.MILLISECONDS); + if ((moveTimeoutMillis - moveReplicationTimeoutMillis - datanodeOffsetMillis) + < MIN_DELETE_PHASE_MILLIS) { + String msg = String.format("(hdds.container.balancer.move.timeout (%sms) - " + + "hdds.container.balancer.move.replication.timeout (%sms) - " + + "hdds.scm.replication.event.timeout.datanode.offset (%sms)) " + + "should be greater than or equal to 540000ms or 9 minutes.", + moveTimeoutMillis, + moveReplicationTimeoutMillis, + datanodeOffsetMillis); + throw new IllegalArgumentException(msg); + } + } + + private static List selectProfiles(AdvisorRequest request) { + if (request.profile != null) { + return Collections.singletonList(request.profile); + } + return Arrays.asList( + ContainerBalancerProfile.SLOW, + ContainerBalancerProfile.MEDIUM, + ContainerBalancerProfile.FAST); + } + + /** + * Input for {@link ContainerBalancerAdvisor}: cluster usage data and optional overrides. + * Unset fields fall back to {@link ContainerBalancerConfiguration} or profile presets. + */ + public static final class AdvisorRequest { + private List nodes; + private Set includeNodes; + private Set excludeNodes; + private Double thresholdPercent; + private ContainerBalancerProfile profile; + private Integer maxDatanodesPercentageToInvolvePerIteration; + private Long maxSizeToMovePerIteration; + private Long maxSizeEnteringTarget; + private Long maxSizeLeavingSource; + private Long moveTimeoutMillis; + private Long moveReplicationTimeoutMillis; + private Long balancingIntervalMillis; + + public AdvisorRequest setNodes(List nodesList) { + this.nodes = nodesList; + return this; + } + + public AdvisorRequest setIncludeNodes(Set includeNodesSet) { + this.includeNodes = includeNodesSet; + return this; + } + + public AdvisorRequest setExcludeNodes(Set excludeNodesSet) { + this.excludeNodes = excludeNodesSet; + return this; + } + + public AdvisorRequest setThresholdPercent(Double threshold) { + this.thresholdPercent = threshold; + return this; + } + + public AdvisorRequest setProfile(ContainerBalancerProfile profileValue) { + this.profile = profileValue; + return this; + } + + public AdvisorRequest setMaxDatanodesPercentageToInvolvePerIteration(Integer percentage) { + this.maxDatanodesPercentageToInvolvePerIteration = percentage; + return this; + } + + public AdvisorRequest setMaxSizeToMovePerIteration(Long maxSize) { + this.maxSizeToMovePerIteration = maxSize; + return this; + } + + public AdvisorRequest setMaxSizeEnteringTarget(Long maxSize) { + this.maxSizeEnteringTarget = maxSize; + return this; + } + + public AdvisorRequest setMaxSizeLeavingSource(Long maxSize) { + this.maxSizeLeavingSource = maxSize; + return this; + } + + public AdvisorRequest setMoveTimeoutMillis(Long timeoutMillis) { + this.moveTimeoutMillis = timeoutMillis; + return this; + } + + public AdvisorRequest setMoveReplicationTimeoutMillis(Long timeoutMillis) { + this.moveReplicationTimeoutMillis = timeoutMillis; + return this; + } + + public AdvisorRequest setBalancingIntervalMillis(Long intervalMillis) { + this.balancingIntervalMillis = intervalMillis; + return this; + } + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java index 65acb9da8e15..ebd79f71b029 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java @@ -152,6 +152,51 @@ public final class ContainerBalancerConfiguration { "OVER_REPLICATED CLOSED/QUASI_CLOSED and HEALTHY QUASI_CLOSED containers.") private boolean includeNonStandardContainers = false; + @Config(key = "hdds.container.balancer.profile.slow.datanodes.involved.max.percentage", type = ConfigType.INT, + defaultValue = "10", tags = {ConfigTag.BALANCER}, + description = "SLOW profile: max percent of eligible datanodes used in one iteration.") + private int profileSlowDatanodesMaxPercentage = 10; + + @Config(key = "hdds.container.balancer.profile.slow.size.entering.target.max", type = ConfigType.SIZE, + defaultValue = "10GB", tags = {ConfigTag.BALANCER}, + description = "SLOW profile: max bytes a target datanode may receive in one iteration.") + private long profileSlowMaxSizeEnteringTarget = 10 * OzoneConsts.GB; + + @Config(key = "hdds.container.balancer.profile.slow.size.leaving.source.max", type = ConfigType.SIZE, + defaultValue = "10GB", tags = {ConfigTag.BALANCER}, + description = "SLOW profile: max bytes a source datanode may send in one iteration.") + private long profileSlowMaxSizeLeavingSource = 10 * OzoneConsts.GB; + + @Config(key = "hdds.container.balancer.profile.medium.datanodes.involved.max.percentage", type = ConfigType.INT, + defaultValue = "20", tags = {ConfigTag.BALANCER}, + description = "MEDIUM profile: max percent of eligible datanodes used in one iteration.") + private int profileMediumDatanodesMaxPercentage = 20; + + @Config(key = "hdds.container.balancer.profile.medium.size.entering.target.max", type = ConfigType.SIZE, + defaultValue = "26GB", tags = {ConfigTag.BALANCER}, + description = "MEDIUM profile: max bytes a target datanode may receive in one iteration.") + private long profileMediumMaxSizeEnteringTarget = 26 * OzoneConsts.GB; + + @Config(key = "hdds.container.balancer.profile.medium.size.leaving.source.max", type = ConfigType.SIZE, + defaultValue = "26GB", tags = {ConfigTag.BALANCER}, + description = "MEDIUM profile: max bytes a source datanode may send in one iteration.") + private long profileMediumMaxSizeLeavingSource = 26 * OzoneConsts.GB; + + @Config(key = "hdds.container.balancer.profile.fast.datanodes.involved.max.percentage", type = ConfigType.INT, + defaultValue = "40", tags = {ConfigTag.BALANCER}, + description = "FAST profile: max percent of eligible datanodes used in one iteration.") + private int profileFastDatanodesMaxPercentage = 40; + + @Config(key = "hdds.container.balancer.profile.fast.size.entering.target.max", type = ConfigType.SIZE, + defaultValue = "100GB", tags = {ConfigTag.BALANCER}, + description = "FAST profile: max bytes a target datanode may receive in one iteration.") + private long profileFastMaxSizeEnteringTarget = 100 * OzoneConsts.GB; + + @Config(key = "hdds.container.balancer.profile.fast.size.leaving.source.max", type = ConfigType.SIZE, + defaultValue = "100GB", tags = {ConfigTag.BALANCER}, + description = "FAST profile: max bytes a source datanode may send in one iteration.") + private long profileFastMaxSizeLeavingSource = 100 * OzoneConsts.GB; + /** * Gets the threshold value for Container Balancer. * @@ -261,7 +306,21 @@ public double getMaxDatanodesRatioToInvolvePerIteration() { * @return maximum datanodes that may be involved in one iteration */ public int computeMaxDatanodesToInvolvePerIteration(int eligibleDatanodeCount) { - return (int) (getMaxDatanodesRatioToInvolvePerIteration() * eligibleDatanodeCount); + return computeMaxDatanodesToInvolvePerIteration( + getMaxDatanodesRatioToInvolvePerIteration(), eligibleDatanodeCount); + } + + /** + * Computes the maximum number of datanodes that may be involved in an + * iteration for the given percentage and eligible datanode count. + * + * @param maxDatanodesPercentage percentage of eligible datanodes to involve + * @param eligibleDatanodeCount number of healthy, in-service datanodes + * @return maximum datanodes that may be involved in one iteration + */ + public static int computeMaxDatanodesToInvolvePerIteration( + double maxDatanodesPercentage, int eligibleDatanodeCount) { + return (int) (maxDatanodesPercentage * eligibleDatanodeCount); } /** @@ -467,6 +526,48 @@ public void setIncludeNonStandardContainers(boolean enable) { includeNonStandardContainers = enable; } + /** Returns the preset max datanode involvement percent for the given profile. */ + public int getProfileDatanodesMaxPercentage(ContainerBalancerProfile profile) { + switch (profile) { + case SLOW: + return profileSlowDatanodesMaxPercentage; + case MEDIUM: + return profileMediumDatanodesMaxPercentage; + case FAST: + return profileFastDatanodesMaxPercentage; + default: + throw new IllegalArgumentException("Unknown profile: " + profile); + } + } + + /** Returns the preset max bytes entering a target datanode per iteration for the given profile. */ + public long getProfileMaxSizeEnteringTarget(ContainerBalancerProfile profile) { + switch (profile) { + case SLOW: + return profileSlowMaxSizeEnteringTarget; + case MEDIUM: + return profileMediumMaxSizeEnteringTarget; + case FAST: + return profileFastMaxSizeEnteringTarget; + default: + throw new IllegalArgumentException("Unknown profile: " + profile); + } + } + + /** Returns the preset max bytes leaving a source datanode per iteration for the given profile. */ + public long getProfileMaxSizeLeavingSource(ContainerBalancerProfile profile) { + switch (profile) { + case SLOW: + return profileSlowMaxSizeLeavingSource; + case MEDIUM: + return profileMediumMaxSizeLeavingSource; + case FAST: + return profileFastMaxSizeLeavingSource; + default: + throw new IllegalArgumentException("Unknown profile: " + profile); + } + } + @Override public String toString() { return String.format("Container Balancer Configuration values:%n" + diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerEstimation.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerEstimation.java new file mode 100644 index 000000000000..59082cc5da6c --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerEstimation.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.balancer; + +import java.util.Objects; + +/** + * Dry-run estimate for a single container balancer profile. + */ +public final class ContainerBalancerEstimation { + + private final ContainerBalancerProfile profile; + private final String failureMessage; + private final long bytesToMove; + private final long perIterationBytes; + private final long estimatedIterations; + private final long estimatedDurationMillis; + private final int maxDatanodesPercentage; + private final long maxSizeEnteringTarget; + private final long maxSizeLeavingSource; + private final long maxSizeToMovePerIteration; + private final long moveTimeoutMillis; + private final long balancingIntervalMillis; + + private ContainerBalancerEstimation(Builder b) { + this.profile = Objects.requireNonNull(b.profile, "profile == null"); + this.failureMessage = b.failureMessage; + this.bytesToMove = b.bytesToMove; + this.perIterationBytes = b.perIterationBytes; + this.estimatedIterations = b.estimatedIterations; + this.estimatedDurationMillis = b.estimatedDurationMillis; + this.maxDatanodesPercentage = b.maxDatanodesPercentage; + this.maxSizeEnteringTarget = b.maxSizeEnteringTarget; + this.maxSizeLeavingSource = b.maxSizeLeavingSource; + this.maxSizeToMovePerIteration = b.maxSizeToMovePerIteration; + this.moveTimeoutMillis = b.moveTimeoutMillis; + this.balancingIntervalMillis = b.balancingIntervalMillis; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public ContainerBalancerProfile getProfile() { + return profile; + } + + public boolean succeeded() { + return failureMessage == null; + } + + public String getFailureMessage() { + return failureMessage; + } + + public long getBytesToMove() { + return bytesToMove; + } + + public long getPerIterationBytes() { + return perIterationBytes; + } + + public long getEstimatedIterations() { + return estimatedIterations; + } + + public long getEstimatedDurationMillis() { + return estimatedDurationMillis; + } + + public int getMaxDatanodesPercentage() { + return maxDatanodesPercentage; + } + + public long getMaxSizeEnteringTarget() { + return maxSizeEnteringTarget; + } + + public long getMaxSizeLeavingSource() { + return maxSizeLeavingSource; + } + + public long getMaxSizeToMovePerIteration() { + return maxSizeToMovePerIteration; + } + + public long getMoveTimeoutMillis() { + return moveTimeoutMillis; + } + + public long getBalancingIntervalMillis() { + return balancingIntervalMillis; + } + + /** Builder for {@link ContainerBalancerEstimation}. */ + public static final class Builder { + private ContainerBalancerProfile profile; + private String failureMessage; + private long bytesToMove; + private long perIterationBytes; + private long estimatedIterations; + private long estimatedDurationMillis; + private int maxDatanodesPercentage; + private long maxSizeEnteringTarget; + private long maxSizeLeavingSource; + private long maxSizeToMovePerIteration; + private long moveTimeoutMillis; + private long balancingIntervalMillis; + + private Builder() { + } + + public Builder setProfile(ContainerBalancerProfile profileValue) { + this.profile = profileValue; + return this; + } + + public Builder setFailureMessage(String message) { + this.failureMessage = message; + return this; + } + + public Builder setBytesToMove(long bytes) { + this.bytesToMove = bytes; + return this; + } + + public Builder setPerIterationBytes(long bytes) { + this.perIterationBytes = bytes; + return this; + } + + public Builder setEstimatedIterations(long iterations) { + this.estimatedIterations = iterations; + return this; + } + + public Builder setEstimatedDurationMillis(long durationMillis) { + this.estimatedDurationMillis = durationMillis; + return this; + } + + public Builder setMaxDatanodesPercentage(int percentage) { + this.maxDatanodesPercentage = percentage; + return this; + } + + public Builder setMaxSizeEnteringTarget(long bytes) { + this.maxSizeEnteringTarget = bytes; + return this; + } + + public Builder setMaxSizeLeavingSource(long bytes) { + this.maxSizeLeavingSource = bytes; + return this; + } + + public Builder setMaxSizeToMovePerIteration(long bytes) { + this.maxSizeToMovePerIteration = bytes; + return this; + } + + public Builder setMoveTimeoutMillis(long millis) { + this.moveTimeoutMillis = millis; + return this; + } + + public Builder setBalancingIntervalMillis(long millis) { + this.balancingIntervalMillis = millis; + return this; + } + + public ContainerBalancerEstimation build() { + return new ContainerBalancerEstimation(this); + } + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerProfile.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerProfile.java new file mode 100644 index 000000000000..f2368776d83b --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerProfile.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.balancer; + +/** + * Throttling profile for container balancer. + */ + +public enum ContainerBalancerProfile { + SLOW, + MEDIUM, + FAST; + + public int getDatanodesMaxPercentage(ContainerBalancerConfiguration conf) { + return conf.getProfileDatanodesMaxPercentage(this); + } + + public long getMaxSizeEnteringTarget(ContainerBalancerConfiguration conf) { + return conf.getProfileMaxSizeEnteringTarget(this); + } + + public long getMaxSizeLeavingSource(ContainerBalancerConfiguration conf) { + return conf.getProfileMaxSizeLeavingSource(this); + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java new file mode 100644 index 000000000000..8ec7390d97d8 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.balancer; + +import static org.apache.hadoop.ozone.ClientVersion.DEFAULT_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeUsageInfoProto; +import org.apache.hadoop.ozone.OzoneConsts; +import org.junit.jupiter.api.Test; + +/** Tests for {@link ContainerBalancerAdvisor} dry-run estimation. */ +public final class TestContainerBalancerAdvisor { + + @Test + void testComputePerIterationBytesLimitedByEnteringTarget() { + int[] involved = {7, 7}; + long expected = 26L * OzoneConsts.GB * 7; + + assertEquals(expected, ContainerBalancerAdvisor.computePerIterationBytes( + expected * 10, + 500L * OzoneConsts.GB, + 26 * OzoneConsts.GB, + 26 * OzoneConsts.GB, + involved)); + } + + @Test + void testComputePerIterationBytesNeverExceedsBytesToMove() { + int[] involved = {3, 3}; + long bytesToMove = 50L * OzoneConsts.GB; + + assertEquals(bytesToMove, ContainerBalancerAdvisor.computePerIterationBytes( + bytesToMove, + 500L * OzoneConsts.GB, + 26 * OzoneConsts.GB, + 26 * OzoneConsts.GB, + involved)); + } + + @Test + void testEstimateDryRunDefaultReturnsThreeProfiles() { + OzoneConfiguration conf = new OzoneConfiguration(); + List results = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(buildCluster(70, 14, 14))); + + assertEquals(3, results.size()); + assertEquals(ContainerBalancerProfile.SLOW, results.get(0).getProfile()); + assertEquals(ContainerBalancerProfile.MEDIUM, results.get(1).getProfile()); + assertEquals(ContainerBalancerProfile.FAST, results.get(2).getProfile()); + assertTrue(results.get(0).succeeded()); + assertTrue(results.get(1).succeeded()); + assertTrue(results.get(2).succeeded()); + + long bytesToMove = results.get(0).getBytesToMove(); + assertTrue(bytesToMove > 0); + for (ContainerBalancerEstimation result : results) { + assertEquals(bytesToMove, result.getBytesToMove()); + } + + // buildCluster(70, 14, 14): 42 sources, 14 targets; SLOW [4,3], MEDIUM [7,7], FAST [14,14] + assertEquals(30L * OzoneConsts.GB, results.get(0).getPerIterationBytes()); + assertEquals(26L * OzoneConsts.GB * 7, results.get(1).getPerIterationBytes()); + assertEquals(500L * OzoneConsts.GB, results.get(2).getPerIterationBytes()); + } + + @Test + void testEstimateDryRunSingleProfileMedium() { + OzoneConfiguration conf = new OzoneConfiguration(); + ContainerBalancerConfiguration balancerConfig = conf.getObject(ContainerBalancerConfiguration.class); + long expectedCycleTimeMillis = ContainerBalancerAdvisor.computeCycleTimeMillis( + balancerConfig.getMoveTimeout().toMillis(), + balancerConfig.getBalancingInterval().toMillis()); + + List results = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(buildCluster(70, 14, 14)) + .setProfile(ContainerBalancerProfile.MEDIUM)); + + assertEquals(1, results.size()); + ContainerBalancerEstimation estimation = results.get(0); + assertTrue(estimation.succeeded()); + assertEquals(ContainerBalancerProfile.MEDIUM, estimation.getProfile()); + + // maxInvolved=14 -> [7 sources, 7 targets]; MEDIUM cap=26GB/target -> 7*26GB + assertEquals(26L * OzoneConsts.GB * 7, estimation.getPerIterationBytes()); + assertEquals(expectedCycleTimeMillis, + estimation.getMoveTimeoutMillis() + estimation.getBalancingIntervalMillis()); + assertEquals( + (long) Math.ceil((double) estimation.getBytesToMove() / estimation.getPerIterationBytes()), + estimation.getEstimatedIterations()); + assertEquals( + estimation.getEstimatedIterations() * expectedCycleTimeMillis, + estimation.getEstimatedDurationMillis()); + assertEquals(20, estimation.getMaxDatanodesPercentage()); + } + + @Test + void testEstimateDryRunRespectsThresholdOverride() { + OzoneConfiguration conf = new OzoneConfiguration(); + List nodes = buildCluster(70, 14, 14); + + long defaultThresholdBytesToMove = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(nodes)) + .get(0) + .getBytesToMove(); + + long tighterThresholdBytesToMove = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(nodes) + .setThresholdPercent(5.0)) + .get(0) + .getBytesToMove(); + + assertTrue(tighterThresholdBytesToMove > defaultThresholdBytesToMove); + } + + @Test + void testEstimateDryRunRespectsExplicitMaxDatanodesPercentageOverride() { + OzoneConfiguration conf = new OzoneConfiguration(); + List results = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(buildCluster(70, 14, 14)) + .setProfile(ContainerBalancerProfile.SLOW) + .setMaxDatanodesPercentageToInvolvePerIteration(1)); + assertEquals(1, results.size()); + assertFalse(results.get(0).succeeded()); + assertTrue(results.get(0).getFailureMessage().contains("at least 2 are required")); + } + + @Test + void testEstimateDryRunRespectsMaxSizeLeavingSourceOverride() { + OzoneConfiguration conf = new OzoneConfiguration(); + List nodes = buildCluster(70, 14, 14); + + ContainerBalancerEstimation baseline = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(nodes) + .setProfile(ContainerBalancerProfile.FAST)) + .get(0); + + long overriddenLeavingSource = 10L * OzoneConsts.GB; + ContainerBalancerEstimation overridden = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(nodes) + .setProfile(ContainerBalancerProfile.FAST) + .setMaxSizeLeavingSource(overriddenLeavingSource)) + .get(0); + + assertEquals(baseline.getBytesToMove(), overridden.getBytesToMove()); + assertEquals(500L * OzoneConsts.GB, baseline.getPerIterationBytes()); + assertEquals(14L * overriddenLeavingSource, overridden.getPerIterationBytes()); + + assertTrue(overridden.getEstimatedIterations() > baseline.getEstimatedIterations()); + assertTrue(overridden.getEstimatedDurationMillis() > baseline.getEstimatedDurationMillis()); + } + + @Test + void testEstimateDryRunReturnsFailedResultWhenMaxMoveOverrideConflictsWithFastPreset() { + OzoneConfiguration conf = new OzoneConfiguration(); + List results = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(buildCluster(70, 14, 14)) + .setMaxSizeToMovePerIteration(70L * OzoneConsts.GB)); + + assertEquals(3, results.size()); + assertTrue(results.get(0).succeeded()); + assertTrue(results.get(1).succeeded()); + assertFalse(results.get(2).succeeded()); + assertEquals(ContainerBalancerProfile.FAST, results.get(2).getProfile()); + assertTrue(results.get(2).getFailureMessage().contains( + "max-size-entering-target must be less than or equal to max-size-to-move-per-iteration.")); + assertEquals(100L * OzoneConsts.GB, results.get(2).getMaxSizeEnteringTarget()); + assertEquals(70L * OzoneConsts.GB, results.get(2).getMaxSizeToMovePerIteration()); + } + + @Test + void testEstimateDryRunFailsWhenNodesNull() { + OzoneConfiguration conf = new OzoneConfiguration(); + assertThrows(NullPointerException.class, () -> + ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest())); + } + + @Test + void testEstimateDryRunFailsWhenNodesEmpty() { + OzoneConfiguration conf = new OzoneConfiguration(); + assertThrows(IllegalArgumentException.class, () -> + ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(new ArrayList<>()))); + } + + @Test + void testEstimateDryRunFailsWhenClusterBalanced() { + OzoneConfiguration conf = new OzoneConfiguration(); + List balanced = new ArrayList<>(); + balanced.add(proto("dn-1", OzoneConsts.TB, (long) (0.70 * OzoneConsts.TB))); + balanced.add(proto("dn-2", OzoneConsts.TB, (long) (0.70 * OzoneConsts.TB))); + + assertThrows(IllegalArgumentException.class, () -> + ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest().setNodes(balanced))); + } + + @Test + void testEstimateDryRunFailsWhenEnteringTargetTooSmall() { + OzoneConfiguration conf = new OzoneConfiguration(); + List results = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(buildCluster(70, 14, 14)) + .setProfile(ContainerBalancerProfile.SLOW) + .setMaxSizeEnteringTarget(OzoneConsts.GB)); + assertEquals(1, results.size()); + assertFalse(results.get(0).succeeded()); + } + + /** + * Builds an imbalanced cluster for dry-run tests. + * + *

With default 10% threshold and {@code buildCluster(70, 14, 14)}: + *

    + *
  • {@code under-*} — 5% used — 14 under-utilized targets
    • + *
  • {@code mid-*} — 40% used — 14 nodes near cluster average (neutral)
    • + *
  • {@code over-*} — 65% used — 42 over-utilized sources
    • + *
+ */ + private static List buildCluster( + int totalNodes, int underUtilNodeCount, int midUtilNodeCount) { + List nodes = new ArrayList<>(totalNodes); + long capacity = OzoneConsts.TB; + for (int i = 0; i < underUtilNodeCount; i++) { + nodes.add(proto("under-" + i, capacity, (long) (capacity * 0.95))); + } + for (int i = 0; i < midUtilNodeCount; i++) { + nodes.add(proto("mid-" + i, capacity, (long) (capacity * 0.60))); + } + int overUtil = totalNodes - underUtilNodeCount - midUtilNodeCount; + for (int i = 0; i < overUtil; i++) { + nodes.add(proto("over-" + i, capacity, (long) (capacity * 0.35))); + } + return nodes; + } + + private static DatanodeUsageInfoProto proto(String hostname, long capacity, long remaining) { + DatanodeDetails datanode = DatanodeDetails.newBuilder() + .setHostName(hostname) + .setIpAddress("127.0.0.1") + .setUuid(UUID.randomUUID()) + .build(); + return DatanodeUsageInfoProto.newBuilder() + .setNode(datanode.toProto(DEFAULT_VERSION.toProtoValue())) + .setCapacity(capacity) + .setRemaining(remaining) + .setUsed(capacity - remaining) + .build(); + } +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java index 951d447d0d5e..2e06d3af82b2 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java @@ -54,11 +54,36 @@ * involved in balancing * ozone admin containerbalancer start -s 10 * start balancer with maximum size of 10GB to move in one iteration + * To estimate (dry-run): + * ozone admin containerbalancer dry-run + * [ --profile {@literal } ] + * [ -t/--threshold {@literal } ] + * [ -d/--max-datanodes-percentage-to-involve-per-iteration {@literal } ] + * [ -s/--max-size-to-move-per-iteration-in-gb {@literal } ] + * [ -e/--max-size-entering-target-in-gb {@literal } ] + * [ -l/--max-size-leaving-source-in-gb {@literal } ] + * [ --balancing-iteration-interval-minutes {@literal } ] + * [ --move-timeout-minutes {@literal } ] + * [ --move-replication-timeout-minutes {@literal } ] + * [ --include-datanodes {@literal } ] + * [ --exclude-datanodes {@literal } ] + * Examples: + * ozone admin containerbalancer dry-run + * estimate bytes to move, number of iterations, per-iteration throughput, and duration for + * SLOW, MEDIUM, and FAST profiles (does not start the balancer) + * ozone admin containerbalancer dry-run --profile medium + * estimate for the MEDIUM profile only + * ozone admin containerbalancer dry-run --profile fast -t 5 + * estimate FAST profile with a 5% threshold * To stop: * ozone admin containerbalancer stop * * *

DESCRIPTION + *

Dry-run fetches datanode usage from SCM and estimates from + * local configurations and cluster analysis made. It does not start the balancer. Start does not yet + * support {@code --profile}, compare dry-run profiles to the config you plan + * to pass on start, or wait until profile support is added to start. dry-run produces upper-bound estimates. *

The threshold parameter is a fraction in the range of (1%, 100%) with a * default value of 10%. The threshold sets a target for whether the cluster * is balanced. A cluster is balanced if for each datanode, the utilization @@ -82,7 +107,8 @@ subcommands = { ContainerBalancerStartSubcommand.class, ContainerBalancerStopSubcommand.class, - ContainerBalancerStatusSubcommand.class + ContainerBalancerStatusSubcommand.class, + ContainerBalancerDryRunSubcommand.class }) @MetaInfServices(AdminSubcommand.class) public class ContainerBalancerCommands implements AdminSubcommand { diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java new file mode 100644 index 000000000000..0372319f5b3e --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerConfigOptions.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerAdvisor; +import org.apache.hadoop.ozone.OzoneConsts; +import picocli.CommandLine.Option; + +/** + * Shared Picocli options for container balancer commands. + */ +public class ContainerBalancerConfigOptions { + + @Option(names = {"-t", "--threshold"}, + description = "Percentage deviation from average utilization of " + + "the cluster after which a datanode will be rebalanced. The value " + + "should be in the range [0.0, 100.0), with a default of 10 " + + "(specify '10' for 10%%).") + private Optional threshold; + + @Option(names = {"-d", "--max-datanodes-percentage-to-involve-per-iteration"}, + description = "Max percentage of healthy, in service datanodes " + + "that can be involved in balancing in one iteration. The value " + + "should be in the range [0,100]. When omitted on dry-run, each profile uses its default preset. " + + "When omitted on start, the global config with a default of 20 (specify '20' for 20%%).") + private Optional maxDatanodesPercentageToInvolvePerIteration; + + @Option(names = {"-s", "--max-size-to-move-per-iteration-in-gb"}, + description = "Maximum size that can be moved per iteration of " + + "balancing. The value should be positive. When omitted, the " + + "global config default 500 (specify '500' for 500GB) is used on both dry-run and start.") + private Optional maxSizeToMovePerIterationInGB; + + @Option(names = {"-e", "--max-size-entering-target-in-gb"}, + description = "Maximum size that can enter a target datanode while " + + "balancing. This is the sum of data from multiple sources. The value " + + "should be positive. When omitted on dry-run, each profile uses its default preset. " + + "When omitted on start, the global config default 26 (specify '26' for 26GB) is used.") + private Optional maxSizeEnteringTargetInGB; + + @Option(names = {"-l", "--max-size-leaving-source-in-gb"}, + description = "Maximum size that can leave a source datanode while " + + "balancing. This is the sum of data moving to multiple targets. " + + "The value should be positive. When omitted on dry-run, each profile uses its default preset. " + + "When omitted on start, the global config default 26 (specify '26' for 26GB) is used.") + private Optional maxSizeLeavingSourceInGB; + + @Option(names = {"--balancing-iteration-interval-minutes"}, + description = "The interval period in minutes between each iteration of Container Balancer. " + + "The value should be positive, with a default of 70 (specify '70' for 70 minutes).") + private Optional balancingInterval; + + @Option(names = {"--move-timeout-minutes"}, + description = "The amount of time in minutes to allow a single container to move " + + "from source to target. The value should be positive, with a default of 65 " + + "(specify '65' for 65 minutes).") + private Optional moveTimeout; + + @Option(names = {"--move-replication-timeout-minutes"}, + description = "The " + + "amount of time in minutes to allow a single container's replication from source " + + "to target as part of container move. The value should be positive, with " + + "a default of 50. For example, if \"hdds.container" + + ".balancer.move.timeout\" is 65 minutes, then out of those 65 minutes " + + "50 minutes will be the deadline for replication to complete (specify " + + "'50' for 50 minutes).") + private Optional moveReplicationTimeout; + + @Option(names = {"--include-datanodes"}, + description = "A list of Datanode " + + "hostnames or ip addresses separated by commas. Only the Datanodes " + + "specified in this list are balanced. This configuration is empty by " + + "default and is applicable only if it is non-empty (specify \"hostname1,hostname2,hostname3\").") + private Optional includeNodes; + + @Option(names = {"--exclude-datanodes"}, + description = "A list of Datanode " + + "hostnames or ip addresses separated by commas. The Datanodes specified " + + "in this list are excluded from balancing. This configuration is empty " + + "by default (specify \"hostname1,hostname2,hostname3\").") + private Optional excludeNodes; + + public Optional getThreshold() { + return threshold; + } + + public Optional getMaxDatanodesPercentageToInvolvePerIteration() { + return maxDatanodesPercentageToInvolvePerIteration; + } + + public Optional getMaxSizeToMovePerIterationInGB() { + return maxSizeToMovePerIterationInGB; + } + + public Optional getMaxSizeEnteringTargetInGB() { + return maxSizeEnteringTargetInGB; + } + + public Optional getMaxSizeLeavingSourceInGB() { + return maxSizeLeavingSourceInGB; + } + + public Optional getBalancingIntervalMinutes() { + return balancingInterval; + } + + public Optional getMoveTimeoutMinutes() { + return moveTimeout; + } + + public Optional getMoveReplicationTimeoutMinutes() { + return moveReplicationTimeout; + } + + public Optional getIncludeNodes() { + return includeNodes; + } + + public Optional getExcludeNodes() { + return excludeNodes; + } + + /** Applies CLI overrides to a dry-run request. */ + public void applyToDryRunRequest(ContainerBalancerAdvisor.AdvisorRequest request) { + threshold.ifPresent(request::setThresholdPercent); + maxDatanodesPercentageToInvolvePerIteration.ifPresent( + request::setMaxDatanodesPercentageToInvolvePerIteration); + maxSizeToMovePerIterationInGB.ifPresent(gb -> + request.setMaxSizeToMovePerIteration(gb * OzoneConsts.GB)); + maxSizeEnteringTargetInGB.ifPresent(gb -> + request.setMaxSizeEnteringTarget(gb * OzoneConsts.GB)); + maxSizeLeavingSourceInGB.ifPresent(gb -> + request.setMaxSizeLeavingSource(gb * OzoneConsts.GB)); + balancingInterval.ifPresent(minutes -> + request.setBalancingIntervalMillis(Duration.ofMinutes(minutes).toMillis())); + moveTimeout.ifPresent(minutes -> + request.setMoveTimeoutMillis(Duration.ofMinutes(minutes).toMillis())); + moveReplicationTimeout.ifPresent(minutes -> + request.setMoveReplicationTimeoutMillis(Duration.ofMinutes(minutes).toMillis())); + includeNodes.ifPresent(value -> request.setIncludeNodes(parseNodeSet(value))); + excludeNodes.ifPresent(value -> request.setExcludeNodes(parseNodeSet(value))); + } + + private static Set parseNodeSet(String nodes) { + if (StringUtils.isBlank(nodes)) { + return Collections.emptySet(); + } + return Arrays.stream(nodes.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toSet()); + } +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java new file mode 100644 index 000000000000..a92677e16b17 --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli; + +import static org.apache.hadoop.util.StringUtils.byteDesc; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeUsageInfoProto; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerAdvisor; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerEstimation; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerProfile; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +/** + * Estimates container balancer bytes to move, per iteration bytes, estimated iterations, estimated duration + * without starting the balancer. + */ +@Command( + name = "dry-run", + description = "Estimate container balancer bytes to move, iterations, per iteration bytes " + + "and upper-bound duration without starting it. Limits and default profile presets are read from " + + "local ozone-site.xml, datanode usage is fetched from SCM.", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class ContainerBalancerDryRunSubcommand extends ScmSubcommand { + + private static final double PLANNING_ITERATION_BUFFER = 1.3d; + + @CommandLine.Mixin + private ContainerBalancerConfigOptions configOptions; + + @Option(names = {"--profile"}, + description = "Throttling profile: slow, medium, or fast. When set, only this profile is estimated. " + + "When omitted, dry-run estimates all three profiles. Start does not support --profile yet.") + private Optional profileName = Optional.empty(); + + @Override + public void execute(ScmClient scmClient) throws IOException { + + List nodes = scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE); + if (nodes == null || nodes.isEmpty()) { + throw new IOException("No datanode usage information available from SCM."); + } + + OzoneConfiguration conf = getOzoneConf(); + ContainerBalancerAdvisor.AdvisorRequest request = buildRequest(nodes); + List estimations; + try { + estimations = ContainerBalancerAdvisor.estimateDryRun(conf, request); + } catch (IllegalArgumentException e) { + throw new IOException(e.getMessage(), e); + } + + boolean anySucceeded = false; + for (ContainerBalancerEstimation result : estimations) { + out().printf("Profile: %s%n", result.getProfile().name()); + printBasedOn(result); + if (result.succeeded()) { + anySucceeded = true; + printEstimation(result); + } else { + out().printf(" Estimation failed: %s%n%n", result.getFailureMessage()); + } + } + if (!anySucceeded) { + throw new IOException(estimations.get(0).getFailureMessage()); + } + } + + private ContainerBalancerAdvisor.AdvisorRequest buildRequest(List nodes) throws IOException { + ContainerBalancerAdvisor.AdvisorRequest request = new ContainerBalancerAdvisor.AdvisorRequest().setNodes(nodes); + configOptions.applyToDryRunRequest(request); + + if (profileName.isPresent()) { + request.setProfile(parseProfile(profileName.get())); + } + return request; + } + + private static ContainerBalancerProfile parseProfile(String name) throws IOException { + try { + return ContainerBalancerProfile.valueOf(name.trim().toUpperCase(Locale.ENGLISH)); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid profile: " + name + ". Expected slow, medium, or fast."); + } + } + + private void printBasedOn(ContainerBalancerEstimation estimation) { + long moveTimeoutMinutes = Math.round(estimation.getMoveTimeoutMillis() / 60000d); + long balancingIntervalMinutes = Math.round(estimation.getBalancingIntervalMillis() / 60000d); + out().println(" Based on:"); + out().printf(Locale.ENGLISH, " Datanode involvement: %d%%%n", + estimation.getMaxDatanodesPercentage()); + out().printf(" Max entering target: %s / node%n", byteDesc(estimation.getMaxSizeEnteringTarget())); + out().printf(" Max leaving source: %s / node%n", byteDesc(estimation.getMaxSizeLeavingSource())); + out().printf(" Max per iteration: %s%n", byteDesc(estimation.getMaxSizeToMovePerIteration())); + out().printf(" Move timeout: %d min%n", moveTimeoutMinutes); + out().printf(" Balancing interval: %d min%n", balancingIntervalMinutes); + } + + private void printEstimation(ContainerBalancerEstimation estimation) { + long estimatedIterations = estimation.getEstimatedIterations(); + long planningIterations = (long) Math.ceil(estimatedIterations * PLANNING_ITERATION_BUFFER); + long cycleTimeMillis = estimation.getMoveTimeoutMillis() + estimation.getBalancingIntervalMillis(); + long baseDurationMillis = estimation.getEstimatedDurationMillis(); + long planningDurationMillis = planningIterations * cycleTimeMillis; + out().printf(" Bytes to move: %s%n", byteDesc(estimation.getBytesToMove())); + out().printf(" Per iteration (estimate): ~%s%n", byteDesc(estimation.getPerIterationBytes())); + out().printf(" Estimated iterations: %d (planning estimate: %d, includes +30%% buffer)%n", + estimatedIterations, planningIterations); + out().printf(" Estimated duration: upper bound %s (planning estimate: %s, includes +30%% buffer)%n", + formatEstimatedDuration(baseDurationMillis), + formatEstimatedDuration(planningDurationMillis)); + out().println(" (assumes full move timeout + interval each cycle)"); + out().println(); + } + + private static String formatEstimatedDuration(long durationMillis) { + double days = durationMillis / 86400000d; + if (days >= 1) { + return String.format(Locale.ENGLISH, "~%.1f days", days); + } + double hours = durationMillis / 3600000d; + if (hours >= 1) { + return String.format(Locale.ENGLISH, "~%.1f hours", hours); + } + long minutes = durationMillis / 60000; + return String.format(Locale.ENGLISH, "~%d min", minutes); + } +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java index 09de9e2b7580..4583a6d9be5b 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStartSubcommand.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto; import org.apache.hadoop.hdds.scm.client.ScmClient; +import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -35,12 +36,8 @@ versionProvider = HddsVersionProvider.class) public class ContainerBalancerStartSubcommand extends ScmSubcommand { - @Option(names = {"-t", "--threshold"}, - description = "Percentage deviation from average utilization of " + - "the cluster after which a datanode will be rebalanced. The value " + - "should be in the range [0.0, 100.0), with a default of 10 " + - "(specify '10' for 10%%).") - private Optional threshold; + @CommandLine.Mixin + private ContainerBalancerConfigOptions configOptions; @Option(names = {"-i", "--iterations"}, description = "Maximum consecutive iterations that " + @@ -48,73 +45,12 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { "or -1, with a default of 10 (specify '10' for 10 iterations).") private Optional iterations; - @Option(names = {"-d", "--max-datanodes-percentage-to-involve-per-iteration"}, - description = "Max percentage of healthy, in service datanodes " + - "that can be involved in balancing in one iteration. The value " + - "should be in the range [0,100], with a default of 20 (specify " + - "'20' for 20%%).") - private Optional maxDatanodesPercentageToInvolvePerIteration; - - @Option(names = {"-s", "--max-size-to-move-per-iteration-in-gb"}, - description = "Maximum size that can be moved per iteration of " + - "balancing. The value should be positive, with a default of 500 " + - "(specify '500' for 500GB).") - private Optional maxSizeToMovePerIterationInGB; - - @Option(names = {"-e", "--max-size-entering-target-in-gb"}, - description = "Maximum size that can enter a target datanode while " + - "balancing. This is the sum of data from multiple sources. The value " + - "should be positive, with a default of 26 (specify '26' for 26GB).") - private Optional maxSizeEnteringTargetInGB; - - @Option(names = {"-l", "--max-size-leaving-source-in-gb"}, - description = "Maximum size that can leave a source datanode while " + - "balancing. This is the sum of data moving to multiple targets. " + - "The value should be positive, with a default of 26 " + - "(specify '26' for 26GB).") - private Optional maxSizeLeavingSourceInGB; - - @Option(names = {"--balancing-iteration-interval-minutes"}, - description = "The interval period in minutes between each iteration of Container Balancer. " + - "The value should be positive, with a default of 70 (specify '70' for 70 minutes).") - private Optional balancingInterval; - - @Option(names = {"--move-timeout-minutes"}, - description = "The amount of time in minutes to allow a single container to move " + - "from source to target. The value should be positive, with a default of 65 " + - "(specify '65' for 65 minutes).") - private Optional moveTimeout; - - @Option(names = {"--move-replication-timeout-minutes"}, - description = "The " + - "amount of time in minutes to allow a single container's replication from source " + - "to target as part of container move. The value should be positive, with " + - "a default of 50. For example, if \"hdds.container" + - ".balancer.move.timeout\" is 65 minutes, then out of those 65 minutes " + - "50 minutes will be the deadline for replication to complete (specify " + - "'50' for 50 minutes).") - private Optional moveReplicationTimeout; - @Option(names = {"--move-network-topology-enable"}, description = "Whether to take network topology into account when " + "selecting a target for a source. " + "This configuration is false by default.") private Optional networkTopologyEnable; - @Option(names = {"--include-datanodes"}, - description = "A list of Datanode " + - "hostnames or ip addresses separated by commas. Only the Datanodes " + - "specified in this list are balanced. This configuration is empty by " + - "default and is applicable only if it is non-empty (specify \"hostname1,hostname2,hostname3\").") - private Optional includeNodes; - - @Option(names = {"--exclude-datanodes"}, - description = "A list of Datanode " + - "hostnames or ip addresses separated by commas. The Datanodes specified " + - "in this list are excluded from balancing. This configuration is empty " + - "by default (specify \"hostname1,hostname2,hostname3\").") - private Optional excludeNodes; - @Option(names = {"--exclude-containers"}, description = "A list of container IDs separated by commas. " + "The containers specified in this list are excluded from balancing. " + @@ -132,13 +68,21 @@ public class ContainerBalancerStartSubcommand extends ScmSubcommand { @Override public void execute(ScmClient scmClient) throws IOException { - StartContainerBalancerResponseProto response = scmClient. - startContainerBalancer(threshold, iterations, - maxDatanodesPercentageToInvolvePerIteration, - maxSizeToMovePerIterationInGB, maxSizeEnteringTargetInGB, - maxSizeLeavingSourceInGB, balancingInterval, moveTimeout, - moveReplicationTimeout, networkTopologyEnable, includeNodes, - excludeNodes, excludeContainers, includeContainers); + StartContainerBalancerResponseProto response = scmClient.startContainerBalancer( + configOptions.getThreshold(), + iterations, + configOptions.getMaxDatanodesPercentageToInvolvePerIteration(), + configOptions.getMaxSizeToMovePerIterationInGB(), + configOptions.getMaxSizeEnteringTargetInGB(), + configOptions.getMaxSizeLeavingSourceInGB(), + configOptions.getBalancingIntervalMinutes(), + configOptions.getMoveTimeoutMinutes(), + configOptions.getMoveReplicationTimeoutMinutes(), + networkTopologyEnable, + configOptions.getIncludeNodes(), + configOptions.getExcludeNodes(), + excludeContainers, + includeContainers); if (response.getStart()) { System.out.println("Container Balancer started successfully."); } else { diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java index 5eca86a8a6c6..c59557511047 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java @@ -17,27 +17,37 @@ package org.apache.hadoop.hdds.scm.cli.datanode; +import static org.apache.hadoop.hdds.DatanodeVersion.DEFAULT_VERSION; import static org.apache.hadoop.ozone.OzoneConsts.GB; +import static org.apache.hadoop.util.StringUtils.byteDesc; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoResponseProto; +import org.apache.hadoop.hdds.scm.cli.ContainerBalancerDryRunSubcommand; import org.apache.hadoop.hdds.scm.cli.ContainerBalancerStartSubcommand; import org.apache.hadoop.hdds.scm.cli.ContainerBalancerStatusSubcommand; import org.apache.hadoop.hdds.scm.cli.ContainerBalancerStopSubcommand; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -160,6 +170,7 @@ class TestContainerBalancerSubCommand { private ContainerBalancerStopSubcommand stopCmd; private ContainerBalancerStartSubcommand startCmd; private ContainerBalancerStatusSubcommand statusCmd; + private ContainerBalancerDryRunSubcommand dryRunCmd; private GenericTestUtils.PrintStreamCapturer out; private GenericTestUtils.PrintStreamCapturer err; private AtomicBoolean verbose; @@ -369,6 +380,8 @@ protected boolean isVerbose() { return verbose.get(); } }; + parseSubcommand(startCmd); + dryRunCmd = new ContainerBalancerDryRunSubcommand(); out = GenericTestUtils.captureOut(); err = GenericTestUtils.captureErr(); } @@ -574,7 +587,7 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsNotRunning() throws IOException { ScmClient scmClient = mock(ScmClient.class); when(scmClient.startContainerBalancer( - null, null, null, null, null, null, null, null, null, null, null, null, null, null)) + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any())) .thenReturn( StorageContainerLocationProtocolProtos .StartContainerBalancerResponseProto.newBuilder() @@ -590,7 +603,7 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsRunning() throws IOException { ScmClient scmClient = mock(ScmClient.class); when(scmClient.startContainerBalancer( - null, null, null, null, null, null, null, null, null, null, null, null, null, null)) + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any())) .thenReturn(StorageContainerLocationProtocolProtos .StartContainerBalancerResponseProto.newBuilder() .setStart(false) @@ -821,4 +834,212 @@ void testContainerBalancerStatusVerboseShowsNoBreakdownWhenFailuresMissing() thr .contains("Failed to move containers 3") .contains("Failed container moves (no breakdown available)"); } + + @Test + void testContainerBalancerDryRunSubcommandDefaultShowsAllProfiles() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(dryRunCmd); + dryRunCmd.execute(scmClient); + + String output = out.get(); + String[] blocks = output.split("Profile:"); + assertThat(blocks).hasSize(4); + + String slow = blocks[1]; + assertThat(slow) + .contains("Based on:") + .contains("Datanode involvement: 10%") + .contains("Bytes to move:") + .contains("Per iteration (estimate): ~" + byteDesc(30L * GB)) + .contains("planning estimate:") + .contains("upper bound") + .contains("assumes full move timeout + interval each cycle") + .doesNotContain("Estimation failed:"); + + String medium = blocks[2]; + assertThat(medium) + .contains("Based on:") + .contains("Datanode involvement: 20%") + .contains("Bytes to move:") + .contains("Per iteration (estimate): ~" + byteDesc(26L * GB * 7)) + .contains("planning estimate:") + .contains("upper bound") + .doesNotContain("Estimation failed:"); + + String fast = blocks[3]; + assertThat(fast) + .contains("Based on:") + .contains("Datanode involvement: 40%") + .contains("Bytes to move:") + .contains("Per iteration (estimate): ~" + byteDesc(500L * GB)) + .contains("planning estimate:") + .contains("upper bound") + .doesNotContain("Estimation failed:"); + } + + @Test + void testContainerBalancerDryRunSubcommandInvalidThresholdFails() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + parseSubcommand(dryRunCmd, "-t", "100"); + IOException ex = assertThrows(IOException.class, () -> dryRunCmd.execute(scmClient)); + assertThat(ex.getMessage()).contains("No over-utilized datanodes (sources) found."); + } + + @Test + void testContainerBalancerDryRunSubcommandInvalidDatanodePercentageFails() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + parseSubcommand(dryRunCmd, "-d", "0"); + IOException ex = assertThrows(IOException.class, () -> dryRunCmd.execute(scmClient)); + assertThat(ex.getMessage()).contains("at least 2 are required"); + } + + @Test + void testContainerBalancerDryRunSubcommandInvalidMaxSizeFails() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + parseSubcommand(dryRunCmd, "-s", "0"); + IOException ex = assertThrows(IOException.class, () -> dryRunCmd.execute(scmClient)); + assertThat(ex.getMessage()).contains( + "max-size-entering-target must be less than or equal to max-size-to-move-per-iteration."); + } + + @Test + void testContainerBalancerDryRunSubcommandInvalidProfileFails() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(dryRunCmd, "--profile", "turbo"); + IOException ex = assertThrows(IOException.class, () -> dryRunCmd.execute(scmClient)); + assertThat(ex.getMessage()).contains("Invalid profile: turbo"); + } + + @Test + void testContainerBalancerDryRunSubcommandWithProfileShowsOneProfile() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(dryRunCmd, "--profile", "medium"); + dryRunCmd.execute(scmClient); + + String output = out.get(); + assertThat(output).contains("Profile: MEDIUM"); + assertThat(output).doesNotContain("Profile: SLOW"); + assertThat(output).doesNotContain("Profile: FAST"); + assertThat(output.split("Profile:")).hasSize(2); + assertThat(output) + .contains("Datanode involvement: 20%") + .contains("Per iteration (estimate): ~" + byteDesc(26L * GB * 7)) + .contains("Estimated duration: upper bound"); + } + + @Test + void testContainerBalancerDryRunSubcommandCliOverrideUsesResolvedValuesInOutput() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(dryRunCmd, "--profile", "slow", "-e", "6"); + dryRunCmd.execute(scmClient); + + assertThat(out.get()) + .contains("Max entering target: " + byteDesc(6L * GB) + " / node") + .contains("Per iteration (estimate): ~" + byteDesc(18L * GB)); + } + + @Test + void testContainerBalancerDryRunSubcommandPartialFailureWhenMaxMoveOverrideConflictsWithFastPreset() + throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(dryRunCmd, "-s", "70", "-t", "5"); + dryRunCmd.execute(scmClient); + + String output = out.get(); + String[] blocks = output.split("Profile:"); + assertThat(blocks).hasSize(4); + + String slow = blocks[1]; + assertThat(slow) + .contains("Based on:") + .contains("Max per iteration: " + byteDesc(70L * GB)) + .contains("Bytes to move:") + .contains("Per iteration (estimate): ~" + byteDesc(30L * GB)) + .doesNotContain("Estimation failed:"); + + String medium = blocks[2]; + assertThat(medium) + .contains("Based on:") + .contains("Max per iteration: " + byteDesc(70L * GB)) + .contains("Bytes to move:") + .contains("Per iteration (estimate): ~" + byteDesc(70L * GB)) + .doesNotContain("Estimation failed:"); + + String fast = blocks[3]; + assertThat(fast) + .contains("Based on:") + .contains("Max entering target: " + byteDesc(100L * GB) + " / node") + .contains("Max per iteration: " + byteDesc(70L * GB)) + .contains("Estimation failed: max-size-entering-target must be less than or equal to " + + "max-size-to-move-per-iteration.") + .doesNotContain("Bytes to move:") + .doesNotContain("Per iteration (estimate):"); + } + + /** + * Imbalanced cluster for dry-run CLI tests. + * + *

With default 10% threshold: 14 targets, 42 sources, 14 neutral (70 eligible). + */ + private static List buildImbalancedCluster() { + return buildCluster(70, 14, 14); + } + + private static List buildCluster( + int totalNodes, int underUtilNodeCount, int midUtilNodeCount) { + List nodes = new ArrayList<>(totalNodes); + long capacity = OzoneConsts.TB; + for (int i = 0; i < underUtilNodeCount; i++) { + nodes.add(datanodeUsageProto("under-" + i, capacity, (long) (capacity * 0.95))); + } + for (int i = 0; i < midUtilNodeCount; i++) { + nodes.add(datanodeUsageProto("mid-" + i, capacity, (long) (capacity * 0.60))); + } + int overUtil = totalNodes - underUtilNodeCount - midUtilNodeCount; + for (int i = 0; i < overUtil; i++) { + nodes.add(datanodeUsageProto("over-" + i, capacity, (long) (capacity * 0.35))); + } + return nodes; + } + + private static HddsProtos.DatanodeUsageInfoProto datanodeUsageProto( + String hostname, long capacity, long remaining) { + DatanodeDetails datanode = DatanodeDetails.newBuilder() + .setHostName(hostname) + .setIpAddress("127.0.0.1") + .setUuid(UUID.randomUUID()) + .build(); + return HddsProtos.DatanodeUsageInfoProto.newBuilder() + .setNode(datanode.toProto(DEFAULT_VERSION.toProtoValue())) + .setCapacity(capacity) + .setRemaining(remaining) + .setUsed(capacity - remaining) + .build(); + } + + /** Picocli must parse args so @Mixin, @Option, and @Spec fields are injected. */ + private static void parseSubcommand(Object subcommand, String... args) { + new CommandLine(subcommand).parseArgs(args); + } } From 17a8f1e86a3991fd604af8de6104a783f4b62838 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati <115860222+sreejasahithi@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:14:56 +0530 Subject: [PATCH 2/3] Added more validations --- .../balancer/ContainerBalancerAdvisor.java | 44 +++++++++++++++++-- .../ContainerBalancerDryRunSubcommand.java | 4 +- .../TestContainerBalancerSubCommand.java | 9 ++-- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java index 629fed5dfe4b..800901f95bbd 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java @@ -57,9 +57,11 @@ public static List estimateDryRun(OzoneConfiguratio ContainerBalancerConfiguration balancerConfig = conf.getObject(ContainerBalancerConfiguration.class); - double thresholdRatio = request.thresholdPercent != null - ? request.thresholdPercent / 100.0 - : balancerConfig.getThresholdAsRatio(); + double thresholdPercent = request.thresholdPercent != null + ? request.thresholdPercent + : balancerConfig.getThreshold(); + validateThresholdPercent(thresholdPercent); + double thresholdRatio = thresholdPercent / 100.0; Set includeNodes = request.includeNodes != null ? request.includeNodes : balancerConfig.getIncludeNodes(); @@ -116,7 +118,9 @@ private static ContainerBalancerEstimation estimateForProfile(OzoneConfiguration .setBalancingIntervalMillis(balancingIntervalMillis); try { + validateMaxDatanodesPercentageToInvolvePerIteration(maxDatanodesPercentage); validateMoveTimeouts(conf, moveReplicationTimeoutMillis, moveTimeoutMillis); + validateBalancingIntervalMillis(balancingIntervalMillis); validateResolvedMoveLimits(conf, maxSizeEnteringTarget, maxSizeLeavingSource, maxSizeToMovePerIteration); int eligibleDatanodeCount = snapshot.getTotalEligibleDatanodes(); @@ -274,16 +278,25 @@ private static void validateResolvedMoveLimits(OzoneConfiguration conf, long max ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); + if (maxSizeEnteringTarget <= 0) { + throw new IllegalArgumentException("Max Size Entering Target must be greater than zero."); + } if (maxSizeEnteringTarget <= containerSizeBytes) { throw new IllegalArgumentException( "max-size-entering-target must be greater than ozone.scm.container.size (" + containerSizeBytes + " bytes)."); } + if (maxSizeLeavingSource <= 0) { + throw new IllegalArgumentException("Max Size Leaving Source must be greater than zero."); + } if (maxSizeLeavingSource <= containerSizeBytes) { throw new IllegalArgumentException( "max-size-leaving-source must be greater than ozone.scm.container.size (" + containerSizeBytes + " bytes)."); } + if (maxSizeToMovePerIteration <= 0) { + throw new IllegalArgumentException("Max Size To Move Per Iteration In GB must be positive."); + } if (maxSizeEnteringTarget > maxSizeToMovePerIteration) { throw new IllegalArgumentException( "max-size-entering-target must be less than or equal to " @@ -298,6 +311,12 @@ private static void validateResolvedMoveLimits(OzoneConfiguration conf, long max private static void validateMoveTimeouts(OzoneConfiguration conf, long moveReplicationTimeoutMillis, long moveTimeoutMillis) { + if (moveTimeoutMillis <= 0) { + throw new IllegalArgumentException("Move Timeout must be greater than zero."); + } + if (moveReplicationTimeoutMillis <= 0) { + throw new IllegalArgumentException("Move Replication Timeout must be greater than zero."); + } if (moveReplicationTimeoutMillis >= moveTimeoutMillis) { throw new IllegalArgumentException("hdds.container.balancer.move.replication.timeout should " + "be less than hdds.container.balancer.move.timeout."); @@ -317,6 +336,25 @@ private static void validateMoveTimeouts(OzoneConfiguration conf, long moveRepli } } + private static void validateThresholdPercent(double thresholdPercent) { + if (thresholdPercent < 0d || thresholdPercent >= 100d) { + throw new IllegalArgumentException("Threshold should be specified in the range [0.0, 100.0)."); + } + } + + private static void validateMaxDatanodesPercentageToInvolvePerIteration(int percentage) { + if (percentage <= 0 || percentage > 100) { + throw new IllegalArgumentException("Max Datanodes Percentage To Involve Per Iteration " + + "should be specified in the range (0, 100]"); + } + } + + private static void validateBalancingIntervalMillis(long balancingIntervalMillis) { + if (balancingIntervalMillis <= 0) { + throw new IllegalArgumentException("Balancing Interval must be greater than zero."); + } + } + private static List selectProfiles(AdvisorRequest request) { if (request.profile != null) { return Collections.singletonList(request.profile); diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java index a92677e16b17..483f72b316db 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java @@ -41,8 +41,8 @@ @Command( name = "dry-run", description = "Estimate container balancer bytes to move, iterations, per iteration bytes " + - "and upper-bound duration without starting it. Limits and default profile presets are read from " + - "local ozone-site.xml, datanode usage is fetched from SCM.", + "and upper-bound duration without starting it. Balancer limits and profile presets are read from " + + "the local Ozone configuration (including ozone-site.xml), datanode usage is fetched from SCM.", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) public class ContainerBalancerDryRunSubcommand extends ScmSubcommand { diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java index c59557511047..f11c4e0f09d1 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java @@ -885,9 +885,9 @@ void testContainerBalancerDryRunSubcommandInvalidThresholdFails() throws IOExcep ScmClient scmClient = mock(ScmClient.class); when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) .thenReturn(buildImbalancedCluster()); - parseSubcommand(dryRunCmd, "-t", "100"); + parseSubcommand(dryRunCmd, "-t", "-1"); IOException ex = assertThrows(IOException.class, () -> dryRunCmd.execute(scmClient)); - assertThat(ex.getMessage()).contains("No over-utilized datanodes (sources) found."); + assertThat(ex.getMessage()).contains("Threshold should be specified in the range [0.0, 100.0)."); } @Test @@ -897,7 +897,8 @@ void testContainerBalancerDryRunSubcommandInvalidDatanodePercentageFails() throw .thenReturn(buildImbalancedCluster()); parseSubcommand(dryRunCmd, "-d", "0"); IOException ex = assertThrows(IOException.class, () -> dryRunCmd.execute(scmClient)); - assertThat(ex.getMessage()).contains("at least 2 are required"); + assertThat(ex.getMessage()).contains( + "Max Datanodes Percentage To Involve Per Iteration should be specified in the range (0, 100]"); } @Test @@ -908,7 +909,7 @@ void testContainerBalancerDryRunSubcommandInvalidMaxSizeFails() throws IOExcepti parseSubcommand(dryRunCmd, "-s", "0"); IOException ex = assertThrows(IOException.class, () -> dryRunCmd.execute(scmClient)); assertThat(ex.getMessage()).contains( - "max-size-entering-target must be less than or equal to max-size-to-move-per-iteration."); + "Max Size To Move Per Iteration In GB must be positive."); } @Test From ec3bcaaa80397459fde43233d763ef6b4a216189 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati <115860222+sreejasahithi@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:48:47 +0530 Subject: [PATCH 3/3] Updated dry-run to have --all option and by default show medium profile estimation --- .../balancer/ContainerBalancerAdvisor.java | 20 ++++++-- .../TestContainerBalancerAdvisor.java | 47 ++++++++++--------- .../scm/cli/ContainerBalancerCommands.java | 9 ++-- .../ContainerBalancerDryRunSubcommand.java | 27 ++++++++--- .../TestContainerBalancerSubCommand.java | 39 ++++++++++++--- 5 files changed, 99 insertions(+), 43 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java index 800901f95bbd..51294500b417 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerAdvisor.java @@ -45,8 +45,9 @@ private ContainerBalancerAdvisor() { /** * Estimates per-iteration size, iterations and duration for one or more balancer profiles. * - * If profile is unset, returns SLOW, MEDIUM, and FAST. + * If {@link AdvisorRequest#allProfiles} is true, returns SLOW, MEDIUM, and FAST. * If profile is set, returns a result for that profile only. + * Otherwise returns MEDIUM only. * Per-profile validation failures are returned with {@link ContainerBalancerEstimation#succeeded()} false * instead of aborting other profiles. */ @@ -356,13 +357,16 @@ private static void validateBalancingIntervalMillis(long balancingIntervalMillis } private static List selectProfiles(AdvisorRequest request) { + if (request.allProfiles) { + return Arrays.asList( + ContainerBalancerProfile.SLOW, + ContainerBalancerProfile.MEDIUM, + ContainerBalancerProfile.FAST); + } if (request.profile != null) { return Collections.singletonList(request.profile); } - return Arrays.asList( - ContainerBalancerProfile.SLOW, - ContainerBalancerProfile.MEDIUM, - ContainerBalancerProfile.FAST); + return Collections.singletonList(ContainerBalancerProfile.MEDIUM); } /** @@ -375,6 +379,7 @@ public static final class AdvisorRequest { private Set excludeNodes; private Double thresholdPercent; private ContainerBalancerProfile profile; + private boolean allProfiles; private Integer maxDatanodesPercentageToInvolvePerIteration; private Long maxSizeToMovePerIteration; private Long maxSizeEnteringTarget; @@ -408,6 +413,11 @@ public AdvisorRequest setProfile(ContainerBalancerProfile profileValue) { return this; } + public AdvisorRequest setAllProfiles(boolean allProfilesValue) { + this.allProfiles = allProfilesValue; + return this; + } + public AdvisorRequest setMaxDatanodesPercentageToInvolvePerIteration(Integer percentage) { this.maxDatanodesPercentageToInvolvePerIteration = percentage; return this; diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java index 8ec7390d97d8..0c4703bf17a3 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerAdvisor.java @@ -35,19 +35,6 @@ /** Tests for {@link ContainerBalancerAdvisor} dry-run estimation. */ public final class TestContainerBalancerAdvisor { - @Test - void testComputePerIterationBytesLimitedByEnteringTarget() { - int[] involved = {7, 7}; - long expected = 26L * OzoneConsts.GB * 7; - - assertEquals(expected, ContainerBalancerAdvisor.computePerIterationBytes( - expected * 10, - 500L * OzoneConsts.GB, - 26 * OzoneConsts.GB, - 26 * OzoneConsts.GB, - involved)); - } - @Test void testComputePerIterationBytesNeverExceedsBytesToMove() { int[] involved = {3, 3}; @@ -62,12 +49,29 @@ void testComputePerIterationBytesNeverExceedsBytesToMove() { } @Test - void testEstimateDryRunDefaultReturnsThreeProfiles() { + void testEstimateDryRunDefaultReturnsMediumProfile() { OzoneConfiguration conf = new OzoneConfiguration(); List results = ContainerBalancerAdvisor.estimateDryRun( conf, new ContainerBalancerAdvisor.AdvisorRequest().setNodes(buildCluster(70, 14, 14))); + assertEquals(1, results.size()); + ContainerBalancerEstimation estimation = results.get(0); + assertTrue(estimation.succeeded()); + assertEquals(ContainerBalancerProfile.MEDIUM, estimation.getProfile()); + assertTrue(estimation.getBytesToMove() > 0); + assertEquals(26L * OzoneConsts.GB * 7, estimation.getPerIterationBytes()); + } + + @Test + void testEstimateDryRunAllProfilesReturnsThreeProfiles() { + OzoneConfiguration conf = new OzoneConfiguration(); + List results = ContainerBalancerAdvisor.estimateDryRun( + conf, + new ContainerBalancerAdvisor.AdvisorRequest() + .setNodes(buildCluster(70, 14, 14)) + .setAllProfiles(true)); + assertEquals(3, results.size()); assertEquals(ContainerBalancerProfile.SLOW, results.get(0).getProfile()); assertEquals(ContainerBalancerProfile.MEDIUM, results.get(1).getProfile()); @@ -89,7 +93,7 @@ void testEstimateDryRunDefaultReturnsThreeProfiles() { } @Test - void testEstimateDryRunSingleProfileMedium() { + void testEstimateDryRunSingleProfileFast() { OzoneConfiguration conf = new OzoneConfiguration(); ContainerBalancerConfiguration balancerConfig = conf.getObject(ContainerBalancerConfiguration.class); long expectedCycleTimeMillis = ContainerBalancerAdvisor.computeCycleTimeMillis( @@ -100,15 +104,15 @@ void testEstimateDryRunSingleProfileMedium() { conf, new ContainerBalancerAdvisor.AdvisorRequest() .setNodes(buildCluster(70, 14, 14)) - .setProfile(ContainerBalancerProfile.MEDIUM)); + .setProfile(ContainerBalancerProfile.FAST)); assertEquals(1, results.size()); ContainerBalancerEstimation estimation = results.get(0); assertTrue(estimation.succeeded()); - assertEquals(ContainerBalancerProfile.MEDIUM, estimation.getProfile()); - - // maxInvolved=14 -> [7 sources, 7 targets]; MEDIUM cap=26GB/target -> 7*26GB - assertEquals(26L * OzoneConsts.GB * 7, estimation.getPerIterationBytes()); + assertEquals(ContainerBalancerProfile.FAST, estimation.getProfile()); + // 40% of 70 -> maxInvolved=28 -> [14 sources, 14 targets]; + // 14*100GB exceeds the 500GB iteration cap, so the cap binds. + assertEquals(500L * OzoneConsts.GB, estimation.getPerIterationBytes()); assertEquals(expectedCycleTimeMillis, estimation.getMoveTimeoutMillis() + estimation.getBalancingIntervalMillis()); assertEquals( @@ -117,7 +121,7 @@ void testEstimateDryRunSingleProfileMedium() { assertEquals( estimation.getEstimatedIterations() * expectedCycleTimeMillis, estimation.getEstimatedDurationMillis()); - assertEquals(20, estimation.getMaxDatanodesPercentage()); + assertEquals(40, estimation.getMaxDatanodesPercentage()); } @Test @@ -192,6 +196,7 @@ void testEstimateDryRunReturnsFailedResultWhenMaxMoveOverrideConflictsWithFastPr conf, new ContainerBalancerAdvisor.AdvisorRequest() .setNodes(buildCluster(70, 14, 14)) + .setAllProfiles(true) .setMaxSizeToMovePerIteration(70L * OzoneConsts.GB)); assertEquals(3, results.size()); diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java index 2e06d3af82b2..7a2520f45fd8 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java @@ -57,6 +57,7 @@ * To estimate (dry-run): * ozone admin containerbalancer dry-run * [ --profile {@literal } ] + * [ --all ] * [ -t/--threshold {@literal } ] * [ -d/--max-datanodes-percentage-to-involve-per-iteration {@literal } ] * [ -s/--max-size-to-move-per-iteration-in-gb {@literal } ] @@ -70,9 +71,11 @@ * Examples: * ozone admin containerbalancer dry-run * estimate bytes to move, number of iterations, per-iteration throughput, and duration for - * SLOW, MEDIUM, and FAST profiles (does not start the balancer) - * ozone admin containerbalancer dry-run --profile medium - * estimate for the MEDIUM profile only + * the MEDIUM profile (does not start the balancer) + * ozone admin containerbalancer dry-run --all + * estimate for SLOW, MEDIUM, and FAST profiles + * ozone admin containerbalancer dry-run --profile slow + * estimate for the SLOW profile only * ozone admin containerbalancer dry-run --profile fast -t 5 * estimate FAST profile with a 5% threshold * To stop: diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java index 483f72b316db..f7be07c7587d 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerDryRunSubcommand.java @@ -52,10 +52,8 @@ public class ContainerBalancerDryRunSubcommand extends ScmSubcommand { @CommandLine.Mixin private ContainerBalancerConfigOptions configOptions; - @Option(names = {"--profile"}, - description = "Throttling profile: slow, medium, or fast. When set, only this profile is estimated. " - + "When omitted, dry-run estimates all three profiles. Start does not support --profile yet.") - private Optional profileName = Optional.empty(); + @CommandLine.ArgGroup(exclusive = true, multiplicity = "0..1") + private ProfileSelection profileSelection; @Override public void execute(ScmClient scmClient) throws IOException { @@ -94,8 +92,12 @@ private ContainerBalancerAdvisor.AdvisorRequest buildRequest(List profileName; + + @Option(names = {"--all"}, + description = "Estimate SLOW, MEDIUM, and FAST profiles.") + private boolean allProfiles; + } } diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java index f11c4e0f09d1..56d46f92e62f 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java @@ -836,7 +836,7 @@ void testContainerBalancerStatusVerboseShowsNoBreakdownWhenFailuresMissing() thr } @Test - void testContainerBalancerDryRunSubcommandDefaultShowsAllProfiles() throws IOException { + void testContainerBalancerDryRunSubcommandDefaultShowsMediumProfile() throws IOException { ScmClient scmClient = mock(ScmClient.class); when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) .thenReturn(buildImbalancedCluster()); @@ -844,6 +844,31 @@ void testContainerBalancerDryRunSubcommandDefaultShowsAllProfiles() throws IOExc parseSubcommand(dryRunCmd); dryRunCmd.execute(scmClient); + String output = out.get(); + assertThat(output).contains("Profile: MEDIUM"); + assertThat(output).doesNotContain("Profile: SLOW"); + assertThat(output).doesNotContain("Profile: FAST"); + assertThat(output.split("Profile:")).hasSize(2); + assertThat(output) + .contains("Based on:") + .contains("Datanode involvement: 20%") + .contains("Bytes to move:") + .contains("Per iteration (estimate): ~" + byteDesc(26L * GB * 7)) + .contains("planning estimate:") + .contains("upper bound") + .contains("assumes full move timeout + interval each cycle") + .doesNotContain("Estimation failed:"); + } + + @Test + void testContainerBalancerDryRunSubcommandAllShowsAllProfiles() throws IOException { + ScmClient scmClient = mock(ScmClient.class); + when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) + .thenReturn(buildImbalancedCluster()); + + parseSubcommand(dryRunCmd, "--all"); + dryRunCmd.execute(scmClient); + String output = out.get(); String[] blocks = output.split("Profile:"); assertThat(blocks).hasSize(4); @@ -929,17 +954,17 @@ void testContainerBalancerDryRunSubcommandWithProfileShowsOneProfile() throws IO when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) .thenReturn(buildImbalancedCluster()); - parseSubcommand(dryRunCmd, "--profile", "medium"); + parseSubcommand(dryRunCmd, "--profile", "FAST"); dryRunCmd.execute(scmClient); String output = out.get(); - assertThat(output).contains("Profile: MEDIUM"); + assertThat(output).contains("Profile: FAST"); assertThat(output).doesNotContain("Profile: SLOW"); - assertThat(output).doesNotContain("Profile: FAST"); + assertThat(output).doesNotContain("Profile: MEDIUM"); assertThat(output.split("Profile:")).hasSize(2); assertThat(output) - .contains("Datanode involvement: 20%") - .contains("Per iteration (estimate): ~" + byteDesc(26L * GB * 7)) + .contains("Datanode involvement: 40%") + .contains("Per iteration (estimate): ~" + byteDesc(500L * GB)) .contains("Estimated duration: upper bound"); } @@ -964,7 +989,7 @@ void testContainerBalancerDryRunSubcommandPartialFailureWhenMaxMoveOverrideConfl when(scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE)) .thenReturn(buildImbalancedCluster()); - parseSubcommand(dryRunCmd, "-s", "70", "-t", "5"); + parseSubcommand(dryRunCmd, "--all", "-s", "70", "-t", "5"); dryRunCmd.execute(scmClient); String output = out.get();