From 116bb263624439b8afbf861d7b58f356465808eb Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Fri, 18 Sep 2026 10:04:24 +0530 Subject: [PATCH 1/3] HDDS-16174. Add container balancer assessment command to report cluster imbalance --- .../ContainerBalancerClusterAnalyzer.java | 29 ++- ...ContainerBalancerAssessmentSubcommand.java | 240 ++++++++++++++++++ .../scm/cli/ContainerBalancerCommands.java | 8 +- ...ContainerBalancerAssessmentSubcommand.java | 177 +++++++++++++ 4 files changed, 449 insertions(+), 5 deletions(-) create mode 100644 hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java create mode 100644 hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerAssessmentSubcommand.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerClusterAnalyzer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerClusterAnalyzer.java index 56a5675b18d1..683a2eb44bf9 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerClusterAnalyzer.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerClusterAnalyzer.java @@ -120,6 +120,27 @@ public static ContainerBalancerClusterSnapshot analyze( double thresholdRatio, Set includeNodes, Set excludeNodes) { + return analyze(nodes, thresholdRatio, includeNodes, excludeNodes, TOP_NODE_LIMIT); + } + + /** + * Builds a cluster snapshot after applying include/exclude filters. + * + * @param nodes all nodes from getDatanodeUsageInfo (typically healthy IN_SERVICE) + * @param thresholdRatio threshold as ratio, e.g. 0.10 for 10% + * @param includeNodes empty = all included; non-empty = allow-list + * @param excludeNodes nodes to skip + * @param topNodeLimit max source/target nodes to include in the snapshot lists + */ + public static ContainerBalancerClusterSnapshot analyze( + List nodes, + double thresholdRatio, + Set includeNodes, + Set excludeNodes, + int topNodeLimit) { + if (topNodeLimit < 1) { + throw new IllegalArgumentException("topNodeLimit must be at least 1."); + } List eligible = filterEligibleNodes(nodes, includeNodes, excludeNodes); if (eligible.isEmpty()) { @@ -174,8 +195,8 @@ public static ContainerBalancerClusterSnapshot analyze( .setTotalOverUtilizedBytes(totalOverUtilizedBytes) .setTotalUnderUtilizedBytes(totalUnderUtilizedBytes) .setImbalance(imbalance) - .setTopSourceNodes(topNodes(sources)) - .setBottomTargetNodes(topNodes(targets)) + .setTopSourceNodes(topNodes(sources, topNodeLimit)) + .setBottomTargetNodes(topNodes(targets, topNodeLimit)) .build(); } @@ -242,8 +263,8 @@ private static String getDisplayHostname(DatanodeUsageInfoProto node) { return datanode.getIpAddress(); } - private static List topNodes(List nodes) { - int limit = Math.min(TOP_NODE_LIMIT, nodes.size()); + private static List topNodes(List nodes, int topNodeLimit) { + int limit = Math.min(topNodeLimit, nodes.size()); List result = new ArrayList<>(limit); for (int i = 0; i < limit; i++) { result.add(nodes.get(i)); diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java new file mode 100644 index 000000000000..3481c8ea1f0b --- /dev/null +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli; + +import static org.apache.hadoop.util.StringUtils.byteDesc; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeUsageInfoProto; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerClusterAnalyzer; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerClusterSnapshot; +import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +/** Reports cluster imbalance without starting the container balancer. */ +@Command( + name = "assessment", + description = "Report cluster imbalance before starting ContainerBalancer", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class ContainerBalancerAssessmentSubcommand extends ScmSubcommand { + + @Option(names = {"-t", "--threshold"}, + description = "Percentage deviation from average utilization of " + + "the cluster after which a datanode is considered over- or under-utilized. " + + "The value should be in the range [0.0, 100.0), with a default of 10 " + + "(specify '10' for 10%%).") + private Optional threshold; + + @Option(names = {"-n", "--limit"}, + description = "Maximum number of over- and under-utilized datanodes to list in the report. Default: 5.") + private int nodeLimit = 5; + + @Option(names = {"--include-datanodes"}, + description = "A list of Datanode hostnames or ip addresses separated by commas. " + + "Only the Datanodes specified in this list are included in the assessment.") + private Optional includeNodes; + + @Option(names = {"--exclude-datanodes"}, + description = "A list of Datanode hostnames or ip addresses separated by commas. " + + "The Datanodes specified in this list are excluded from the assessment.") + private Optional excludeNodes; + + @Override + public void execute(ScmClient scmClient) throws IOException { + ContainerBalancerConfiguration config = + getOzoneConf().getObject(ContainerBalancerConfiguration.class); + + if (nodeLimit < 1) { + throw new IllegalArgumentException("limit must be at least 1."); + } + + double thresholdRatio = threshold + .map(t -> { + if (t < 0d || t >= 100d) { + throw new IllegalArgumentException( + "Threshold must be a percentage in the range [0.0, 100.0)."); + } + return t / 100.0; + }) + .orElseGet(config::getThresholdAsRatio); + + Set include = includeNodes + .map(ContainerBalancerAssessmentSubcommand::parseNodeList) + .orElseGet(config::getIncludeNodes); + Set exclude = excludeNodes + .map(ContainerBalancerAssessmentSubcommand::parseNodeList) + .orElseGet(config::getExcludeNodes); + + List nodes = + scmClient.getDatanodeUsageInfo(true, Integer.MAX_VALUE); + + ContainerBalancerClusterSnapshot snapshot = + ContainerBalancerClusterAnalyzer.analyze(nodes, thresholdRatio, include, exclude, nodeLimit); + + printReport(snapshot, nodeLimit); + } + + private static void printReport(ContainerBalancerClusterSnapshot snapshot, int nodeLimit) { + System.out.println("CLUSTER BALANCE ASSESSMENT"); + + if (snapshot.getTotalEligibleDatanodes() == 0) { + System.out.println(getSummaryPrettyString(snapshot)); + System.out.println(); + System.out.println("No eligible datanodes found for assessment."); + return; + } + + System.out.println(getSummaryPrettyString(snapshot)); + System.out.println(); + System.out.println(getSourceNodesPrettyString(snapshot, nodeLimit)); + System.out.println(); + System.out.println(getTargetNodesPrettyString(snapshot, nodeLimit)); + System.out.println(); + System.out.println(getMovementSummaryPrettyString(snapshot)); + } + + private static String getSummaryPrettyString(ContainerBalancerClusterSnapshot snapshot) { + if (snapshot.getTotalEligibleDatanodes() == 0) { + return String.format("%-50s %s%n" + + "%-50s %s%n", "Key", "Value", + "Eligible datanodes", "0 datanodes"); + } + return String.format("%-50s %s%n" + + "%-50s %s%n" + + "%-50s %s%n" + + "%-50s %s%n" + + "%-50s %s%n", "Key", "Value", + "Drift", String.format(Locale.US, + "%.1f%% (max utilization %.1f%% - min utilization %.1f%%)", + snapshot.getImbalance() * 100, + snapshot.getMaxUtilization() * 100, + snapshot.getMinUtilization() * 100), + "Mean Utilization", formatPercent(snapshot.getClusterAvgUtilization()), + "Eligible datanodes", snapshot.getTotalEligibleDatanodes() + " datanode" + + (snapshot.getTotalEligibleDatanodes() == 1 ? "" : "s"), + "Category", formatCategory(snapshot)); + } + + private static String getSourceNodesPrettyString(ContainerBalancerClusterSnapshot snapshot, int nodeLimit) { + StringBuilder builder = new StringBuilder(); + builder.append("Source Nodes (over-utilized):").append(System.lineSeparator()); + builder.append(String.format("%-50s %s%n", "Datanodes above threshold", snapshot.getSourceCount())); + appendNodeList(builder, snapshot.getTopSourceNodes(), snapshot.getClusterAvgUtilization(), true, nodeLimit); + return builder.toString(); + } + + private static String getTargetNodesPrettyString(ContainerBalancerClusterSnapshot snapshot, int nodeLimit) { + StringBuilder builder = new StringBuilder(); + builder.append("Target Nodes (under-utilized):").append(System.lineSeparator()); + builder.append(String.format("%-50s %s%n", "Datanodes below threshold", snapshot.getTargetCount())); + appendNodeList(builder, snapshot.getBottomTargetNodes(), snapshot.getClusterAvgUtilization(), false, nodeLimit); + return builder.toString(); + } + + private static String getMovementSummaryPrettyString(ContainerBalancerClusterSnapshot snapshot) { + double movementRatio = snapshot.getClusterCapacityBytes() == 0 ? 0 + : (double) snapshot.getBytesToMove() / snapshot.getClusterCapacityBytes(); + return String.format("Movement Summary:%n" + + "%-50s %s%n" + + "%-50s %s%n", "Total bytes to move", byteDesc(snapshot.getBytesToMove()), + "Movement ratio", + String.format(Locale.US, "%.1f%% of cluster capacity", movementRatio * 100)); + } + + private static void appendNodeList(StringBuilder builder, + List nodes, + double clusterAvgUtilization, boolean aboveMean, int nodeLimit) { + if (nodes.isEmpty()) { + return; + } + builder.append(System.lineSeparator()); + builder.append(aboveMean ? "Top " + nodeLimit + ":" : "Bottom " + nodeLimit + ":") + .append(System.lineSeparator()); + for (ContainerBalancerClusterSnapshot.NodeUtilization node : nodes) { + double deltaFromMean = (node.getUtilization() - clusterAvgUtilization) * 100; + builder.append(String.format(Locale.US, "%-50s %s%n", node.getHostname(), + String.format(Locale.US, "%.1f%% (%+.1f%% %s mean)", + node.getUtilization() * 100, deltaFromMean, aboveMean ? "above" : "below"))); + } + } + + private static String formatPercent(double ratio) { + return String.format(Locale.US, "%.1f%%", ratio * 100); + } + + private static String formatCategory(ContainerBalancerClusterSnapshot snapshot) { + return formatClusterSize(snapshot.getTotalEligibleDatanodes()) + ", " + + formatImbalance(snapshot.getImbalance()) + ", " + + formatMovementRatio(snapshot); + } + + private static String formatClusterSize(int eligibleDatanodes) { + if (eligibleDatanodes >= 100) { + return "Large cluster"; + } + if (eligibleDatanodes >= 20) { + return "Medium cluster"; + } + return "Small cluster"; + } + + private static String formatImbalance(double imbalance) { + if (imbalance >= 0.20) { + return "High imbalance"; + } + if (imbalance >= 0.10) { + return "Medium imbalance"; + } + return "Low imbalance"; + } + + private static String formatMovementRatio(ContainerBalancerClusterSnapshot snapshot) { + if (snapshot.getClusterCapacityBytes() == 0) { + return "Low movement ratio"; + } + double ratio = (double) snapshot.getBytesToMove() / snapshot.getClusterCapacityBytes(); + if (ratio >= 0.05) { + return "High movement ratio"; + } + if (ratio >= 0.01) { + return "Medium movement ratio"; + } + return "Low movement ratio"; + } + + private static Set parseNodeList(String commaSeparated) { + if (commaSeparated == null || commaSeparated.trim().isEmpty()) { + return Collections.emptySet(); + } + return Arrays.stream(commaSeparated.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toSet()); + } +} diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java index 951d447d0d5e..d386058183a5 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerCommands.java @@ -56,6 +56,11 @@ * start balancer with maximum size of 10GB to move in one iteration * To stop: * ozone admin containerbalancer stop + * To assess: + * ozone admin containerbalancer assessment + * [ -t/--threshold {@literal }] + * [ --include-datanodes {@literal }] + * [ --exclude-datanodes {@literal }] * * *

DESCRIPTION @@ -82,7 +87,8 @@ subcommands = { ContainerBalancerStartSubcommand.class, ContainerBalancerStopSubcommand.class, - ContainerBalancerStatusSubcommand.class + ContainerBalancerStatusSubcommand.class, + ContainerBalancerAssessmentSubcommand.class }) @MetaInfServices(AdminSubcommand.class) public class ContainerBalancerCommands implements AdminSubcommand { diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerAssessmentSubcommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerAssessmentSubcommand.java new file mode 100644 index 000000000000..347dee2565c3 --- /dev/null +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerAssessmentSubcommand.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.cli.datanode; + +import static org.apache.hadoop.ozone.ClientVersion.DEFAULT_VERSION; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.anyBoolean; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeUsageInfoProto; +import org.apache.hadoop.hdds.scm.cli.ContainerBalancerAssessmentSubcommand; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.ozone.test.GenericTestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import picocli.CommandLine; + +/** Tests for {@link ContainerBalancerAssessmentSubcommand}. */ +class TestContainerBalancerAssessmentSubcommand { + + private ContainerBalancerAssessmentSubcommand cmd; + private GenericTestUtils.PrintStreamCapturer out; + + @BeforeEach + void setup() { + cmd = new ContainerBalancerAssessmentSubcommand(); + out = GenericTestUtils.captureOut(); + } + + @AfterEach + void tearDown() { + IOUtils.closeQuietly(out); + } + + @Test + void testDefaultAssessmentOutput() { + ScmClient scmClient = mock(ScmClient.class); + mockUsageInfo(scmClient, sampleNodes()); + + runAssessment(scmClient); + + String output = out.get(); + assertThat(output).contains("CLUSTER BALANCE ASSESSMENT"); + assertThat(output).contains("Drift"); + assertThat(output).contains("Source Nodes (over-utilized):"); + assertThat(output).contains("Target Nodes (under-utilized):"); + assertThat(output).contains("Movement Summary:"); + assertThat(output).contains("source-1"); + assertThat(output).contains("target-1"); + assertThat(output).contains("10 B"); + } + + @Test + void testThresholdOverride() { + ScmClient scmClient = mock(ScmClient.class); + mockUsageInfo(scmClient, sampleNodes()); + + runAssessment(scmClient, "-t", "5"); + + String output = out.get(); + assertThat(output).contains("15 B"); + assertThat(output).doesNotContain("10 B"); + } + + @Test + void testIncludeDatanodes() { + ScmClient scmClient = mock(ScmClient.class); + mockUsageInfo(scmClient, Arrays.asList( + proto("keep-source", 100, 10), + proto("keep-target", 100, 50), + proto("drop-me", 100, 5))); + + runAssessment(scmClient, "--include-datanodes", "keep-source,keep-target"); + + String output = out.get(); + assertThat(output).contains("keep-source"); + assertThat(output).contains("keep-target"); + assertThat(output).doesNotContain("drop-me"); + assertThat(output).contains("2 datanodes"); + } + + @Test + void testExcludeDatanodes() { + ScmClient scmClient = mock(ScmClient.class); + mockUsageInfo(scmClient, Arrays.asList( + proto("source-1", 100, 10), + proto("target-1", 100, 50), + proto("extra-node", 100, 90))); + + runAssessment(scmClient, "--exclude-datanodes", "extra-node"); + + String output = out.get(); + assertThat(output).doesNotContain("extra-node"); + assertThat(output).contains("source-1"); + assertThat(output).contains("target-1"); + assertThat(output).contains("2 datanodes"); + } + + @Test + void testNodeLimit() { + ScmClient scmClient = mock(ScmClient.class); + mockUsageInfo(scmClient, Arrays.asList( + proto("source-high", 100, 5), + proto("source-mid", 100, 20), + proto("target-low", 100, 80))); + + runAssessment(scmClient, "-n", "1"); + + String output = out.get(); + assertThat(output).contains("Top 1:"); + assertThat(output).contains("source-high"); + assertThat(output).doesNotContain("source-mid"); + } + + private static void mockUsageInfo(ScmClient scmClient, List nodes) { + try { + when(scmClient.getDatanodeUsageInfo(anyBoolean(), anyInt())).thenReturn(nodes); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void runAssessment(ScmClient scmClient, String... args) { + try { + CommandLine cli = new CommandLine(cmd); + cli.parseArgs(args); + cmd.execute(scmClient); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static List sampleNodes() { + return Arrays.asList( + proto("source-1", 100, 10), + proto("target-1", 100, 50)); + } + + private static DatanodeUsageInfoProto proto(String hostname, long capacity, long remaining) { + DatanodeDetails datanode = DatanodeDetails.newBuilder() + .setHostName(hostname) + .setIpAddress("127.0.0.1") + .setUuid(UUID.randomUUID()) + .build(); + long used = capacity - remaining; + return DatanodeUsageInfoProto.newBuilder() + .setNode(datanode.toProto(DEFAULT_VERSION.toProtoValue())) + .setCapacity(capacity) + .setRemaining(remaining) + .setUsed(used) + .build(); + } +} From fcb31360c610f9a58a27031dca1de60a36351e36 Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Fri, 18 Sep 2026 11:09:16 +0530 Subject: [PATCH 2/3] pmd --- .../cli/ContainerBalancerAssessmentSubcommand.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java index 3481c8ea1f0b..8cded86e2b0b 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java @@ -143,16 +143,16 @@ private static String getSummaryPrettyString(ContainerBalancerClusterSnapshot sn private static String getSourceNodesPrettyString(ContainerBalancerClusterSnapshot snapshot, int nodeLimit) { StringBuilder builder = new StringBuilder(); - builder.append("Source Nodes (over-utilized):").append(System.lineSeparator()); - builder.append(String.format("%-50s %s%n", "Datanodes above threshold", snapshot.getSourceCount())); + builder.append("Source Nodes (over-utilized):").append(System.lineSeparator()) + .append(String.format("%-50s %s%n", "Datanodes above threshold", snapshot.getSourceCount())); appendNodeList(builder, snapshot.getTopSourceNodes(), snapshot.getClusterAvgUtilization(), true, nodeLimit); return builder.toString(); } private static String getTargetNodesPrettyString(ContainerBalancerClusterSnapshot snapshot, int nodeLimit) { StringBuilder builder = new StringBuilder(); - builder.append("Target Nodes (under-utilized):").append(System.lineSeparator()); - builder.append(String.format("%-50s %s%n", "Datanodes below threshold", snapshot.getTargetCount())); + builder.append("Target Nodes (under-utilized):").append(System.lineSeparator()) + .append(String.format("%-50s %s%n", "Datanodes below threshold", snapshot.getTargetCount())); appendNodeList(builder, snapshot.getBottomTargetNodes(), snapshot.getClusterAvgUtilization(), false, nodeLimit); return builder.toString(); } @@ -173,8 +173,8 @@ private static void appendNodeList(StringBuilder builder, if (nodes.isEmpty()) { return; } - builder.append(System.lineSeparator()); - builder.append(aboveMean ? "Top " + nodeLimit + ":" : "Bottom " + nodeLimit + ":") + builder.append(System.lineSeparator()) + .append(aboveMean ? "Top " + nodeLimit + ":" : "Bottom " + nodeLimit + ":") .append(System.lineSeparator()); for (ContainerBalancerClusterSnapshot.NodeUtilization node : nodes) { double deltaFromMean = (node.getUtilization() - clusterAvgUtilization) * 100; @@ -229,7 +229,7 @@ private static String formatMovementRatio(ContainerBalancerClusterSnapshot snaps } private static Set parseNodeList(String commaSeparated) { - if (commaSeparated == null || commaSeparated.trim().isEmpty()) { + if (commaSeparated == null || commaSeparated.isEmpty()) { return Collections.emptySet(); } return Arrays.stream(commaSeparated.split(",")) From f9c78663b1da79b78ce5e7e83cb5e73166094f60 Mon Sep 17 00:00:00 2001 From: sravani-revuri Date: Tue, 22 Sep 2026 11:59:33 +0530 Subject: [PATCH 3/3] output changes --- ...ContainerBalancerAssessmentSubcommand.java | 93 +++++++++++++++++-- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java index 8cded86e2b0b..613a14ecb258 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerAssessmentSubcommand.java @@ -22,8 +22,10 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -33,6 +35,7 @@ import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerClusterAnalyzer; import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerClusterSnapshot; import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration; +import org.apache.hadoop.hdds.server.JsonUtils; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -65,6 +68,11 @@ public class ContainerBalancerAssessmentSubcommand extends ScmSubcommand { "The Datanodes specified in this list are excluded from the assessment.") private Optional excludeNodes; + @Option(names = {"--json"}, + defaultValue = "false", + description = "Format output as JSON") + private boolean json; + @Override public void execute(ScmClient scmClient) throws IOException { ContainerBalancerConfiguration config = @@ -97,20 +105,24 @@ public void execute(ScmClient scmClient) throws IOException { ContainerBalancerClusterSnapshot snapshot = ContainerBalancerClusterAnalyzer.analyze(nodes, thresholdRatio, include, exclude, nodeLimit); - printReport(snapshot, nodeLimit); + if (json) { + printJsonReport(snapshot, thresholdRatio); + } else { + printReport(snapshot, thresholdRatio, nodeLimit); + } } - private static void printReport(ContainerBalancerClusterSnapshot snapshot, int nodeLimit) { + private static void printReport(ContainerBalancerClusterSnapshot snapshot, double thresholdRatio, int nodeLimit) { System.out.println("CLUSTER BALANCE ASSESSMENT"); if (snapshot.getTotalEligibleDatanodes() == 0) { - System.out.println(getSummaryPrettyString(snapshot)); + System.out.println(getSummaryPrettyString(snapshot, thresholdRatio)); System.out.println(); System.out.println("No eligible datanodes found for assessment."); return; } - System.out.println(getSummaryPrettyString(snapshot)); + System.out.println(getSummaryPrettyString(snapshot, thresholdRatio)); System.out.println(); System.out.println(getSourceNodesPrettyString(snapshot, nodeLimit)); System.out.println(); @@ -119,17 +131,30 @@ private static void printReport(ContainerBalancerClusterSnapshot snapshot, int n System.out.println(getMovementSummaryPrettyString(snapshot)); } - private static String getSummaryPrettyString(ContainerBalancerClusterSnapshot snapshot) { + private static String getSummaryPrettyString(ContainerBalancerClusterSnapshot snapshot, + double thresholdRatio) { if (snapshot.getTotalEligibleDatanodes() == 0) { return String.format("%-50s %s%n" + + "%-50s %s%n" + + "%-50s %s%n" + + "%-50s %s%n" + "%-50s %s%n", "Key", "Value", + "Threshold", formatPercent(thresholdRatio), + "Upper limit", formatPercent(snapshot.getUpperLimit()), + "Lower limit", formatPercent(snapshot.getLowerLimit()), "Eligible datanodes", "0 datanodes"); } return String.format("%-50s %s%n" + + "%-50s %s%n" + + "%-50s %s%n" + + "%-50s %s%n" + "%-50s %s%n" + "%-50s %s%n" + "%-50s %s%n" + "%-50s %s%n", "Key", "Value", + "Threshold", formatPercent(thresholdRatio), + "Upper limit", formatPercent(snapshot.getUpperLimit()), + "Lower limit", formatPercent(snapshot.getLowerLimit()), "Drift", String.format(Locale.US, "%.1f%% (max utilization %.1f%% - min utilization %.1f%%)", snapshot.getImbalance() * 100, @@ -171,10 +196,14 @@ private static void appendNodeList(StringBuilder builder, List nodes, double clusterAvgUtilization, boolean aboveMean, int nodeLimit) { if (nodes.isEmpty()) { + builder.append(System.lineSeparator()) + .append("(none)") + .append(System.lineSeparator()); return; } + int headerCount = Math.min(nodeLimit, nodes.size()); builder.append(System.lineSeparator()) - .append(aboveMean ? "Top " + nodeLimit + ":" : "Bottom " + nodeLimit + ":") + .append(aboveMean ? "Top " + headerCount + ":" : "Bottom " + headerCount + ":") .append(System.lineSeparator()); for (ContainerBalancerClusterSnapshot.NodeUtilization node : nodes) { double deltaFromMean = (node.getUtilization() - clusterAvgUtilization) * 100; @@ -228,6 +257,58 @@ private static String formatMovementRatio(ContainerBalancerClusterSnapshot snaps return "Low movement ratio"; } + private static void printJsonReport(ContainerBalancerClusterSnapshot snapshot, double thresholdRatio) + throws IOException { + Map result = new LinkedHashMap<>(); + result.put("totalEligibleDatanodes", snapshot.getTotalEligibleDatanodes()); + result.put("thresholdPercentage", String.format(Locale.US, "%.1f%%", thresholdRatio * 100)); + result.put("clusterAvgUtilizationPercentage", + String.format(Locale.US, "%.1f%%", snapshot.getClusterAvgUtilization() * 100)); + result.put("driftPercentage", + String.format(Locale.US, "%.1f%%", snapshot.getImbalance() * 100)); + result.put("maxUtilizationPercentage", + String.format(Locale.US, "%.1f%%", snapshot.getMaxUtilization() * 100)); + result.put("minUtilizationPercentage", + String.format(Locale.US, "%.1f%%", snapshot.getMinUtilization() * 100)); + result.put("upperLimitPercentage", + String.format(Locale.US, "%.1f%%", snapshot.getUpperLimit() * 100)); + result.put("lowerLimitPercentage", + String.format(Locale.US, "%.1f%%", snapshot.getLowerLimit() * 100)); + + Map sourceNodes = new LinkedHashMap<>(); + sourceNodes.put("count", snapshot.getSourceCount()); + sourceNodes.put("nodes", snapshot.getTopSourceNodes().stream() + .map(node -> { + Map nodeMap = new LinkedHashMap<>(); + nodeMap.put("hostname", node.getHostname()); + nodeMap.put("utilizationPercentage", + String.format(Locale.US, "%.1f%%", node.getUtilization() * 100)); + return nodeMap; + }) + .collect(Collectors.toList())); + result.put("sourceNodes", sourceNodes); + + Map targetNodes = new LinkedHashMap<>(); + targetNodes.put("count", snapshot.getTargetCount()); + targetNodes.put("nodes", snapshot.getBottomTargetNodes().stream() + .map(node -> { + Map nodeMap = new LinkedHashMap<>(); + nodeMap.put("hostname", node.getHostname()); + nodeMap.put("utilizationPercentage", + String.format(Locale.US, "%.1f%%", node.getUtilization() * 100)); + return nodeMap; + }) + .collect(Collectors.toList())); + result.put("targetNodes", targetNodes); + + result.put("bytesToMove", byteDesc(snapshot.getBytesToMove())); + result.put("movementRatioPercentage", String.format(Locale.US, "%.1f%%", + snapshot.getClusterCapacityBytes() == 0 ? 0 + : (double) snapshot.getBytesToMove() / snapshot.getClusterCapacityBytes() * 100)); + + System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(result)); + } + private static Set parseNodeList(String commaSeparated) { if (commaSeparated == null || commaSeparated.isEmpty()) { return Collections.emptySet();