HDDS-16179. Dry run command for container balancer with profiles

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerConfiguration.java

@@ -152,6 +152,51 @@ public final class ContainerBalancerConfiguration {
           "OVER_REPLICATED CLOSED/QUASI_CLOSED and HEALTHY QUASI_CLOSED containers.")
   private boolean includeNonStandardContainers = false;
 
+  @Config(key = "hdds.container.balancer.profile.slow.datanodes.involved.max.percentage", type = ConfigType.INT,
+      defaultValue = "10", tags = {ConfigTag.BALANCER},
+      description = "SLOW profile: max percent of eligible datanodes used in one iteration.")
+  private int profileSlowDatanodesMaxPercentage = 10;
+
+  @Config(key = "hdds.container.balancer.profile.slow.size.entering.target.max", type = ConfigType.SIZE,
+      defaultValue = "10GB", tags = {ConfigTag.BALANCER},
+      description = "SLOW profile: max bytes a target datanode may receive in one iteration.")
+  private long profileSlowMaxSizeEnteringTarget = 10 * OzoneConsts.GB;
+
+  @Config(key = "hdds.container.balancer.profile.slow.size.leaving.source.max", type = ConfigType.SIZE,
+      defaultValue = "10GB", tags = {ConfigTag.BALANCER},
+      description = "SLOW profile: max bytes a source datanode may send in one iteration.")
+  private long profileSlowMaxSizeLeavingSource = 10 * OzoneConsts.GB;
+
+  @Config(key = "hdds.container.balancer.profile.medium.datanodes.involved.max.percentage", type = ConfigType.INT,
+      defaultValue = "20", tags = {ConfigTag.BALANCER},
+      description = "MEDIUM profile: max percent of eligible datanodes used in one iteration.")
+  private int profileMediumDatanodesMaxPercentage = 20;
+
+  @Config(key = "hdds.container.balancer.profile.medium.size.entering.target.max", type = ConfigType.SIZE,
+      defaultValue = "26GB", tags = {ConfigTag.BALANCER},
+      description = "MEDIUM profile: max bytes a target datanode may receive in one iteration.")
+  private long profileMediumMaxSizeEnteringTarget = 26 * OzoneConsts.GB;
+
+  @Config(key = "hdds.container.balancer.profile.medium.size.leaving.source.max", type = ConfigType.SIZE,
+      defaultValue = "26GB", tags = {ConfigTag.BALANCER},
+      description = "MEDIUM profile: max bytes a source datanode may send in one iteration.")
+  private long profileMediumMaxSizeLeavingSource = 26 * OzoneConsts.GB;
+
+  @Config(key = "hdds.container.balancer.profile.fast.datanodes.involved.max.percentage", type = ConfigType.INT,
+      defaultValue = "40", tags = {ConfigTag.BALANCER},
+      description = "FAST profile: max percent of eligible datanodes used in one iteration.")
+  private int profileFastDatanodesMaxPercentage = 40;
+
+  @Config(key = "hdds.container.balancer.profile.fast.size.entering.target.max", type = ConfigType.SIZE,
+      defaultValue = "100GB", tags = {ConfigTag.BALANCER},
+      description = "FAST profile: max bytes a target datanode may receive in one iteration.")
+  private long profileFastMaxSizeEnteringTarget = 100 * OzoneConsts.GB;
+
+  @Config(key = "hdds.container.balancer.profile.fast.size.leaving.source.max", type = ConfigType.SIZE,
+      defaultValue = "100GB", tags = {ConfigTag.BALANCER},
+      description = "FAST profile: max bytes a source datanode may send in one iteration.")
+  private long profileFastMaxSizeLeavingSource = 100 * OzoneConsts.GB;
+
   /**
    * Gets the threshold value for Container Balancer.
    *
@@ -261,7 +306,21 @@ public double getMaxDatanodesRatioToInvolvePerIteration() {
    * @return maximum datanodes that may be involved in one iteration
    */
   public int computeMaxDatanodesToInvolvePerIteration(int eligibleDatanodeCount) {
-    return (int) (getMaxDatanodesRatioToInvolvePerIteration() * eligibleDatanodeCount);
+    return computeMaxDatanodesToInvolvePerIteration(
+        getMaxDatanodesRatioToInvolvePerIteration(), eligibleDatanodeCount);
+  }
+
+  /**
+   * Computes the maximum number of datanodes that may be involved in an
+   * iteration for the given percentage and eligible datanode count.
+   *
+   * @param maxDatanodesPercentage percentage of eligible datanodes to involve
+   * @param eligibleDatanodeCount number of healthy, in-service datanodes
+   * @return maximum datanodes that may be involved in one iteration
+   */
+  public static int computeMaxDatanodesToInvolvePerIteration(
+      double maxDatanodesPercentage, int eligibleDatanodeCount) {
+    return (int) (maxDatanodesPercentage * eligibleDatanodeCount);
   }
 
   /**
@@ -467,6 +526,48 @@ public void setIncludeNonStandardContainers(boolean enable) {
     includeNonStandardContainers = enable;
   }
 
+  /** Returns the preset max datanode involvement percent for the given profile. */
+  public int getProfileDatanodesMaxPercentage(ContainerBalancerProfile profile) {
+    switch (profile) {
+    case SLOW:
+      return profileSlowDatanodesMaxPercentage;
+    case MEDIUM:
+      return profileMediumDatanodesMaxPercentage;
+    case FAST:
+      return profileFastDatanodesMaxPercentage;
+    default:
+      throw new IllegalArgumentException("Unknown profile: " + profile);
+    }
+  }
+
+  /** Returns the preset max bytes entering a target datanode per iteration for the given profile. */
+  public long getProfileMaxSizeEnteringTarget(ContainerBalancerProfile profile) {
+    switch (profile) {
+    case SLOW:
+      return profileSlowMaxSizeEnteringTarget;
+    case MEDIUM:
+      return profileMediumMaxSizeEnteringTarget;
+    case FAST:
+      return profileFastMaxSizeEnteringTarget;
+    default:
+      throw new IllegalArgumentException("Unknown profile: " + profile);
+    }
+  }
+
+  /** Returns the preset max bytes leaving a source datanode per iteration for the given profile. */
+  public long getProfileMaxSizeLeavingSource(ContainerBalancerProfile profile) {
+    switch (profile) {
+    case SLOW:
+      return profileSlowMaxSizeLeavingSource;
+    case MEDIUM:
+      return profileMediumMaxSizeLeavingSource;
+    case FAST:
+      return profileFastMaxSizeLeavingSource;
+    default:
+      throw new IllegalArgumentException("Unknown profile: " + profile);
+    }
+  }
+
   @Override
   public String toString() {
     return String.format("Container Balancer Configuration values:%n" +

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerEstimation.java

@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+import java.util.Objects;
+
+/**
+ * Dry-run estimate for a single container balancer profile.
+ */
+public final class ContainerBalancerEstimation {
+
+  private final ContainerBalancerProfile profile;
+  private final String failureMessage;
+  private final long bytesToMove;
+  private final long perIterationBytes;
+  private final long estimatedIterations;
+  private final long estimatedDurationMillis;
+  private final int maxDatanodesPercentage;
+  private final long maxSizeEnteringTarget;
+  private final long maxSizeLeavingSource;
+  private final long maxSizeToMovePerIteration;
+  private final long moveTimeoutMillis;
+  private final long balancingIntervalMillis;
+
+  private ContainerBalancerEstimation(Builder b) {
+    this.profile = Objects.requireNonNull(b.profile, "profile == null");
+    this.failureMessage = b.failureMessage;
+    this.bytesToMove = b.bytesToMove;
+    this.perIterationBytes = b.perIterationBytes;
+    this.estimatedIterations = b.estimatedIterations;
+    this.estimatedDurationMillis = b.estimatedDurationMillis;
+    this.maxDatanodesPercentage = b.maxDatanodesPercentage;
+    this.maxSizeEnteringTarget = b.maxSizeEnteringTarget;
+    this.maxSizeLeavingSource = b.maxSizeLeavingSource;
+    this.maxSizeToMovePerIteration = b.maxSizeToMovePerIteration;
+    this.moveTimeoutMillis = b.moveTimeoutMillis;
+    this.balancingIntervalMillis = b.balancingIntervalMillis;
+  }
+
+  public static Builder newBuilder() {
+    return new Builder();
+  }
+
+  public ContainerBalancerProfile getProfile() {
+    return profile;
+  }
+
+  public boolean succeeded() {
+    return failureMessage == null;
+  }
+
+  public String getFailureMessage() {
+    return failureMessage;
+  }
+
+  public long getBytesToMove() {
+    return bytesToMove;
+  }
+
+  public long getPerIterationBytes() {
+    return perIterationBytes;
+  }
+
+  public long getEstimatedIterations() {
+    return estimatedIterations;
+  }
+
+  public long getEstimatedDurationMillis() {
+    return estimatedDurationMillis;
+  }
+
+  public int getMaxDatanodesPercentage() {
+    return maxDatanodesPercentage;
+  }
+
+  public long getMaxSizeEnteringTarget() {
+    return maxSizeEnteringTarget;
+  }
+
+  public long getMaxSizeLeavingSource() {
+    return maxSizeLeavingSource;
+  }
+
+  public long getMaxSizeToMovePerIteration() {
+    return maxSizeToMovePerIteration;
+  }
+
+  public long getMoveTimeoutMillis() {
+    return moveTimeoutMillis;
+  }
+
+  public long getBalancingIntervalMillis() {
+    return balancingIntervalMillis;
+  }
+
+  /** Builder for {@link ContainerBalancerEstimation}. */
+  public static final class Builder {
+    private ContainerBalancerProfile profile;
+    private String failureMessage;
+    private long bytesToMove;
+    private long perIterationBytes;
+    private long estimatedIterations;
+    private long estimatedDurationMillis;
+    private int maxDatanodesPercentage;
+    private long maxSizeEnteringTarget;
+    private long maxSizeLeavingSource;
+    private long maxSizeToMovePerIteration;
+    private long moveTimeoutMillis;
+    private long balancingIntervalMillis;
+
+    private Builder() {
+    }
+
+    public Builder setProfile(ContainerBalancerProfile profileValue) {
+      this.profile = profileValue;
+      return this;
+    }
+
+    public Builder setFailureMessage(String message) {
+      this.failureMessage = message;
+      return this;
+    }
+
+    public Builder setBytesToMove(long bytes) {
+      this.bytesToMove = bytes;
+      return this;
+    }
+
+    public Builder setPerIterationBytes(long bytes) {
+      this.perIterationBytes = bytes;
+      return this;
+    }
+
+    public Builder setEstimatedIterations(long iterations) {
+      this.estimatedIterations = iterations;
+      return this;
+    }
+
+    public Builder setEstimatedDurationMillis(long durationMillis) {
+      this.estimatedDurationMillis = durationMillis;
+      return this;
+    }
+
+    public Builder setMaxDatanodesPercentage(int percentage) {
+      this.maxDatanodesPercentage = percentage;
+      return this;
+    }
+
+    public Builder setMaxSizeEnteringTarget(long bytes) {
+      this.maxSizeEnteringTarget = bytes;
+      return this;
+    }
+
+    public Builder setMaxSizeLeavingSource(long bytes) {
+      this.maxSizeLeavingSource = bytes;
+      return this;
+    }
+
+    public Builder setMaxSizeToMovePerIteration(long bytes) {
+      this.maxSizeToMovePerIteration = bytes;
+      return this;
+    }
+
+    public Builder setMoveTimeoutMillis(long millis) {
+      this.moveTimeoutMillis = millis;
+      return this;
+    }
+
+    public Builder setBalancingIntervalMillis(long millis) {
+      this.balancingIntervalMillis = millis;
+      return this;
+    }
+
+    public ContainerBalancerEstimation build() {
+      return new ContainerBalancerEstimation(this);
+    }
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerProfile.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+/**
+ * Throttling profile for container balancer.
+ */
+
+public enum ContainerBalancerProfile {
+  SLOW,
+  MEDIUM,
+  FAST;
+
+  public int getDatanodesMaxPercentage(ContainerBalancerConfiguration conf) {
+    return conf.getProfileDatanodesMaxPercentage(this);
+  }
+
+  public long getMaxSizeEnteringTarget(ContainerBalancerConfiguration conf) {
+    return conf.getProfileMaxSizeEnteringTarget(this);
+  }
+
+  public long getMaxSizeLeavingSource(ContainerBalancerConfiguration conf) {
+    return conf.getProfileMaxSizeLeavingSource(this);
+  }
+}