From 7e8e1d384a56c4eece284bb7278f9ccbffb42ad9 Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Sat, 19 Sep 2026 09:50:32 +0530 Subject: [PATCH 1/4] HDDS-16504. Datanode startup can hang indefinitely when Ratis group recovery stalls (add a timeout/watchdog around container initialization) - Initial Commit --- .../statemachine/DatanodeConfiguration.java | 25 ++++++++ .../container/ozoneimpl/OzoneContainer.java | 62 ++++++++++++++++++- .../ozoneimpl/TestOzoneContainer.java | 37 +++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index 0287084058aa..ea9eb4a1bf58 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@ -94,6 +94,7 @@ public class DatanodeConfiguration extends ReconfigurableConfig { public static final String CONTAINER_CLIENT_CACHE_SIZE = "hdds.datanode.container.client.cache.size"; public static final String CONTAINER_CLIENT_CACHE_STALE_THRESHOLD = "hdds.datanode.container.client.cache.stale.threshold"; + public static final String CONTAINER_INIT_TIMEOUT_KEY = "hdds.datanode.container.init.timeout"; static final boolean CHUNK_DATA_VALIDATION_CHECK_DEFAULT = false; @@ -471,6 +472,22 @@ public class DatanodeConfiguration extends ReconfigurableConfig { ) private Duration diskCheckTimeout = DISK_CHECK_TIMEOUT_DEFAULT; + @Config(key = "hdds.datanode.container.init.timeout", + defaultValue = "0s", + type = ConfigType.TIME, + tags = { DATANODE }, + description = "Maximum time allowed for datanode container services to" + + " initialize at startup. Initialization builds the container set" + + " and starts the Ratis write channel, which recovers all Raft" + + " groups from disk. If it does not complete within this time," + + " startup is failed so the datanode shuts down cleanly instead of" + + " hanging indefinitely (for example when Ratis group recovery" + + " stalls on a failing volume). A value of 0 (the default) disables" + + " the watchdog and preserves the previous behavior of waiting" + + " indefinitely. Unit could be defined with postfix (ns,ms,s,m,h,d)." + ) + private Duration containerInitTimeout = Duration.ZERO; + @Config(key = "hdds.datanode.disk.check.sliding.window.timeout", defaultValue = "70m", type = ConfigType.TIME, @@ -1093,6 +1110,14 @@ public void setDiskCheckTimeout(Duration duration) { diskCheckTimeout = duration; } + public Duration getContainerInitTimeout() { + return containerInitTimeout; + } + + public void setContainerInitTimeout(Duration duration) { + containerInitTimeout = duration; + } + public void setDiskCheckEnabled(boolean diskCheckEnabled) { isDiskCheckEnabled = diskCheckEnabled; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index ff1fedaa6ace..c08cba5a0887 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -44,12 +44,16 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -561,7 +565,7 @@ public void start(String clusterId) throws IOException { initializingStatus = InitializingStatus.INITIALIZING; try { - initializeContainerServices(clusterId); + initializeContainerServicesWithTimeout(clusterId); initializingStatus = InitializingStatus.INITIALIZED; } catch (IOException | RuntimeException | Error ex) { // Partially started services cannot safely be initialized again. @@ -572,6 +576,62 @@ public void start(String clusterId) throws IOException { } } + /** + * Runs {@link #initializeContainerServices(String)}, optionally bounded by a + * watchdog timeout ({@code hdds.datanode.container.init.timeout}). + *

+ * When the timeout is a positive duration, initialization runs on a separate + * thread so a stall - for example Ratis group recovery blocked on a failing + * volume inside {@code writeChannel.start()} - is turned into an + * {@link IOException} instead of an indefinite hang. The caller ({@code + * start}) then records the failure ({@code FAILED}) and the datanode shuts + * down cleanly. A non-positive timeout disables the watchdog and runs + * initialization inline, preserving the previous behavior. + */ + private void initializeContainerServicesWithTimeout(String clusterId) throws IOException { + Duration initTimeout = + config.getObject(DatanodeConfiguration.class).getContainerInitTimeout(); + if (initTimeout == null || initTimeout.isZero() || initTimeout.isNegative()) { + initializeContainerServices(clusterId); + return; + } + + ExecutorService initExecutor = Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("OzoneContainerInit").build()); + Future initFuture = initExecutor.submit(() -> { + initializeContainerServices(clusterId); + return null; + }); + try { + initFuture.get(initTimeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + // Best-effort interrupt; a thread blocked on disk I/O may not respond, + // but the datanode will shut down once the caller marks startup FAILED. + initFuture.cancel(true); + throw new IOException("OzoneContainer initialization did not complete within " + initTimeout + + ". Failing datanode startup; a stalled Ratis group recovery or volume I/O is the likely cause.", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw (IOException) cause; + } + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new IOException("OzoneContainer initialization failed", cause); + } catch (InterruptedException e) { + initFuture.cancel(true); + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while initializing OzoneContainer", e); + } finally { + initExecutor.shutdownNow(); + } + } + private void initializeContainerServices(String clusterId) throws IOException { DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(config); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 9f8f9fe4604f..d9a5ab43934c 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -51,6 +51,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -69,6 +70,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.interfaces.DBHandle; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; @@ -82,6 +84,7 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -197,6 +200,40 @@ void testConcurrentStartup(int failureType) throws Exception { } } + @Test + void testInitTimeoutFailsStartupWhenInitializationStalls() throws Exception { + conf = SCMTestUtils.getConf(folder.toFile()); + conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_RANDOM_PORT, true); + conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_IPC_RANDOM_PORT, true); + // Enable the init watchdog with a short timeout so a stall fails fast. + conf.setTimeDuration(DatanodeConfiguration.CONTAINER_INIT_TIMEOUT_KEY, 1, TimeUnit.SECONDS); + ContainerTestUtils.initializeDatanodeLayout(conf, datanodeDetails); + OzoneContainer container = spy(ContainerTestUtils.getOzoneContainer(datanodeDetails, conf)); + + CountDownLatch stalling = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + // Simulate an initialization step that stalls past the configured timeout. + doAnswer(invocation -> { + stalling.countDown(); + release.await(30, TimeUnit.SECONDS); + return invocation.callRealMethod(); + }).when(container).buildContainerSet(); + + // The watchdog turns the stall into an IOException caused by a TimeoutException. + IOException firstFailure = assertThrows(IOException.class, () -> container.start(clusterId)); + assertThat(firstFailure).hasCauseInstanceOf(TimeoutException.class); + assertThat(stalling.await(5, TimeUnit.SECONDS)).isTrue(); + + // Startup is now FAILED: later callers fail fast with the same recorded cause. + assertThat(assertThrows(IOException.class, () -> container.start(clusterId))) + .hasCause(firstFailure); + } finally { + release.countDown(); + container.stop(); + } + } + /** * Create a mock {@link HddsVolume} to track container IDs. */ From 2b652dc0cf63d4a804a98334732f0f6c107cb96d Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Sat, 19 Sep 2026 21:35:51 +0530 Subject: [PATCH 2/4] HDDS-16504. Fixed review comments for handling using timeout thread to timeout and terminate the datanode --- .../statemachine/DatanodeStateMachine.java | 12 +++ .../container/ozoneimpl/OzoneContainer.java | 89 ++++++++++--------- .../common/TestDatanodeStateMachine.java | 46 ++++++++++ .../ozoneimpl/TestOzoneContainer.java | 37 -------- 4 files changed, 104 insertions(+), 80 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index 4f6078d0bd2f..04e714b8aed4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -400,6 +400,18 @@ private void startStateMachineThread() throws IOException { } } + /** + * Terminates the datanode when startup cannot make progress and the state + * machine loop cannot drive the shutdown itself (for example when container + * initialization has stalled on a thread that ignores interruption). Invoked + * from the {@link org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer} + * startup watchdog. + */ + public void triggerFatalShutdown(String reason) { + LOG.error("DatanodeStateMachine shutdown triggered: {}", reason); + hddsDatanodeStopService.stopService(); + } + public void handleFatalVolumeFailures() { LOG.error("DatanodeStateMachine Shutdown due to too many bad volumes, " + "check " + DatanodeConfiguration.FAILED_DATA_VOLUMES_TOLERATED_KEY diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index c08cba5a0887..b3c3060482c0 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -44,16 +44,13 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -88,6 +85,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.report.IncrementalReportSender; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerDomainSocket; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc; @@ -577,16 +575,19 @@ public void start(String clusterId) throws IOException { } /** - * Runs {@link #initializeContainerServices(String)}, optionally bounded by a - * watchdog timeout ({@code hdds.datanode.container.init.timeout}). + * Runs {@link #initializeContainerServices(String)} on the calling thread, + * optionally guarded by a startup watchdog + * ({@code hdds.datanode.container.init.timeout}). *

- * When the timeout is a positive duration, initialization runs on a separate - * thread so a stall - for example Ratis group recovery blocked on a failing - * volume inside {@code writeChannel.start()} - is turned into an - * {@link IOException} instead of an indefinite hang. The caller ({@code - * start}) then records the failure ({@code FAILED}) and the datanode shuts - * down cleanly. A non-positive timeout disables the watchdog and runs - * initialization inline, preserving the previous behavior. + * When the timeout is a positive duration, a watchdog is scheduled before + * initialization begins. If initialization has not finished by then - for + * example because Ratis group recovery inside {@code writeChannel.start()} + * has stalled on a failing volume - the watchdog terminates the datanode via + * {@link DatanodeStateMachine#triggerFatalShutdown(String)} instead of + * letting startup hang indefinitely. Because initialization runs on a single + * thread, no separate worker can keep starting services after the timeout. + * A non-positive timeout disables the watchdog, preserving the previous + * behavior. */ private void initializeContainerServicesWithTimeout(String clusterId) throws IOException { Duration initTimeout = @@ -596,39 +597,41 @@ private void initializeContainerServicesWithTimeout(String clusterId) throws IOE return; } - ExecutorService initExecutor = Executors.newSingleThreadExecutor( + // Guards against the watchdog firing just as initialization finishes. + AtomicBoolean initInProgress = new AtomicBoolean(true); + ScheduledExecutorService watchdog = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setDaemon(true) - .setNameFormat("OzoneContainerInit").build()); - Future initFuture = initExecutor.submit(() -> { - initializeContainerServices(clusterId); - return null; - }); - try { - initFuture.get(initTimeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - // Best-effort interrupt; a thread blocked on disk I/O may not respond, - // but the datanode will shut down once the caller marks startup FAILED. - initFuture.cancel(true); - throw new IOException("OzoneContainer initialization did not complete within " + initTimeout - + ". Failing datanode startup; a stalled Ratis group recovery or volume I/O is the likely cause.", e); - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof IOException) { - throw (IOException) cause; - } - if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; + .setNameFormat("OzoneContainerInitWatchdog").build()); + watchdog.schedule(() -> { + if (initInProgress.compareAndSet(true, false)) { + terminateOnInitializationTimeout(initTimeout); } - if (cause instanceof Error) { - throw (Error) cause; - } - throw new IOException("OzoneContainer initialization failed", cause); - } catch (InterruptedException e) { - initFuture.cancel(true); - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while initializing OzoneContainer", e); + }, initTimeout.toMillis(), TimeUnit.MILLISECONDS); + try { + initializeContainerServices(clusterId); } finally { - initExecutor.shutdownNow(); + initInProgress.set(false); + watchdog.shutdownNow(); + } + } + + /** + * Terminates the datanode when container initialization has stalled past the + * watchdog timeout. A hung step (for example a Ratis {@code join()} or a + * blocking disk read) cannot be interrupted reliably, so the only safe + * recovery is to fail the process and let it be restarted; the JVM exit also + * stops the stalled initialization thread. + */ + private void terminateOnInitializationTimeout(Duration initTimeout) { + String message = "OzoneContainer initialization did not complete within " + initTimeout + + ". Terminating datanode; a stalled Ratis group recovery or volume I/O is the likely cause."; + LOG.error(message); + StateContext current = context; + DatanodeStateMachine dsm = current == null ? null : current.getParent(); + if (dsm != null) { + dsm.triggerFatalShutdown(message); + } else { + LOG.error("No datanode state machine available; cannot automatically terminate the stalled datanode."); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java index ef03aa148dff..d69b1b882279 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java @@ -58,6 +58,7 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; @@ -240,6 +241,51 @@ void testDelayedRatisStartupFailureStopsDatanode() throws Exception { } } + @Test + @Timeout(60) + void testStalledInitializationTimeoutTerminatesDatanode() throws Exception { + conf.setFromObject(conf.getObject(ReplicationConfig.class).setPort(0)); + // Enable the startup watchdog with a short timeout. + conf.setTimeDuration(DatanodeConfiguration.CONTAINER_INIT_TIMEOUT_KEY, 2, TimeUnit.SECONDS); + DatanodeDetails datanodeDetails = getNewDatanodeDetails(); + ContainerTestUtils.initializeDatanodeLayout(conf, datanodeDetails); + CountDownLatch initializing = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch shutdown = new CountDownLatch(1); + HddsDatanodeStopService stopService = mock(HddsDatanodeStopService.class); + doAnswer(invocation -> { + shutdown.countDown(); + return null; + }).when(stopService).stopService(); + DatanodeStateMachine stateMachine = new DatanodeStateMachine(null, datanodeDetails, conf, null, null, + stopService, new ReconfigurationHandler("DN", conf, op -> { })); + try { + OzoneContainer container = stateMachine.getContainer(); + XceiverServerSpi writeChannel = spy(container.getWriteChannel()); + Field writeChannelField = OzoneContainer.class.getDeclaredField("writeChannel"); + writeChannelField.setAccessible(true); + writeChannelField.set(container, writeChannel); + // Stall Ratis startup: never returns and never throws, simulating a hang + // (for example a group recovery blocked on a failing volume). + doAnswer(invocation -> { + initializing.countDown(); + assertThat(release.await(30, TimeUnit.SECONDS)).isTrue(); + return null; + }).when(writeChannel).start(); + + stateMachine.startDaemon(); + assertThat(initializing.await(10, TimeUnit.SECONDS)).isTrue(); + + // The watchdog must terminate the datanode even though writeChannel.start() + // never returns and never throws. + assertThat(shutdown.await(20, TimeUnit.SECONDS)).isTrue(); + verify(stopService, times(1)).stopService(); + } finally { + release.countDown(); + stateMachine.stopDaemon(); + } + } + /** * This test explores the state machine by invoking each call in sequence just * like as if the state machine would call it. Because this is a test we are diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index d9a5ab43934c..9f8f9fe4604f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -51,7 +51,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -70,7 +69,6 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.interfaces.DBHandle; -import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; @@ -84,7 +82,6 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -200,40 +197,6 @@ void testConcurrentStartup(int failureType) throws Exception { } } - @Test - void testInitTimeoutFailsStartupWhenInitializationStalls() throws Exception { - conf = SCMTestUtils.getConf(folder.toFile()); - conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_RANDOM_PORT, true); - conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_IPC_RANDOM_PORT, true); - // Enable the init watchdog with a short timeout so a stall fails fast. - conf.setTimeDuration(DatanodeConfiguration.CONTAINER_INIT_TIMEOUT_KEY, 1, TimeUnit.SECONDS); - ContainerTestUtils.initializeDatanodeLayout(conf, datanodeDetails); - OzoneContainer container = spy(ContainerTestUtils.getOzoneContainer(datanodeDetails, conf)); - - CountDownLatch stalling = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - try { - // Simulate an initialization step that stalls past the configured timeout. - doAnswer(invocation -> { - stalling.countDown(); - release.await(30, TimeUnit.SECONDS); - return invocation.callRealMethod(); - }).when(container).buildContainerSet(); - - // The watchdog turns the stall into an IOException caused by a TimeoutException. - IOException firstFailure = assertThrows(IOException.class, () -> container.start(clusterId)); - assertThat(firstFailure).hasCauseInstanceOf(TimeoutException.class); - assertThat(stalling.await(5, TimeUnit.SECONDS)).isTrue(); - - // Startup is now FAILED: later callers fail fast with the same recorded cause. - assertThat(assertThrows(IOException.class, () -> container.start(clusterId))) - .hasCause(firstFailure); - } finally { - release.countDown(); - container.stop(); - } - } - /** * Create a mock {@link HddsVolume} to track container IDs. */ From 942476612165929238612cbf3ea52380f500fb95 Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Mon, 21 Sep 2026 10:20:10 +0530 Subject: [PATCH 3/4] HDDS-16504. Fixed review comments for handling using timeout thread to timeout and terminate the datanode --- .../statemachine/DatanodeStateMachine.java | 12 --------- .../container/ozoneimpl/OzoneContainer.java | 26 +++++++++---------- .../common/TestDatanodeStateMachine.java | 22 ++++++++-------- 3 files changed, 23 insertions(+), 37 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index 04e714b8aed4..4f6078d0bd2f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -400,18 +400,6 @@ private void startStateMachineThread() throws IOException { } } - /** - * Terminates the datanode when startup cannot make progress and the state - * machine loop cannot drive the shutdown itself (for example when container - * initialization has stalled on a thread that ignores interruption). Invoked - * from the {@link org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer} - * startup watchdog. - */ - public void triggerFatalShutdown(String reason) { - LOG.error("DatanodeStateMachine shutdown triggered: {}", reason); - hddsDatanodeStopService.stopService(); - } - public void handleFatalVolumeFailures() { LOG.error("DatanodeStateMachine Shutdown due to too many bad volumes, " + "check " + DatanodeConfiguration.FAILED_DATA_VOLUMES_TOLERATED_KEY diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index b3c3060482c0..65da4304d205 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -85,7 +85,6 @@ import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.report.IncrementalReportSender; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; -import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerDomainSocket; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc; @@ -110,6 +109,7 @@ import org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig; import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures.SchemaV3; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; import org.apache.ratis.grpc.GrpcTlsConfig; @@ -582,12 +582,11 @@ public void start(String clusterId) throws IOException { * When the timeout is a positive duration, a watchdog is scheduled before * initialization begins. If initialization has not finished by then - for * example because Ratis group recovery inside {@code writeChannel.start()} - * has stalled on a failing volume - the watchdog terminates the datanode via - * {@link DatanodeStateMachine#triggerFatalShutdown(String)} instead of - * letting startup hang indefinitely. Because initialization runs on a single - * thread, no separate worker can keep starting services after the timeout. - * A non-positive timeout disables the watchdog, preserving the previous - * behavior. + * has stalled on a failing volume - the watchdog terminates the JVM via + * {@link ExitUtil#terminate(int, String)} instead of letting startup hang + * indefinitely. Because initialization runs on a single thread, no separate + * worker can keep starting services after the timeout. A non-positive timeout + * disables the watchdog, preserving the previous behavior. */ private void initializeContainerServicesWithTimeout(String clusterId) throws IOException { Duration initTimeout = @@ -626,13 +625,12 @@ private void terminateOnInitializationTimeout(Duration initTimeout) { String message = "OzoneContainer initialization did not complete within " + initTimeout + ". Terminating datanode; a stalled Ratis group recovery or volume I/O is the likely cause."; LOG.error(message); - StateContext current = context; - DatanodeStateMachine dsm = current == null ? null : current.getParent(); - if (dsm != null) { - dsm.triggerFatalShutdown(message); - } else { - LOG.error("No datanode state machine available; cannot automatically terminate the stalled datanode."); - } + // Enter JVM shutdown directly rather than calling the datanode stop service. + // Its synchronous stop() can itself block on the same failing disk (for + // example a container scanner's Thread.join()), so it could hang before the + // process exits. System.exit runs the datanode cleanup registered with + // ShutdownHookManager, which bounds each hook with a timeout. + ExitUtil.terminate(1, message); } private void initializeContainerServices(String clusterId) throws IOException { diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java index d69b1b882279..d2e6aed7e310 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java @@ -71,6 +71,7 @@ import org.apache.hadoop.ozone.container.common.volume.CapacityVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; @@ -244,6 +245,8 @@ void testDelayedRatisStartupFailureStopsDatanode() throws Exception { @Test @Timeout(60) void testStalledInitializationTimeoutTerminatesDatanode() throws Exception { + ExitUtil.disableSystemExit(); + ExitUtil.resetFirstExitException(); conf.setFromObject(conf.getObject(ReplicationConfig.class).setPort(0)); // Enable the startup watchdog with a short timeout. conf.setTimeDuration(DatanodeConfiguration.CONTAINER_INIT_TIMEOUT_KEY, 2, TimeUnit.SECONDS); @@ -251,14 +254,8 @@ void testStalledInitializationTimeoutTerminatesDatanode() throws Exception { ContainerTestUtils.initializeDatanodeLayout(conf, datanodeDetails); CountDownLatch initializing = new CountDownLatch(1); CountDownLatch release = new CountDownLatch(1); - CountDownLatch shutdown = new CountDownLatch(1); - HddsDatanodeStopService stopService = mock(HddsDatanodeStopService.class); - doAnswer(invocation -> { - shutdown.countDown(); - return null; - }).when(stopService).stopService(); DatanodeStateMachine stateMachine = new DatanodeStateMachine(null, datanodeDetails, conf, null, null, - stopService, new ReconfigurationHandler("DN", conf, op -> { })); + mock(HddsDatanodeStopService.class), new ReconfigurationHandler("DN", conf, op -> { })); try { OzoneContainer container = stateMachine.getContainer(); XceiverServerSpi writeChannel = spy(container.getWriteChannel()); @@ -276,13 +273,16 @@ void testStalledInitializationTimeoutTerminatesDatanode() throws Exception { stateMachine.startDaemon(); assertThat(initializing.await(10, TimeUnit.SECONDS)).isTrue(); - // The watchdog must terminate the datanode even though writeChannel.start() - // never returns and never throws. - assertThat(shutdown.await(20, TimeUnit.SECONDS)).isTrue(); - verify(stopService, times(1)).stopService(); + // The watchdog must enter JVM shutdown even though writeChannel.start() + // never returns and never throws. Going straight to ExitUtil (System.exit) + // avoids the synchronous stop() path, which could itself block on the disk. + GenericTestUtils.waitFor(ExitUtil::terminateCalled, 100, 20000); + assertThat(ExitUtil.getFirstExitException().getExitCode()).isEqualTo(1); + assertThat(ExitUtil.getFirstExitException().getMessage()).contains("did not complete within"); } finally { release.countDown(); stateMachine.stopDaemon(); + ExitUtil.resetFirstExitException(); } } From 56b9ecd829ca4e0c349043c5f765a18dc90447ad Mon Sep 17 00:00:00 2001 From: Devesh Singh Date: Mon, 21 Sep 2026 16:11:01 +0530 Subject: [PATCH 4/4] HDDS-16504. Fixed review comments for handling using timeout thread to timeout and terminate the datanode --- .../statemachine/DatanodeStateMachine.java | 12 +++++++++ .../container/ozoneimpl/OzoneContainer.java | 26 ++++++++++++------- .../common/TestDatanodeStateMachine.java | 16 ++++++------ 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index 4f6078d0bd2f..bf00fb576fbb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -400,6 +400,18 @@ private void startStateMachineThread() throws IOException { } } + /** + * Terminates the datanode when startup cannot make progress and the state + * machine loop cannot drive the shutdown itself (for example when container + * initialization has stalled on a thread that ignores interruption). Invoked + * from the {@link org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer} + * startup watchdog. Enters JVM shutdown directly rather than the synchronous + * stop service, whose stop() could itself block on the same failing disk. + */ + public void triggerFatalShutdown(String reason) { + ExitUtils.terminate(1, reason, LOG); + } + public void handleFatalVolumeFailures() { LOG.error("DatanodeStateMachine Shutdown due to too many bad volumes, " + "check " + DatanodeConfiguration.FAILED_DATA_VOLUMES_TOLERATED_KEY diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 65da4304d205..424290b92b27 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -85,6 +85,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.report.IncrementalReportSender; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerDomainSocket; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc; @@ -109,7 +110,6 @@ import org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig; import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures.SchemaV3; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; -import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; import org.apache.ratis.grpc.GrpcTlsConfig; @@ -582,11 +582,12 @@ public void start(String clusterId) throws IOException { * When the timeout is a positive duration, a watchdog is scheduled before * initialization begins. If initialization has not finished by then - for * example because Ratis group recovery inside {@code writeChannel.start()} - * has stalled on a failing volume - the watchdog terminates the JVM via - * {@link ExitUtil#terminate(int, String)} instead of letting startup hang - * indefinitely. Because initialization runs on a single thread, no separate - * worker can keep starting services after the timeout. A non-positive timeout - * disables the watchdog, preserving the previous behavior. + * has stalled on a failing volume - the watchdog terminates the datanode via + * {@link DatanodeStateMachine#triggerFatalShutdown(String)} (which enters JVM + * shutdown) instead of letting startup hang indefinitely. Because + * initialization runs on a single thread, no separate worker can keep starting + * services after the timeout. A non-positive timeout disables the watchdog, + * preserving the previous behavior. */ private void initializeContainerServicesWithTimeout(String clusterId) throws IOException { Duration initTimeout = @@ -628,9 +629,16 @@ private void terminateOnInitializationTimeout(Duration initTimeout) { // Enter JVM shutdown directly rather than calling the datanode stop service. // Its synchronous stop() can itself block on the same failing disk (for // example a container scanner's Thread.join()), so it could hang before the - // process exits. System.exit runs the datanode cleanup registered with - // ShutdownHookManager, which bounds each hook with a timeout. - ExitUtil.terminate(1, message); + // process exits. triggerFatalShutdown() calls ExitUtils.terminate, and + // System.exit runs the datanode cleanup registered with ShutdownHookManager, + // which bounds each hook with a timeout. + StateContext current = context; + DatanodeStateMachine dsm = current == null ? null : current.getParent(); + if (dsm != null) { + dsm.triggerFatalShutdown(message); + } else { + LOG.error("No datanode state machine available; cannot automatically terminate the stalled datanode."); + } } private void initializeContainerServices(String clusterId) throws IOException { diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java index d2e6aed7e310..3db702ef3b51 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java @@ -71,10 +71,10 @@ import org.apache.hadoop.ozone.container.common.volume.CapacityVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig; -import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; +import org.apache.ratis.util.ExitUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -245,8 +245,8 @@ void testDelayedRatisStartupFailureStopsDatanode() throws Exception { @Test @Timeout(60) void testStalledInitializationTimeoutTerminatesDatanode() throws Exception { - ExitUtil.disableSystemExit(); - ExitUtil.resetFirstExitException(); + ExitUtils.disableSystemExit(); + ExitUtils.clear(); conf.setFromObject(conf.getObject(ReplicationConfig.class).setPort(0)); // Enable the startup watchdog with a short timeout. conf.setTimeDuration(DatanodeConfiguration.CONTAINER_INIT_TIMEOUT_KEY, 2, TimeUnit.SECONDS); @@ -274,15 +274,15 @@ void testStalledInitializationTimeoutTerminatesDatanode() throws Exception { assertThat(initializing.await(10, TimeUnit.SECONDS)).isTrue(); // The watchdog must enter JVM shutdown even though writeChannel.start() - // never returns and never throws. Going straight to ExitUtil (System.exit) + // never returns and never throws. Going straight to ExitUtils (System.exit) // avoids the synchronous stop() path, which could itself block on the disk. - GenericTestUtils.waitFor(ExitUtil::terminateCalled, 100, 20000); - assertThat(ExitUtil.getFirstExitException().getExitCode()).isEqualTo(1); - assertThat(ExitUtil.getFirstExitException().getMessage()).contains("did not complete within"); + GenericTestUtils.waitFor(ExitUtils::isTerminated, 100, 20000); + assertThat(ExitUtils.getFirstExitException().getStatus()).isEqualTo(1); + assertThat(ExitUtils.getFirstExitException().getMessage()).contains("did not complete within"); } finally { release.countDown(); stateMachine.stopDaemon(); - ExitUtil.resetFirstExitException(); + ExitUtils.clear(); } }