CAMEL-24862: the recursive file watcher watches directories created while it runs, and ignores the compile work directory

core/camel-core/src/test/java/org/apache/camel/support/FileWatcherResourceReloadStrategyTest.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.support;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.impl.engine.DefaultCompileStrategy;
+import org.apache.camel.spi.CompileStrategy;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * CAMEL-24862: the file watcher, when recursive, watches a directory created while it runs, and it ignores the compile
+ * work directory where the runtime writes the class files it compiles.
+ */
+@DisabledIfSystemProperty(named = "ci.env.name", matches = ".*",
+                          disabledReason = "Runs only local: the JDK watch service on a CI file system is too slow to time")
+public class FileWatcherResourceReloadStrategyTest extends ContextTestSupport {
+
+    @TempDir
+    Path dir;
+
+    @Override
+    public boolean isUseRouteBuilder() {
+        return false;
+    }
+
+    @Test
+    public void testNewDirectoryIsWatchedAndCompileWorkDirIsNot() throws Exception {
+        Path compile = dir.resolve(".camel-jbang/compile");
+        Files.createDirectories(compile);
+        CompileStrategy cs = new DefaultCompileStrategy();
+        cs.setWorkDir(compile.toString());
+        context.getCamelContextExtension().addContextPlugin(CompileStrategy.class, cs);
+
+        List<String> reloaded = new CopyOnWriteArrayList<>();
+        FileWatcherResourceReloadStrategy strategy = new FileWatcherResourceReloadStrategy(dir.toString(), true);
+        strategy.setCamelContext(context);
+        strategy.setResourceReload((name, resource) -> reloaded.add(name));
+        strategy.start();
+        try {
+            // a tree created after the start, with its first file in it
+            Path tree = dir.resolve("src/main/java/camel/example");
+            Files.createDirectories(tree);
+            Files.writeString(tree.resolve("OrderNumber.java"), "package camel.example; public class OrderNumber {}");
+            await().atMost(java.time.Duration.ofSeconds(30))
+                    .until(() -> reloaded.stream().anyMatch(n -> n.endsWith("OrderNumber.java")));
+
+            // a file written later into the new tree is seen as well: the directory is watched now
+            Files.writeString(tree.resolve("Other.java"), "package camel.example; public class Other {}");
+            await().atMost(java.time.Duration.ofSeconds(30))
+                    .until(() -> reloaded.stream().anyMatch(n -> n.endsWith("Other.java")));
+
+            // a class file the runtime writes into the compile work dir is not a change
+            Files.createDirectories(compile.resolve("camel/example"));
+            Files.writeString(compile.resolve("camel/example/OrderNumber.class"), "bytecode");
+            Files.writeString(dir.resolve("marker.txt"), "after the class file");
+            await().atMost(java.time.Duration.ofSeconds(30))
+                    .until(() -> reloaded.stream().anyMatch(n -> n.endsWith("marker.txt")));
+            assertFalse(reloaded.stream().anyMatch(n -> n.endsWith(".class")), "class files reloaded: " + reloaded);
+            assertTrue(reloaded.stream().noneMatch(n -> n.contains(".camel-jbang")), reloaded.toString());
+        } finally {
+            strategy.stop();
+        }
+    }
+}

core/camel-support/src/main/java/org/apache/camel/support/FileWatcherResourceReloadStrategy.java

@@ -27,15 +27,20 @@
 import java.nio.file.WatchKey;
 import java.nio.file.WatchService;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.camel.RuntimeCamelException;
 import org.apache.camel.api.management.ManagedAttribute;
 import org.apache.camel.api.management.ManagedResource;
+import org.apache.camel.spi.CompileStrategy;
 import org.apache.camel.spi.Resource;
 import org.apache.camel.util.FileUtil;
 import org.apache.camel.util.IOHelper;
@@ -64,6 +69,10 @@ public class FileWatcherResourceReloadStrategy extends ResourceReloadStrategySup
     ExecutorService executorService;
     WatchFileChangesTask task;
     Map<WatchKey, Path> folderKeys;
+    Set<Path> watchedFolders;
+    WatchEvent.Modifier watchModifier;
+    /** The compile work directory, resolved once at start (null when there is none). */
+    Path compileWorkDir;
     FileFilter fileFilter;
     String folder;
     boolean isRecursive;
@@ -188,8 +197,11 @@ protected void doStart() throws Exception {
                 Path path = dir.toPath();
                 watcher = path.getFileSystem().newWatchService();
                 // we cannot support deleting files as we don't know which routes that would be
+                this.watchModifier = modifier;
+                this.compileWorkDir = resolveCompileWorkDir();
                 if (isRecursive) {
                     this.folderKeys = new HashMap<>();
+                    this.watchedFolders = new HashSet<>();
                     registerRecursive(watcher, path, modifier);
                 } else {
                     registerPathToWatcher(modifier, path, watcher);
@@ -225,13 +237,64 @@ private void registerRecursive(final WatchService watcher, final Path root, fina
         Files.walkFileTree(root, new SimpleFileVisitor<>() {
             @Override
             public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                if (isCompileWorkDir(dir)) {
+                    return FileVisitResult.SKIP_SUBTREE;
+                }
                 WatchKey key = registerPathToWatcher(modifier, dir, watcher);
                 folderKeys.put(key, dir);
+                watchedFolders.add(dir);
                 return FileVisitResult.CONTINUE;
             }
         });
     }
 
+    /**
+     * Registers a directory created while watching recursively, and its subdirectories, and collects the files already
+     * in them as changes: a tree such as src/main/java/com/acme is usually created with its first file in it, before
+     * the watcher can see the directory.
+     */
+    private void registerNewDirectory(Path dir, List<File> changed) {
+        try {
+            Files.walkFileTree(dir, new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult preVisitDirectory(Path d, BasicFileAttributes attrs) throws IOException {
+                    if (isCompileWorkDir(d) || watchedFolders.contains(d)) {
+                        return FileVisitResult.SKIP_SUBTREE;
+                    }
+                    WatchKey k = registerPathToWatcher(watchModifier, d, watcher);
+                    folderKeys.put(k, d);
+                    watchedFolders.add(d);
+                    LOG.debug("Watching new directory: {}", d);
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path f, BasicFileAttributes attrs) {
+                    changed.add(f.toFile());
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            LOG.warn("Cannot watch new directory: {} due to: {}. This exception is ignored.", dir, e.getMessage(), e);
+        }
+    }
+
+    /**
+     * Whether the directory is the compile work directory (or inside it): the class files the runtime writes there when
+     * it compiles a Java source are not changes to watch, and would otherwise trigger a reload of their own, which
+     * compiles again, which writes again (CAMEL-24862).
+     */
+    protected boolean isCompileWorkDir(Path dir) {
+        return compileWorkDir != null && dir.toAbsolutePath().normalize().startsWith(compileWorkDir);
+    }
+
+    private Path resolveCompileWorkDir() {
+        CompileStrategy cs = getCamelContext() != null
+                ? getCamelContext().getCamelContextExtension().getContextPlugin(CompileStrategy.class) : null;
+        String workDir = cs != null ? cs.getWorkDir() : null;
+        return workDir != null ? Path.of(workDir).toAbsolutePath().normalize() : null;
+    }
+
     @Override
     protected void doStop() throws Exception {
         super.doStop();
@@ -290,15 +353,28 @@ public void run() {
                         pathToReload = folder;
                     }
 
+                    // the files of the events, plus the files of a directory created under a watched one
+                    // when recursive (registered here, since the watch service only reports what is registered
+                    // at the time; a class under src/main/java added while running was never seen, CAMEL-24862)
+                    List<File> changed = new ArrayList<>();
                     for (WatchEvent<?> event : key.pollEvents()) {
                         WatchEvent<Path> we = (WatchEvent<Path>) event;
                         Path path = we.context();
                         File file = pathToReload.resolve(path).toFile();
                         LOG.trace("File watch-event: {} on file: {}", we, file);
                         if (file.isDirectory()) {
+                            if (isRecursive && we.kind() == ENTRY_CREATE && !isCompileWorkDir(file.toPath())) {
+                                registerNewDirectory(file.toPath(), changed);
+                            }
                             continue;
                         }
-
+                        if (isCompileWorkDir(file.toPath().getParent())) {
+                            // a class file the runtime wrote while compiling: not a change of ours
+                            continue;
+                        }
+                        changed.add(file);
+                    }
+                    for (File file : changed) {
                         String name = FileUtil.compactPath(file.getPath());
                         LOG.debug("Detected Modified/Created file: {}", name);
                         boolean accept = fileFilter == null || fileFilter.accept(file);

docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc

@@ -481,6 +481,15 @@ file, and a WARN says so. Before, the previous routes stayed stopped until the n
 in one file left the application without routes. The failed file loads again on its next save. The
 `CamelContextReloadFailure` event and the reload error log line are unchanged.
 
+=== camel-core - the recursive file watcher watches directories created while it runs
+
+The file watcher reload strategy (`camel run --dev`, the route watcher reload strategy with a recursive directory)
+registered the directory tree once at start: a directory created afterwards, such as `src/main/java/com/acme` for a
+Java class added to a running project, was never watched and its files never reloaded. A directory created under a
+watched one is now registered as it appears, with the files already in it reloaded. The watcher also ignores the
+compile work directory (`camel.main.compileWorkDir`, `.camel-jbang/compile` for the CLI), where the runtime writes the
+class files it compiles, so a compiled source no longer triggers a reload of its own.
+
 === camel-core - the type of a bean created by a script or a builder is optional
 
 The `type` (class name) of a bean definition — `bean` under `beans`, `templateBean` of a route