Add health endpoints for monitoring

.env

@@ -68,3 +68,12 @@ APP_KEEP_RESULTS=5
 APP_ECONOMICS_URI=https://economics.itkdev.dk
 APP_ECONOMICS_API_KEY=changeme
 ###< economics ###
+
+###> health ###
+# Seconds to cache the health check results, so that monitoring polling
+# /health/ready cannot amplify into load on the database and the broker.
+HEALTH_CACHE_TTL=15
+# Seconds since the last detection result before ingest is reported degraded.
+# The harvester currently reports several hundred times an hour.
+HEALTH_INGEST_MAX_AGE=1800
+###< health ###

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- [#91](https://github.com/itk-dev/devops_itksites/pull/91) Health endpoints
+  - Add `/health/live`, `/health/ready` and `/health/detail` endpoints
+  - Add health checks for database, RabbitMQ transport and detection result freshness
+  - Cache check results in a dedicated `cache.health` pool
+  - Exclude `^/health` from the firewalls and protect `/health/detail` with `ITKBasicAuth`
 - [#90](https://github.com/itk-dev/devops_itksites/pull/90)
   - Fixed user API key migration failing on databases with more than one user
   - Generated an API key for existing users, as users created since already get

README.md

@@ -63,6 +63,43 @@ Use the API key to make an authenticated request, e.g.
 curl --header 'accept: application/json' --header 'authorization: Apikey <the API key>' https://itksites.local.itkdev.dk/api/sites
 ```
 
+## Health checks
+
+Three endpoints report on the application, in increasing order of detail:
+
+| Endpoint | Access | Checks |
+| --- | --- | --- |
+| `/health/live` | Public | Nothing – only that the app responds |
+| `/health/ready` | Public | All checks, aggregated status only |
+| `/health/detail` | `ITKBasicAuth` in Traefik | Per-check results and timings |
+
+`/health/ready` answers `200` when everything is well and `503` when it is not.
+It deliberately does not say *what* is wrong – point monitoring at this one and
+read `/health/detail` when it goes red:
+
+``` shell
+curl --silent https://itksites.local.itkdev.dk/health/detail | jq
+```
+
+The checks cover the database, the RabbitMQ messenger transport and the
+freshness of the most recent detection result. The last one catches an ingest
+pipeline that has stopped while the application itself is still serving
+requests.
+
+`HEALTH_INGEST_MAX_AGE` sets how old the most recent detection result may be
+before ingest is reported as degraded.
+
+Results are cached for `HEALTH_CACHE_TTL` seconds so that polling does not turn
+into load on the dependencies. The cache is the dedicated, filesystem-backed
+`cache.health` pool in `config/packages/cache.yaml` – it has to keep working
+while the database and the broker are down, and the adapter can be swapped
+there without touching code.
+
+`^/health` is excluded from the Symfony firewalls: both user providers are
+Doctrine entity providers, so an authenticated endpoint would fail to
+authenticate during a database outage and answer `500` rather than reporting
+that the database is down.
+
 ## Development
 
 ```sh

config/packages/cache.yaml

@@ -15,5 +15,15 @@ framework:
         #app: cache.adapter.apcu
 
         # Namespaced pools use the above "app" backend by default
-        #pools:
-        #my.dedicated.cache: null
+        pools:
+            # Health check results.
+            #
+            # Filesystem-backed on purpose: this pool has to keep working while
+            # the database and the message broker are down, which is exactly
+            # when the health endpoints matter.
+            #
+            # It is a dedicated pool so the adapter can be swapped without
+            # touching code: cache.adapter.apcu is faster and shared between
+            # FPM workers, at the cost of being cleared on every FPM restart.
+            cache.health:
+                adapter: cache.adapter.filesystem

config/packages/security.yaml

@@ -21,6 +21,14 @@ security:
         dev:
             pattern: ^/(_(profiler|wdt)|css|images|js)/
             security: false
+        # The health endpoints must answer while the database is down, so they
+        # cannot go through a firewall: both user providers above are Doctrine
+        # entity providers and authentication would itself fail. /health/detail
+        # is protected by the ITKBasicAuth middleware in Traefik instead.
+        health:
+            pattern: ^/health
+            security: false
+
         api:
             pattern: ^/api
             custom_authenticators:

config/services.yaml

@@ -27,6 +27,23 @@ services:
         App\Handler\DetectionResultHandlerInterface:
             tags: [app.handler.detection_result_handler]
 
+        App\Health\HealthCheckInterface:
+            tags: [app.health.check]
+
+    App\Health\HealthChecker:
+        arguments:
+            $checks: !tagged_iterator app.health.check
+            $cache: '@cache.health'
+            $cacheTtl: '%env(int:HEALTH_CACHE_TTL)%'
+
+    App\Health\Check\RabbitMqHealthCheck:
+        arguments:
+            $transport: '@messenger.transport.async'
+
+    App\Health\Check\IngestFreshnessHealthCheck:
+        arguments:
+            $maxAgeSeconds: '%env(int:HEALTH_INGEST_MAX_AGE)%'
+
     App\EventListener\RemovedRelationsListener:
         tags:
             - name: 'doctrine.event_listener'

docker-compose.server.yml

@@ -50,3 +50,8 @@ services:
       # Cron-metrics protection.
       - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-metrics.rule=Host(`${COMPOSE_SERVER_DOMAIN:?}`) && PathPrefix(`/cron-metrics`) "
       - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-metrics.middlewares=ITKMetricsAuth@file"
+      # Detailed health check protection. /health/live and /health/ready stay
+      # public; only /health/detail discloses internals.
+      - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-health.rule=Host(`${COMPOSE_SERVER_DOMAIN:?}`) && PathPrefix(`/health/detail`)"
+      - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-health.entrypoints=websecure"
+      - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-health.middlewares=ITKBasicAuth@file"

docker-compose.yml

@@ -75,6 +75,10 @@ services:
       # Cron-metrics protection.
       - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-metrics.rule=Host(`${COMPOSE_DOMAIN:?}`) && PathPrefix(`/cron-metrics`) "
       - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-metrics.middlewares=ITKMetricsAuth@file"
+      # Detailed health check protection. /health/live and /health/ready stay
+      # public; only /health/detail discloses internals.
+      - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-health.rule=Host(`${COMPOSE_DOMAIN:?}`) && PathPrefix(`/health/detail`)"
+      - "traefik.http.routers.${COMPOSE_PROJECT_NAME:?}-health.middlewares=ITKBasicAuth@file"
 
   mail:
     image: axllent/mailpit

src/Controller/HealthController.php

@@ -0,0 +1,98 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Controller;
+
+use App\Health\HealthChecker;
+use App\Health\HealthStatus;
+use Symfony\Component\HttpFoundation\JsonResponse;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\Routing\Attribute\Route;
+
+/**
+ * Health endpoints, in three tiers.
+ *
+ * /health/live    Public. No dependencies at all. Says only that PHP-FPM is up
+ *                 and routing works. It must never touch the database: a
+ *                 liveness probe that fails during a database outage makes an
+ *                 orchestrator restart a container that is not the problem.
+ *
+ * /health/ready   Public, but opaque. Runs every check and answers 200 or 503
+ *                 with the aggregated status only. The status code is the
+ *                 payload; which dependency failed is not disclosed. This is
+ *                 the endpoint monitoring should watch.
+ *
+ * /health/detail  Per-check results, timings, queue depth. Discloses internals
+ *                 and MUST be protected at the edge — see the ITKBasicAuth
+ *                 middleware on the nginx service in docker-compose.server.yml.
+ *
+ * Authentication deliberately happens in Traefik rather than in Symfony. Both
+ * user providers in config/packages/security.yaml are Doctrine entity
+ * providers, so an application-level firewall on these routes would fail to
+ * authenticate during a database outage and answer 500 — precisely when the
+ * endpoint needs to answer "the database is down". ^/health is therefore
+ * excluded from the Symfony firewalls entirely.
+ */
+readonly class HealthController
+{
+    public function __construct(
+        private HealthChecker $healthChecker,
+    ) {
+    }
+
+    #[Route('/health/live', name: 'app_health_live', methods: ['GET'])]
+    public function live(): JsonResponse
+    {
+        return $this->respond(['status' => HealthStatus::Ok->value], true);
+    }
+
+    #[Route('/health/ready', name: 'app_health_ready', methods: ['GET'])]
+    public function ready(): JsonResponse
+    {
+        $healthy = $this->healthChecker->isHealthy($this->healthChecker->run());
+
+        return $this->respond(
+            ['status' => $healthy ? HealthStatus::Ok->value : HealthStatus::Degraded->value],
+            $healthy
+        );
+    }
+
+    #[Route('/health/detail', name: 'app_health_detail', methods: ['GET'])]
+    public function detail(): JsonResponse
+    {
+        $results = $this->healthChecker->run();
+        $healthy = $this->healthChecker->isHealthy($results);
+
+        $checks = [];
+        foreach ($results as $result) {
+            $checks[$result->name] = array_filter([
+                'status' => $result->status->value,
+                'message' => $result->message,
+                'details' => $result->details,
+            ], static fn (mixed $value): bool => null !== $value && [] !== $value);
+        }
+
+        return $this->respond([
+            'status' => $healthy ? HealthStatus::Ok->value : HealthStatus::Degraded->value,
+            'checks' => $checks,
+        ], $healthy);
+    }
+
+    /**
+     * @param array<string, mixed> $payload
+     */
+    private function respond(array $payload, bool $healthy): JsonResponse
+    {
+        $response = new JsonResponse(
+            $payload,
+            $healthy ? Response::HTTP_OK : Response::HTTP_SERVICE_UNAVAILABLE
+        );
+
+        // Health responses are cached inside HealthChecker, never by the client
+        // or an intermediary.
+        $response->headers->set('Cache-Control', 'no-store, private');
+
+        return $response;
+    }
+}

src/Health/Check/DatabaseHealthCheck.php

@@ -0,0 +1,48 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Health\Check;
+
+use App\Health\HealthCheckInterface;
+use App\Health\HealthCheckResult;
+use Doctrine\DBAL\Connection;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Verifies that the database accepts connections and answers queries.
+ */
+readonly class DatabaseHealthCheck implements HealthCheckInterface
+{
+    public function __construct(
+        private Connection $connection,
+        private LoggerInterface $logger,
+    ) {
+    }
+
+    public function getName(): string
+    {
+        return 'database';
+    }
+
+    public function check(): HealthCheckResult
+    {
+        $start = microtime(true);
+
+        try {
+            $this->connection->executeQuery('SELECT 1');
+        } catch (\Throwable $throwable) {
+            $this->logger->error('Health check "database" failed: {message}', [
+                'message' => $throwable->getMessage(),
+                'exception' => $throwable,
+            ]);
+
+            // The caller gets no detail; the reason stays in the logs.
+            return HealthCheckResult::degraded($this->getName(), 'Unable to query the database.');
+        }
+
+        return HealthCheckResult::ok($this->getName(), [
+            'response_time_ms' => round((microtime(true) - $start) * 1000, 1),
+        ]);
+    }
+}

src/Health/Check/IngestFreshnessHealthCheck.php

@@ -0,0 +1,78 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Health\Check;
+
+use App\Health\HealthCheckInterface;
+use App\Health\HealthCheckResult;
+use App\Repository\DetectionResultRepository;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Verifies that detection results are still arriving from the harvester.
+ *
+ * This is the check that catches the failure the dependency checks cannot see
+ * as a whole: a dead harvester, a stalled messenger consumer or a wedged broker
+ * all leave the application itself perfectly able to serve requests while
+ * nothing is actually being ingested.
+ *
+ * lastContact is used rather than createdAt because identical submissions are
+ * deduplicated by content hash and only bump lastContact — a harvester that
+ * keeps reporting unchanged servers is still healthy.
+ */
+readonly class IngestFreshnessHealthCheck implements HealthCheckInterface
+{
+    public function __construct(
+        private DetectionResultRepository $repository,
+        private LoggerInterface $logger,
+        private int $maxAgeSeconds,
+    ) {
+    }
+
+    public function getName(): string
+    {
+        return 'ingest_freshness';
+    }
+
+    public function check(): HealthCheckResult
+    {
+        try {
+            $lastContact = $this->repository->findLastContact();
+        } catch (\Throwable $throwable) {
+            $this->logger->error('Health check "ingest_freshness" failed: {message}', [
+                'message' => $throwable->getMessage(),
+                'exception' => $throwable,
+            ]);
+
+            // Almost certainly the database being down, which the database
+            // check reports separately.
+            return HealthCheckResult::degraded($this->getName(), 'Unable to query the last detection result.');
+        }
+
+        if (!$lastContact instanceof \DateTimeImmutable) {
+            return HealthCheckResult::degraded(
+                $this->getName(),
+                'No detection results have been received yet.',
+                ['max_age_seconds' => $this->maxAgeSeconds]
+            );
+        }
+
+        $ageSeconds = time() - $lastContact->getTimestamp();
+        $details = [
+            'last_contact' => $lastContact->format(\DATE_ATOM),
+            'age_seconds' => $ageSeconds,
+            'max_age_seconds' => $this->maxAgeSeconds,
+        ];
+
+        if ($ageSeconds > $this->maxAgeSeconds) {
+            return HealthCheckResult::degraded(
+                $this->getName(),
+                \sprintf('No detection result received for %d seconds.', $ageSeconds),
+                $details
+            );
+        }
+
+        return HealthCheckResult::ok($this->getName(), $details);
+    }
+}