diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 680f66b8..c3ecdd68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,18 @@ jobs: needs: changes if: ${{ needs.changes.outputs.server == 'true' }} runs-on: ubuntu-latest + services: + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 5 + env: + REDIS_TEST_URL: redis://127.0.0.1:6379/0 defaults: run: working-directory: apps/server diff --git a/AGENTS.md b/AGENTS.md index 3a3b8249..34cc98bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ The aggregate script assumes app dependencies are already installed and intentio ## Current limitations -- Server tests compile TypeScript but do not provide behavioral coverage. +- Server behavioral coverage is currently limited to Redis namespacing and scoped reset behavior. - Collection Android and web have tracked pre-existing issues. - Dashboard analysis has one baseline informational failure and little test coverage. - Website check and formatting commands fail at the migration baseline; production build needs private environment configuration. diff --git a/apps/server/.env.example b/apps/server/.env.example index a98f9c6a..ee6c7821 100644 --- a/apps/server/.env.example +++ b/apps/server/.env.example @@ -8,8 +8,10 @@ BASE_URL=http://localhost:3000 DATABASE_URL="postgresql://username:password@localhost:5432/postgres" -# Connection string for Redis database -REDIS_URL="redis://username:password@localhost:6379/0" +# Connection string for the local Lovat Redis logical database +REDIS_URL="redis://username:password@localhost:6379/1" +# Namespace for every Lovat-owned Redis key +REDIS_KEY_PREFIX="lovat:local:" # Domain identifying the Auth0 tenant of the project. Leave unchanged for production set of accounts. AUTH0_DOMAIN=lovat.us.auth0.com diff --git a/apps/server/AGENTS.md b/apps/server/AGENTS.md index d14d8d50..c7595804 100644 --- a/apps/server/AGENTS.md +++ b/apps/server/AGENTS.md @@ -2,7 +2,7 @@ - Use Node.js 22.20.0 from `.nvmrc` and install with `npm ci`. - Run `npm run build`, `npm test`, and `npm run lint` for code changes. -- `npm test` currently compiles TypeScript; do not describe it as behavioral coverage. +- `npm test` compiles TypeScript and runs the server's behavioral tests; describe coverage only for the behaviors those tests exercise. - Prisma schema and migrations live under `prisma/`. Use reviewed migrations for schema changes and verify database URLs before destructive commands. - PostgreSQL and Redis are required for normal local startup. Do not weaken authentication or source-team visibility to simplify testing. - Treat report events, authentication headers, deep links, and analysis responses as cross-app contracts. diff --git a/apps/server/README.md b/apps/server/README.md index 168cf209..cafa75ba 100644 --- a/apps/server/README.md +++ b/apps/server/README.md @@ -21,6 +21,14 @@ npm run dev Fill the local `.env` without committing it. PostgreSQL and Redis are required to start the service; external integrations are optional only when the exercised code path permits. +The example configuration uses Redis logical database 1 and prefixes every key with `lovat:local:`. Keep both settings isolated from other applications. To remove only Lovat-owned cache entries, run: + +```bash +npm run cache:reset +``` + +The reset scans the Lovat namespace twice around metadata deletion. This prevents a successful reset from leaving an untracked cache entry when a request writes during the operation. Stop the server first when practical; a failed reset should be retried after correcting the reported error. + ## Checks ```bash @@ -29,7 +37,7 @@ npm test npm run lint ``` -`npm test` currently verifies TypeScript compilation rather than behavioral coverage. +`npm test` compiles the server and runs its behavioral tests. ## Optional database restore diff --git a/apps/server/package.json b/apps/server/package.json index 5b390e47..99acb0e5 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -7,7 +7,8 @@ "dev": "NODE_ENV=development tsx watch src/server.ts", "build": "tsc", "start": "node dist/src/server.js --omit=dev", - "test": "tsc", + "cache:reset": "tsx src/resetCache.ts", + "test": "npm run build && node --test dist/test/*.test.js", "lint": "eslint .", "format": "prettier --write ." }, diff --git a/apps/server/src/lib/clearCache.ts b/apps/server/src/lib/clearCache.ts index 37b401de..7daf83f9 100644 --- a/apps/server/src/lib/clearCache.ts +++ b/apps/server/src/lib/clearCache.ts @@ -1,12 +1,16 @@ import prismaClient from "../prismaClient.js"; import { kv } from "../redisClient.js"; +import { resetCacheState } from "./resetCacheState.js"; -export const clearCache = async () => { - await prismaClient.cachedAnalysis.deleteMany(); - - await kv.flush(); +export const resetCache = async (): Promise => { + const deletedRedisKeys = await resetCacheState({ + resetRedis: () => kv.reset(), + deleteMetadata: async () => { + await prismaClient.cachedAnalysis.deleteMany(); + }, + }); - console.log("Cache cleared"); + console.log(`Lovat cache reset (${deletedRedisKeys} Redis keys deleted)`); }; export const invalidateCache = async ( diff --git a/apps/server/src/lib/loadEnvironmentFile.ts b/apps/server/src/lib/loadEnvironmentFile.ts new file mode 100644 index 00000000..bf1826e0 --- /dev/null +++ b/apps/server/src/lib/loadEnvironmentFile.ts @@ -0,0 +1,12 @@ +import { config } from "dotenv"; + +export const loadEnvironmentFile = (path?: string): void => { + const result = config({ path, quiet: true }); + + if ( + result.error && + (result.error as NodeJS.ErrnoException).code !== "ENOENT" + ) { + throw result.error; + } +}; diff --git a/apps/server/src/lib/namespacedKv.ts b/apps/server/src/lib/namespacedKv.ts new file mode 100644 index 00000000..966c0354 --- /dev/null +++ b/apps/server/src/lib/namespacedKv.ts @@ -0,0 +1,92 @@ +const RESET_BATCH_SIZE = 100; + +type RedisInteger = `${number}` | number; +type RedisKey = Buffer | string; + +export type RedisKeyValueClient = { + set(key: string, data: string, options?: { EX: number }): Promise; + get(key: string): Promise; + del(keys: RedisKey[] | RedisKey): Promise; + incr(key: string): Promise; + expire(key: string, seconds: number): Promise; + scanIterator(options: { + MATCH: string; + COUNT: number; + }): AsyncIterable; +}; + +export type NamespacedKv = { + set(key: string, data: string): Promise; + get(key: string): Promise; + del(keys: string[] | string): Promise; + incr(key: string): Promise; + exp(key: string, seconds: number): Promise; + setEx(key: string, data: string, seconds: number): Promise; + reset(): Promise; +}; + +export const createNamespacedKv = ( + redis: RedisKeyValueClient, + keyPrefix: string, +): NamespacedKv => { + if (["*", "?", "[", "]", "\\"].some((char) => keyPrefix.includes(char))) { + throw new Error("Redis key prefix cannot contain Redis glob characters"); + } + + const namespacedKey = (key: string): string => `${keyPrefix}${key}`; + + const set = async (key: string, data: string): Promise => { + return await redis.set(namespacedKey(key), data); + }; + + const get = async (key: string): Promise => { + return await redis.get(namespacedKey(key)); + }; + + const del = async (keys: string[] | string): Promise => { + const namespacedKeys = Array.isArray(keys) + ? keys.map(namespacedKey) + : namespacedKey(keys); + + return Number(await redis.del(namespacedKeys)); + }; + + const incr = async (key: string): Promise => { + return Number(await redis.incr(namespacedKey(key))); + }; + + const exp = async (key: string, seconds: number): Promise => { + return await redis.expire(namespacedKey(key), seconds); + }; + + const setEx = async ( + key: string, + data: string, + seconds: number, + ): Promise => { + return await redis.set(namespacedKey(key), data, { EX: seconds }); + }; + + const reset = async (): Promise => { + let deleted = 0; + + for await (const keys of redis.scanIterator({ + MATCH: `${keyPrefix}*`, + COUNT: RESET_BATCH_SIZE, + })) { + if (keys.length > 0) deleted += Number(await redis.del(keys)); + } + + return deleted; + }; + + return { + set, + get, + del, + incr, + exp, + setEx, + reset, + }; +}; diff --git a/apps/server/src/lib/resetCacheState.ts b/apps/server/src/lib/resetCacheState.ts new file mode 100644 index 00000000..da1cb409 --- /dev/null +++ b/apps/server/src/lib/resetCacheState.ts @@ -0,0 +1,15 @@ +export type CacheResetDependencies = { + resetRedis(): Promise; + deleteMetadata(): Promise; +}; + +export const resetCacheState = async ({ + resetRedis, + deleteMetadata, +}: CacheResetDependencies): Promise => { + let deletedRedisKeys = await resetRedis(); + await deleteMetadata(); + deletedRedisKeys += await resetRedis(); + + return deletedRedisKeys; +}; diff --git a/apps/server/src/lib/startServer.ts b/apps/server/src/lib/startServer.ts new file mode 100644 index 00000000..f4e4256b --- /dev/null +++ b/apps/server/src/lib/startServer.ts @@ -0,0 +1,15 @@ +export type ServerStartupDependencies = { + initializeCache(): Promise; + scheduleJobs(): Promise; + listen(): void; +}; + +export const startServer = async ({ + initializeCache, + scheduleJobs, + listen, +}: ServerStartupDependencies): Promise => { + await initializeCache(); + await scheduleJobs(); + listen(); +}; diff --git a/apps/server/src/redisClient.ts b/apps/server/src/redisClient.ts index 697bf056..3da4c64e 100644 --- a/apps/server/src/redisClient.ts +++ b/apps/server/src/redisClient.ts @@ -1,55 +1,23 @@ import { createClient } from "redis"; +import { createNamespacedKv } from "./lib/namespacedKv.js"; -const redis = createClient({ url: process.env.REDIS_URL }) - .on("error", (err) => console.log("Redis Client Error", err)) - .connect(); +const DEFAULT_KEY_PREFIX = "lovat:"; -const set = async ( - key: string, - data: string, -): ReturnType["set"]> => { - return await (await redis).set(key, data); +const getKeyPrefix = (): string => { + return process.env.REDIS_KEY_PREFIX?.trim() || DEFAULT_KEY_PREFIX; }; -const get = async (key: string): ReturnType["get"]> => { - return await (await redis).get(key); -}; - -const del = async ( - key: string[] | string, -): ReturnType["del"]> => { - return await (await redis).del(key); -}; - -const flush = async (): ReturnType["flushDb"]> => { - return await (await redis).flushDb(); -}; - -const incr = async (key: string): ReturnType["incr"]> => { - return await (await redis).incr(key); -}; +const redisClient = createClient({ url: process.env.REDIS_URL }).on( + "error", + (err) => console.log("Redis Client Error", err), +); -const exp = async ( - key: string, - exp: number, -): ReturnType["expire"]> => { - return await (await redis).expire(key, exp); -}; +export const kv = createNamespacedKv(redisClient, getKeyPrefix()); -const setEx = async ( - key: string, - data: string, - seconds: number, -): ReturnType["set"]> => { - return await (await redis).set(key, data, { EX: seconds }); +export const connectRedis = async (): Promise => { + if (!redisClient.isOpen) await redisClient.connect(); }; -export const kv = { - set, - get, - del, - flush, - incr, - exp, - setEx, +export const closeRedis = async (): Promise => { + if (redisClient.isOpen) await redisClient.close(); }; diff --git a/apps/server/src/resetCache.ts b/apps/server/src/resetCache.ts new file mode 100644 index 00000000..c5c635d5 --- /dev/null +++ b/apps/server/src/resetCache.ts @@ -0,0 +1,24 @@ +import { loadEnvironmentFile } from "./lib/loadEnvironmentFile.js"; + +loadEnvironmentFile(); + +const [ + { resetCache }, + { default: prismaClient }, + { closeRedis, connectRedis }, +] = await Promise.all([ + import("./lib/clearCache.js"), + import("./prismaClient.js"), + import("./redisClient.js"), +]); + +try { + await connectRedis(); + await resetCache(); +} finally { + try { + await prismaClient.$disconnect(); + } finally { + await closeRedis(); + } +} diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index f3430ed7..b8aa9997 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,12 +1,16 @@ import { app } from "./app.js"; +import { startServer } from "./lib/startServer.js"; import scheduleJobs from "./lib/scheduleJobs.js"; -import { clearCache } from "./lib/clearCache.js"; +import { connectRedis } from "./redisClient.js"; const port = process.env.PORT || 3000; -await scheduleJobs(); -await clearCache(); - -app.listen(port, () => { - console.log(`Server running on :${port}`); +await startServer({ + initializeCache: connectRedis, + scheduleJobs, + listen: () => { + app.listen(port, () => { + console.log(`Server running on :${port}`); + }); + }, }); diff --git a/apps/server/test/redisClient.test.ts b/apps/server/test/redisClient.test.ts new file mode 100644 index 00000000..965a1e87 --- /dev/null +++ b/apps/server/test/redisClient.test.ts @@ -0,0 +1,114 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { createNamespacedKv } from "../src/lib/namespacedKv.js"; + +class InMemoryRedis { + readonly values = new Map(); + readonly expirations = new Map(); + + async set( + key: string, + value: string, + options?: { EX: number }, + ): Promise<"OK"> { + this.values.set(key, value); + if (options) this.expirations.set(key, options.EX); + return "OK"; + } + + async get(key: string): Promise { + return this.values.get(key) ?? null; + } + + async del(keys: Buffer | string | (Buffer | string)[]): Promise { + const keysToDelete = Array.isArray(keys) ? keys : [keys]; + let deleted = 0; + + for (const key of keysToDelete) { + const stringKey = key.toString(); + if (this.values.delete(stringKey)) deleted += 1; + this.expirations.delete(stringKey); + } + + return deleted; + } + + async incr(key: string): Promise { + const value = Number(this.values.get(key) ?? "0") + 1; + this.values.set(key, String(value)); + return value; + } + + async expire(key: string, seconds: number): Promise { + if (!this.values.has(key)) return 0; + this.expirations.set(key, seconds); + return 1; + } + + async *scanIterator(options: { + MATCH: string; + COUNT: number; + }): AsyncGenerator { + const prefix = options.MATCH.slice(0, -1); + const keys = [...this.values.keys()].filter((key) => + key.startsWith(prefix), + ); + + if (keys.length > 0) yield keys; + } +} + +test("all key operations stay inside the Lovat namespace", async () => { + const redis = new InMemoryRedis(); + const kv = createNamespacedKv(redis, "lovat:test:"); + + await kv.set("analysis:one", "cached"); + await kv.setEx("posthog:alias:one", "1", 60); + await kv.incr("auth:apikey:one:rate"); + await kv.exp("auth:apikey:one:rate", 3); + + assert.equal(redis.values.get("lovat:test:analysis:one"), "cached"); + assert.equal(redis.values.get("lovat:test:posthog:alias:one"), "1"); + assert.equal(redis.values.get("lovat:test:auth:apikey:one:rate"), "1"); + assert.equal(redis.expirations.get("lovat:test:posthog:alias:one"), 60); + assert.equal(redis.expirations.get("lovat:test:auth:apikey:one:rate"), 3); + assert.equal(await kv.get("analysis:one"), "cached"); + + await kv.del(["analysis:one", "posthog:alias:one"]); + + assert.equal(redis.values.has("lovat:test:analysis:one"), false); + assert.equal(redis.values.has("lovat:test:posthog:alias:one"), false); +}); + +test("reset removes only namespaced keys", async () => { + const redis = new InMemoryRedis(); + const kv = createNamespacedKv(redis, "lovat:test:"); + redis.values.set("foreign:session", "keep"); + redis.values.set("another-app:cache", "keep"); + await kv.set("analysis:one", "remove"); + await kv.set("auth:team:8033", "remove"); + + await kv.reset(); + + assert.deepEqual([...redis.values.entries()].sort(), [ + ["another-app:cache", "keep"], + ["foreign:session", "keep"], + ]); +}); + +test("namespaces reject Redis glob metacharacters", () => { + const redis = new InMemoryRedis(); + + for (const prefix of [ + "lovat:*:", + "lovat:?:", + "lovat:[test]:", + String.raw`lovat:\test:`, + ]) { + assert.throws( + () => createNamespacedKv(redis, prefix), + /cannot contain Redis glob characters/, + ); + } +}); diff --git a/apps/server/test/resetCache.test.ts b/apps/server/test/resetCache.test.ts new file mode 100644 index 00000000..ed27547d --- /dev/null +++ b/apps/server/test/resetCache.test.ts @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { loadEnvironmentFile } from "../src/lib/loadEnvironmentFile.js"; +import { resetCacheState } from "../src/lib/resetCacheState.js"; + +test("cache reset removes Redis keys before database metadata", async () => { + const calls: string[] = []; + let resetCount = 0; + + const deleted = await resetCacheState({ + resetRedis: async () => { + calls.push("redis"); + resetCount += 1; + return resetCount; + }, + deleteMetadata: async () => { + calls.push("database"); + }, + }); + + assert.equal(deleted, 3); + assert.deepEqual(calls, ["redis", "database", "redis"]); +}); + +test("cache reset keeps database metadata when Redis reset fails", async () => { + const calls: string[] = []; + + await assert.rejects( + resetCacheState({ + resetRedis: async () => { + calls.push("redis"); + throw new Error("Redis unavailable"); + }, + deleteMetadata: async () => { + calls.push("database"); + }, + }), + /Redis unavailable/, + ); + + assert.deepEqual(calls, ["redis"]); +}); + +test("cache reset removes entries written during metadata deletion", async () => { + const cacheKeys = new Set(["existing"]); + const metadataKeys = new Set(["existing"]); + + await resetCacheState({ + resetRedis: async () => { + const deleted = cacheKeys.size; + cacheKeys.clear(); + return deleted; + }, + deleteMetadata: async () => { + cacheKeys.add("concurrent"); + metadataKeys.add("concurrent"); + metadataKeys.clear(); + }, + }); + + assert.deepEqual([...cacheKeys], []); + assert.deepEqual([...metadataKeys], []); +}); + +test("explicit reset loads configuration from an environment file", async () => { + const directory = await mkdtemp(join(tmpdir(), "lovat-cache-reset-")); + const environmentPath = join(directory, ".env"); + const previousValue = process.env.REDIS_KEY_PREFIX; + + try { + await writeFile(environmentPath, "REDIS_KEY_PREFIX=lovat:file-test:\n"); + delete process.env.REDIS_KEY_PREFIX; + + loadEnvironmentFile(environmentPath); + + assert.equal(process.env.REDIS_KEY_PREFIX, "lovat:file-test:"); + } finally { + if (previousValue === undefined) delete process.env.REDIS_KEY_PREFIX; + else process.env.REDIS_KEY_PREFIX = previousValue; + + await rm(directory, { recursive: true }); + } +}); diff --git a/apps/server/test/startServer.test.ts b/apps/server/test/startServer.test.ts new file mode 100644 index 00000000..11585838 --- /dev/null +++ b/apps/server/test/startServer.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createClient } from "redis"; + +import { startServer } from "../src/lib/startServer.js"; + +const redisTestUrl = process.env.REDIS_TEST_URL; + +test( + "normal startup leaves pre-existing Redis keys untouched", + { skip: redisTestUrl ? false : "REDIS_TEST_URL is not configured" }, + async () => { + assert.ok(redisTestUrl); + + process.env.REDIS_URL = redisTestUrl; + process.env.REDIS_KEY_PREFIX = "lovat:test:startup:"; + + const { closeRedis, connectRedis, kv } = + await import("../src/redisClient.js"); + const rawRedis = createClient({ url: redisTestUrl }); + const foreignKey = "foreign:test:startup-preservation"; + const lovatKey = "lovat:test:startup:analysis:existing"; + let jobsScheduled = false; + let listening = false; + + await rawRedis.connect(); + + try { + await rawRedis.set(foreignKey, "keep"); + await rawRedis.set(lovatKey, "keep"); + + await startServer({ + initializeCache: connectRedis, + scheduleJobs: async () => { + jobsScheduled = true; + }, + listen: () => { + listening = true; + }, + }); + + assert.equal(jobsScheduled, true); + assert.equal(listening, true); + assert.equal(await rawRedis.get(foreignKey), "keep"); + assert.equal(await rawRedis.get(lovatKey), "keep"); + } finally { + await kv.reset(); + await rawRedis.del(foreignKey); + await closeRedis(); + await rawRedis.close(); + } + }, +); diff --git a/apps/server/tsconfig.json b/apps/server/tsconfig.json index 35b6a094..b35fd5b3 100644 --- a/apps/server/tsconfig.json +++ b/apps/server/tsconfig.json @@ -9,5 +9,5 @@ "outDir": "dist" }, - "include": ["src/**/*"] + "include": ["src/**/*", "test/**/*"] }