From d74b219fca0cd1526ec57bb8d788a74f96f5a718 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 12:40:15 +0200 Subject: [PATCH 01/18] feat(video): add resumable R2 multipart lifecycle Replace the Stream direct-upload lifecycle with durable R2 multipart sessions, immutable queued submissions, active-project database constraints, and confirmed retirement that retains stored objects. Add browser-side part streaming and persisted resume metadata, remove attachment promotion and Stream job endpoints, and cover authorization, concurrency, expiry, idempotency, and retention in local tests. --- migrations/0007_r2_video_lifecycle.sql | 122 ++++ package-lock.json | 174 ----- package.json | 1 - src/app/queries/videos.ts | 70 +- src/app/video/ProjectVideoPanel.tsx | 120 ++-- src/app/video/upload.ts | 180 +++-- src/shared/videos.ts | 86 +-- src/worker/db/schema.ts | 4 +- src/worker/index.ts | 5 +- src/worker/repositories/administration.ts | 2 +- src/worker/routes/stream-webhook.ts | 119 ---- src/worker/routes/video-jobs.ts | 152 ---- src/worker/routes/videos.ts | 250 ++++--- src/worker/services/videos.ts | 830 ++++++++++++---------- test/env.d.ts | 1 + test/migration/migration.test.ts | 14 + test/video-ui/video-ui.test.tsx | 123 +++- test/video/video.test.ts | 569 +++++++-------- wrangler.jsonc | 4 + 19 files changed, 1344 insertions(+), 1482 deletions(-) create mode 100644 migrations/0007_r2_video_lifecycle.sql delete mode 100644 src/worker/routes/stream-webhook.ts delete mode 100644 src/worker/routes/video-jobs.ts diff --git a/migrations/0007_r2_video_lifecycle.sql b/migrations/0007_r2_video_lifecycle.sql new file mode 100644 index 0000000..8590dd4 --- /dev/null +++ b/migrations/0007_r2_video_lifecycle.sql @@ -0,0 +1,122 @@ +PRAGMA foreign_keys = OFF; + +ALTER TABLE project_videos RENAME TO legacy_project_videos; + +CREATE TABLE project_videos ( + id TEXT PRIMARY KEY NOT NULL, + project_id TEXT NOT NULL REFERENCES projects(id) ON UPDATE CASCADE ON DELETE CASCADE, + original_name TEXT NOT NULL CHECK (length(trim(original_name)) BETWEEN 1 AND 255), + content_type TEXT, + size_bytes INTEGER CHECK (size_bytes IS NULL OR size_bytes BETWEEN 1 AND 5368709120), + original_r2_key TEXT UNIQUE, + processed_r2_key TEXT UNIQUE, + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'retired')), + processing_attempt INTEGER NOT NULL DEFAULT 1 CHECK (processing_attempt >= 1), + duration_seconds REAL CHECK (duration_seconds IS NULL OR duration_seconds >= 0), + loudness_lufs REAL, + gain_db REAL CHECK (gain_db IS NULL OR gain_db BETWEEN -12 AND 12), + error_message TEXT, + retired_at TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK ((status = 'retired') = (retired_at IS NOT NULL)), + CHECK (status = 'retired' OR (original_r2_key IS NOT NULL AND size_bytes IS NOT NULL)) +) STRICT; + +INSERT INTO project_videos ( + id, project_id, original_name, status, processing_attempt, + duration_seconds, loudness_lufs, gain_db, error_message, + retired_at, created_at, updated_at +) +SELECT + id, project_id, 'Legacy Stream video', 'retired', 1, + duration_seconds, loudness_lufs, gain_db, error_message, + updated_at, created_at, updated_at +FROM legacy_project_videos; + +DROP TABLE legacy_project_videos; +DROP TABLE stream_events; + +CREATE UNIQUE INDEX project_videos_active_project_idx + ON project_videos(project_id) WHERE retired_at IS NULL; +CREATE INDEX project_videos_status_idx + ON project_videos(status, updated_at); + +CREATE TABLE video_uploads ( + id TEXT PRIMARY KEY NOT NULL, + video_id TEXT NOT NULL UNIQUE, + project_id TEXT NOT NULL REFERENCES projects(id) ON UPDATE CASCADE ON DELETE CASCADE, + creator_id TEXT NOT NULL REFERENCES users(id) ON UPDATE CASCADE ON DELETE RESTRICT, + r2_upload_id TEXT, + original_r2_key TEXT NOT NULL UNIQUE, + original_name TEXT NOT NULL CHECK (length(trim(original_name)) BETWEEN 1 AND 255), + content_type TEXT, + expected_size_bytes INTEGER NOT NULL CHECK (expected_size_bytes BETWEEN 1 AND 5368709120), + part_size_bytes INTEGER NOT NULL CHECK (part_size_bytes >= 5242880), + status TEXT NOT NULL DEFAULT 'creating' + CHECK (status IN ('creating', 'uploading', 'completing', 'completed', 'aborted', 'expired')), + expires_at TEXT NOT NULL, + completed_at TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (r2_upload_id IS NOT NULL OR status IN ('creating', 'aborted')), + CHECK ((status = 'completed') = (completed_at IS NOT NULL)) +) STRICT; + +CREATE UNIQUE INDEX video_uploads_active_project_idx + ON video_uploads(project_id) + WHERE status IN ('creating', 'uploading', 'completing'); +CREATE INDEX video_uploads_expiry_idx + ON video_uploads(status, expires_at); + +CREATE TRIGGER video_uploads_reject_active_submission +BEFORE INSERT ON video_uploads +WHEN NEW.status IN ('creating', 'uploading', 'completing') + AND EXISTS ( + SELECT 1 FROM project_videos + WHERE project_id = NEW.project_id AND retired_at IS NULL + ) +BEGIN + SELECT RAISE(ABORT, 'active project video exists'); +END; + +CREATE TRIGGER project_videos_reject_active_upload +BEFORE INSERT ON project_videos +WHEN NEW.retired_at IS NULL + AND EXISTS ( + SELECT 1 FROM video_uploads + WHERE project_id = NEW.project_id + AND status IN ('creating', 'uploading', 'completing') + ) +BEGIN + SELECT RAISE(ABORT, 'active project upload exists'); +END; + +CREATE TABLE video_upload_parts ( + upload_id TEXT NOT NULL REFERENCES video_uploads(id) ON UPDATE CASCADE ON DELETE CASCADE, + part_number INTEGER NOT NULL CHECK (part_number BETWEEN 1 AND 10000), + etag TEXT NOT NULL CHECK (length(etag) > 0), + size_bytes INTEGER NOT NULL CHECK (size_bytes > 0), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (upload_id, part_number) +) STRICT, WITHOUT ROWID; + +CREATE TABLE video_processing_attempts ( + video_id TEXT NOT NULL REFERENCES project_videos(id) ON UPDATE CASCADE ON DELETE CASCADE, + attempt INTEGER NOT NULL CHECK (attempt >= 1), + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')), + output_r2_key TEXT, + error_message TEXT, + started_at TEXT, + finished_at TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (video_id, attempt) +) STRICT, WITHOUT ROWID; + +CREATE INDEX video_processing_attempts_status_idx + ON video_processing_attempts(status, created_at); + +PRAGMA foreign_keys = ON; diff --git a/package-lock.json b/package-lock.json index afd1860..661d1c6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,6 @@ "react-markdown": "^10.1.0", "remark-breaks": "^4.0.0", "remark-gfm": "^4.0.1", - "tus-js-client": "^4.3.1", "wouter": "^3.10.0" }, "devDependencies": { @@ -3952,12 +3951,6 @@ "dev": true, "license": "MIT" }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "license": "MIT" - }, "node_modules/ccount": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", @@ -4025,15 +4018,6 @@ "dev": true, "license": "MIT" }, - "node_modules/combine-errors": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/combine-errors/-/combine-errors-3.0.3.tgz", - "integrity": "sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==", - "dependencies": { - "custom-error-instance": "2.1.1", - "lodash.uniqby": "4.5.0" - } - }, "node_modules/comma-separated-tokens": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", @@ -4085,12 +4069,6 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, - "node_modules/custom-error-instance": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz", - "integrity": "sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==", - "license": "ISC" - }, "node_modules/data-urls": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", @@ -4334,12 +4312,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "license": "ISC" - }, "node_modules/hast-util-to-jsx-runtime": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", @@ -4528,18 +4500,6 @@ "dev": true, "license": "MIT" }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/jose": { "version": "6.2.8", "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", @@ -4549,12 +4509,6 @@ "url": "https://github.com/sponsors/panva" } }, - "node_modules/js-base64": { - "version": "3.9.2", - "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.2.tgz", - "integrity": "sha512-6zayE8QlUdiweYI6cETD/XBSqFcoCUlufn/29PJR99r82x1yDnIprRca0YvAYpAW+ez0GuQkVBC6xG5QkD7OjA==", - "license": "BSD-3-Clause" - }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -4885,68 +4839,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/lodash._baseiteratee": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz", - "integrity": "sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==", - "license": "MIT", - "dependencies": { - "lodash._stringtopath": "~4.8.0" - } - }, - "node_modules/lodash._basetostring": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz", - "integrity": "sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==", - "license": "MIT" - }, - "node_modules/lodash._baseuniq": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz", - "integrity": "sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==", - "license": "MIT", - "dependencies": { - "lodash._createset": "~4.0.0", - "lodash._root": "~3.0.0" - } - }, - "node_modules/lodash._createset": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/lodash._createset/-/lodash._createset-4.0.3.tgz", - "integrity": "sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==", - "license": "MIT" - }, - "node_modules/lodash._root": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz", - "integrity": "sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==", - "license": "MIT" - }, - "node_modules/lodash._stringtopath": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz", - "integrity": "sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==", - "license": "MIT", - "dependencies": { - "lodash._basetostring": "~4.12.0" - } - }, - "node_modules/lodash.throttle": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", - "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", - "license": "MIT" - }, - "node_modules/lodash.uniqby": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz", - "integrity": "sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==", - "license": "MIT", - "dependencies": { - "lodash._baseiteratee": "~4.7.0", - "lodash._baseuniq": "~4.6.0" - } - }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -6166,17 +6058,6 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, - "node_modules/proper-lockfile": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", - "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.4", - "retry": "^0.12.0", - "signal-exit": "^3.0.2" - } - }, "node_modules/property-information": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.2.0.tgz", @@ -6197,12 +6078,6 @@ "node": ">=6" } }, - "node_modules/querystringify": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", - "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", - "license": "MIT" - }, "node_modules/react": { "version": "19.2.8", "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", @@ -6348,21 +6223,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/requires-port": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "license": "MIT" - }, - "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/rolldown": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.2.tgz", @@ -6494,12 +6354,6 @@ "dev": true, "license": "ISC" }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "license": "ISC" - }, "node_modules/sirv": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.2.tgz", @@ -6758,24 +6612,6 @@ "fsevents": "~2.3.3" } }, - "node_modules/tus-js-client": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/tus-js-client/-/tus-js-client-4.3.1.tgz", - "integrity": "sha512-ZLeYmjrkaU1fUsKbIi8JML52uAocjEZtBx4DKjRrqzrZa0O4MYwT6db+oqePlspV+FxXJAyFBc/L5gwUi2OFsg==", - "license": "MIT", - "dependencies": { - "buffer-from": "^1.1.2", - "combine-errors": "^3.0.3", - "is-stream": "^2.0.0", - "js-base64": "^3.7.2", - "lodash.throttle": "^4.1.1", - "proper-lockfile": "^4.1.2", - "url-parse": "^1.5.7" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/typescript": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", @@ -6925,16 +6761,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/url-parse": { - "version": "1.5.10", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", - "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", - "license": "MIT", - "dependencies": { - "querystringify": "^2.1.1", - "requires-port": "^1.0.0" - } - }, "node_modules/use-sync-external-store": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", diff --git a/package.json b/package.json index 065f73a..8eb1349 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "react-markdown": "^10.1.0", "remark-breaks": "^4.0.0", "remark-gfm": "^4.0.1", - "tus-js-client": "^4.3.1", "wouter": "^3.10.0" }, "devDependencies": { diff --git a/src/app/queries/videos.ts b/src/app/queries/videos.ts index 4d038f6..f3fbbe6 100644 --- a/src/app/queries/videos.ts +++ b/src/app/queries/videos.ts @@ -4,10 +4,10 @@ import type { DirectUploadResponse, PlaybackResponse, PlaylistResponse, - ProjectVideo, ProjectVideoResponse, } from '../../shared/videos'; -import {apiRequest, jsonRequest} from './api'; +import {clearResumeRecord, persistResumeRecord, readResumeRecord} from '../video/upload'; +import {apiRequest, ApiError, jsonRequest} from './api'; export function useProjectVideo(projectId: string) { return useQuery({ @@ -24,31 +24,45 @@ export function useProjectVideo(projectId: string) { } export function useCreateVideoUpload(projectId: string) { - const cache = useQueryClient(); return useMutation({ - mutationFn: (file: File) => - apiRequest( - `/projects/${encodeURIComponent(projectId)}/video/upload`, - jsonRequest('POST', {fileName: file.name, fileSize: file.size}), - ), - onSuccess: ({video}) => - cache.setQueryData( - ['project-video', projectId], - (current) => ({ - video, - streamMode: current?.streamMode ?? 'fake', - }), - ), + mutationFn: (file: File) => prepareVideoUpload(projectId, file), }); } +async function prepareVideoUpload(projectId: string, file: File) { + const resume = readResumeRecord(projectId, file); + if (resume) { + try { + const existing = await apiRequest( + `/projects/${encodeURIComponent(projectId)}/video/upload/${encodeURIComponent(resume.uploadId)}`, + ); + persistResumeRecord(file, existing.upload); + return existing; + } catch (error) { + if (!(error instanceof ApiError) || ![404, 409].includes(error.status)) throw error; + clearResumeRecord(projectId, file); + } + } + const created = await apiRequest( + `/projects/${encodeURIComponent(projectId)}/video/upload`, + jsonRequest('POST', { + fileName: file.name, + fileSize: file.size, + contentType: file.type || null, + }), + ); + persistResumeRecord(file, created.upload); + return created; +} + export function useDeleteVideo(projectId: string) { const cache = useQueryClient(); return useMutation({ mutationFn: () => - apiRequest(`/projects/${encodeURIComponent(projectId)}/video`, { - method: 'DELETE', - }), + apiRequest( + `/projects/${encodeURIComponent(projectId)}/video`, + jsonRequest('DELETE', {confirmed: true}), + ), onSuccess: () => cache.setQueryData( ['project-video', projectId], @@ -60,24 +74,6 @@ export function useDeleteVideo(projectId: string) { }); } -export function useRetryVideo(projectId: string) { - const cache = useQueryClient(); - return useMutation({ - mutationFn: (videoId: string) => - apiRequest<{video: ProjectVideo}>(`/videos/${encodeURIComponent(videoId)}/retry`, { - method: 'POST', - }), - onSuccess: ({video}) => - cache.setQueryData( - ['project-video', projectId], - (current) => ({ - video, - streamMode: current?.streamMode ?? 'fake', - }), - ), - }); -} - export function usePlaylist(yearId: string) { return useQuery({ queryKey: ['video-playlist', yearId], diff --git a/src/app/video/ProjectVideoPanel.tsx b/src/app/video/ProjectVideoPanel.tsx index bef1c72..82f965f 100644 --- a/src/app/video/ProjectVideoPanel.tsx +++ b/src/app/video/ProjectVideoPanel.tsx @@ -1,9 +1,10 @@ -import {useRef, useState, type ChangeEvent} from 'react'; +import {useRef, useState, type ChangeEvent, type DragEvent} from 'react'; +import {useQueryClient} from '@tanstack/react-query'; import {Link} from 'wouter'; import type {ProjectVideo, StreamMode} from '../../shared/videos'; -import {useCreateVideoUpload, useDeleteVideo, useRetryVideo} from '../queries/videos'; -import {createTusUpload, type ResumableUpload, type UploadSnapshot} from './upload'; +import {useCreateVideoUpload, useDeleteVideo} from '../queries/videos'; +import {createMultipartUpload, type ResumableUpload, type UploadSnapshot} from './upload'; const INITIAL_UPLOAD: UploadSnapshot = { phase: 'uploading', @@ -12,44 +13,43 @@ const INITIAL_UPLOAD: UploadSnapshot = { error: null, }; -export function ProjectVideoPanel({ - projectId, - yearId, - video, - canManage, - loading = false, - streamMode, - uploadFactory = createTusUpload, -}: { +type UploadFactory = typeof createMultipartUpload; + +export function ProjectVideoPanel(props: { projectId: string; yearId: string; video: ProjectVideo | null; canManage: boolean; loading?: boolean; streamMode?: StreamMode; - uploadFactory?: typeof createTusUpload; + uploadFactory?: UploadFactory; }) { + const { + projectId, + yearId, + video, + canManage, + loading = false, + uploadFactory = createMultipartUpload, + } = props; + const cache = useQueryClient(); const createUpload = useCreateVideoUpload(projectId); const remove = useDeleteVideo(projectId); - const retry = useRetryVideo(projectId); const controller = useRef(null); const [upload, setUpload] = useState(null); const [error, setError] = useState(null); - function selectFile(event: ChangeEvent) { - const file = event.target.files?.[0]; - event.target.value = ''; - if (!file) return; + function beginUpload(file: File) { setError(null); setUpload({...INITIAL_UPLOAD, bytesTotal: file.size}); createUpload.mutate(file, { onSuccess: (result) => { - const next = uploadFactory( - file, - result.upload.url, - result.upload.chunkSize, - setUpload, - ); + const next = uploadFactory(file, result.upload, (snapshot) => { + setUpload(snapshot); + if (snapshot.phase === 'complete') { + void cache.invalidateQueries({queryKey: ['project-video', projectId]}); + } + }); controller.current = next; next.start(); }, @@ -60,9 +60,20 @@ export function ProjectVideoPanel({ }); } + function selectFile(event: ChangeEvent) { + const file = event.target.files?.[0]; + event.target.value = ''; + if (file) beginUpload(file); + } + + function dropFile(event: DragEvent) { + event.preventDefault(); + const file = event.dataTransfer.files[0]; + if (file) beginUpload(file); + } + const isUploading = upload && upload.phase !== 'complete'; - const disabled = streamMode === 'disabled'; - const actionError = error ?? remove.error?.message ?? retry.error?.message; + const actionError = error ?? remove.error?.message; return (
@@ -71,7 +82,7 @@ export function ProjectVideoPanel({

demo reel

project video

- {video?.status === 'ready' && streamMode === 'real' && ( + {video?.status === 'ready' && ( loading video status…

- ) : disabled ? ( -

- video processing is temporarily unavailable. projects, attachments, voting, - awards, and every non-video workflow remain available. -

) : video ? ( ) : ( @@ -137,13 +143,17 @@ export function ProjectVideoPanel({ )} - {canManage && !disabled && !isUploading && ( + {canManage && !isUploading && (
- {(!video || video.status === 'failed') && ( -
@@ -180,9 +185,8 @@ export function ProjectVideoPanel({

)}

- {disabled - ? 'video uploads and playback will appear here after Cloudflare Stream is enabled.' - : 'uploads go directly to Cloudflare Stream using resumable tus. closing this page does not send video bytes through Hackweek.'} + uploads are sent in resumable parts to private R2 storage. completed originals are + retained when a video is retired.

); @@ -190,10 +194,8 @@ export function ProjectVideoPanel({ function VideoStatusCard({video}: {video: ProjectVideo}) { const labels: Record = { - pending_upload: 'waiting for upload', - uploading: 'uploading to Stream', + queued: 'queued for processing', processing: 'processing video', - measuring: 'measuring audio', ready: 'ready to watch', failed: 'needs attention', }; @@ -213,19 +215,17 @@ function VideoStatusCard({video}: {video: ProjectVideo}) { } function statusDetail(status: ProjectVideo['status']) { - if (status === 'uploading') - return 'the resumable upload can continue after an interruption.'; - if (status === 'processing') return 'Stream is preparing protected playback.'; - if (status === 'measuring') return 'loudness is being measured before screening.'; + if (status === 'queued') return 'the immutable original is ready for processing.'; + if (status === 'processing') return 'the uploaded video is being normalized.'; return 'this video is not included in the screening playlist.'; } function uploadLabel(phase: UploadSnapshot['phase']) { return { - uploading: 'uploading directly to Stream', + uploading: 'uploading to private storage', paused: 'upload paused', interrupted: 'upload interrupted', - complete: 'upload complete — processing next', + complete: 'upload complete — processing queued', }[phase]; } diff --git a/src/app/video/upload.ts b/src/app/video/upload.ts index 3312b39..2f5fec3 100644 --- a/src/app/video/upload.ts +++ b/src/app/video/upload.ts @@ -1,4 +1,4 @@ -import {Upload} from 'tus-js-client'; +import type {VideoUploadPart, VideoUploadSession} from '../../shared/videos'; export type UploadPhase = 'uploading' | 'paused' | 'interrupted' | 'complete'; @@ -16,65 +16,163 @@ export interface ResumableUpload { retry(): void; } -export function createTusUpload( +interface ResumeRecord { + projectId: string; + uploadId: string; + fileName: string; + fileSize: number; + lastModified: number; + completedParts: VideoUploadPart[]; +} + +export function createMultipartUpload( file: File, - uploadUrl: string, - chunkSize: number, + session: VideoUploadSession, onChange: (snapshot: UploadSnapshot) => void, ): ResumableUpload { let phase: UploadPhase = 'uploading'; - let bytesSent = 0; + let parts = [...session.completedParts].sort( + (left, right) => left.partNumber - right.partNumber, + ); + let active: AbortController | null = null; + let running = false; let error: string | null = null; - const notify = () => onChange({phase, bytesSent, bytesTotal: file.size, error}); - const upload = new Upload(file, { - uploadUrl, - chunkSize, - retryDelays: [0, 1_000, 3_000, 5_000], - storeFingerprintForResuming: true, - removeFingerprintOnSuccess: true, - metadata: {filename: file.name, filetype: file.type || 'application/octet-stream'}, - onProgress(sent) { - bytesSent = sent; - phase = 'uploading'; - error = null; - notify(); - }, - onError(uploadError) { - phase = 'interrupted'; - error = uploadError.message; - notify(); - }, - onSuccess() { - bytesSent = file.size; + const bytesSent = () => parts.reduce((total, part) => total + part.sizeBytes, 0); + const notify = () => + onChange({phase, bytesSent: bytesSent(), bytesTotal: file.size, error}); + + async function run() { + if (running || phase === 'complete') return; + running = true; + phase = 'uploading'; + error = null; + notify(); + try { + const partCount = Math.ceil(file.size / session.partSize); + for (let partNumber = 1; partNumber <= partCount; partNumber += 1) { + if (parts.some((part) => part.partNumber === partNumber)) continue; + active = new AbortController(); + const start = (partNumber - 1) * session.partSize; + const body = file.slice(start, Math.min(file.size, start + session.partSize)); + const response = await fetch(partUrl(session, partNumber), { + method: 'PUT', + headers: {'Content-Type': 'application/octet-stream'}, + body, + signal: active.signal, + }); + if (!response.ok) throw await responseError(response); + const result = (await response.json()) as {part: VideoUploadPart}; + parts = [...parts, result.part].sort( + (left, right) => left.partNumber - right.partNumber, + ); + persistResumeRecord(file, session, parts); + notify(); + } + + const completed = await fetch(`${uploadUrl(session)}/complete`, { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + parts: parts.map(({partNumber, etag}) => ({partNumber, etag})), + }), + }); + if (!completed.ok) throw await responseError(completed); + clearResumeRecord(session.projectId, file); phase = 'complete'; error = null; notify(); - }, - }); + } catch (uploadError) { + if (!active?.signal.aborted) { + phase = 'interrupted'; + error = uploadError instanceof Error ? uploadError.message : 'Upload interrupted'; + notify(); + } + } finally { + active = null; + running = false; + } + } return { start() { - phase = 'uploading'; - error = null; - notify(); - upload.start(); + void run(); }, async pause() { - await upload.abort(); phase = 'paused'; + active?.abort(); notify(); }, resume() { - phase = 'uploading'; - error = null; - notify(); - upload.start(); + void run(); }, retry() { - phase = 'uploading'; - error = null; - notify(); - upload.start(); + void run(); }, }; } + +export function resumeStorageKey(projectId: string, file: File) { + return `hackweek:video-upload:${projectId}:${file.name}:${file.size}:${file.lastModified}`; +} + +export function readResumeRecord(projectId: string, file: File) { + if (typeof localStorage === 'undefined') return null; + const value = localStorage.getItem(resumeStorageKey(projectId, file)); + if (!value) return null; + try { + const record = JSON.parse(value) as ResumeRecord; + if ( + record.projectId !== projectId || + record.fileName !== file.name || + record.fileSize !== file.size || + record.lastModified !== file.lastModified || + !record.uploadId + ) { + clearResumeRecord(projectId, file); + return null; + } + return record; + } catch { + clearResumeRecord(projectId, file); + return null; + } +} + +export function persistResumeRecord( + file: File, + session: VideoUploadSession, + completedParts = session.completedParts, +) { + if (typeof localStorage === 'undefined') return; + const record: ResumeRecord = { + projectId: session.projectId, + uploadId: session.uploadId, + fileName: file.name, + fileSize: file.size, + lastModified: file.lastModified, + completedParts, + }; + localStorage.setItem(resumeStorageKey(session.projectId, file), JSON.stringify(record)); +} + +export function clearResumeRecord(projectId: string, file: File) { + if (typeof localStorage === 'undefined') return; + localStorage.removeItem(resumeStorageKey(projectId, file)); +} + +function uploadUrl(session: VideoUploadSession) { + return `/api/projects/${encodeURIComponent(session.projectId)}/video/upload/${encodeURIComponent(session.uploadId)}`; +} + +function partUrl(session: VideoUploadSession, partNumber: number) { + return `${uploadUrl(session)}/parts/${partNumber}`; +} + +async function responseError(response: Response) { + try { + const value = (await response.json()) as {error?: {message?: string}}; + return new Error(value.error?.message || `Upload failed (${response.status})`); + } catch { + return new Error(`Upload failed (${response.status})`); + } +} diff --git a/src/shared/videos.ts b/src/shared/videos.ts index a9295f8..6c8510c 100644 --- a/src/shared/videos.ts +++ b/src/shared/videos.ts @@ -1,48 +1,64 @@ export type StreamMode = 'disabled' | 'fake' | 'real'; -export type VideoStatus = - | 'pending_upload' - | 'uploading' - | 'processing' - | 'measuring' - | 'ready' - | 'failed'; - -export type VideoFailureStage = 'upload' | 'stream' | 'measurement'; +export type VideoStatus = 'queued' | 'processing' | 'ready' | 'failed'; +export type VideoFailureStage = 'processing'; export type ArchiveStatus = 'pending' | 'archiving' | 'archived' | 'failed'; +export type VideoUploadStatus = + | 'creating' + | 'uploading' + | 'completing' + | 'completed' + | 'aborted' + | 'expired'; export interface ProjectVideo { id: string; projectId: string; - streamUid: string | null; - sourceMediaId: string | null; status: VideoStatus; + originalName: string; + contentType: string | null; + sizeBytes: number; durationSeconds: number | null; loudnessLufs: number | null; gainDb: number | null; errorMessage: string | null; failureStage: VideoFailureStage | null; - archiveStatus: ArchiveStatus; - archiveError: string | null; + processingAttempt: number; + createdAt: string; +} + +export interface VideoUploadPart { + partNumber: number; + etag: string; + sizeBytes: number; +} + +export interface VideoUploadSession { + uploadId: string; + videoId: string; + projectId: string; + fileName: string; + contentType: string | null; + fileSize: number; + partSize: number; + expiresAt: string; + status: VideoUploadStatus; + completedParts: VideoUploadPart[]; } export interface DirectUploadRequest { fileName: string; fileSize: number; + contentType: string | null; } export interface DirectUploadResponse { - video: ProjectVideo; - upload: { - protocol: 'tus'; - url: string; - expiresAt: string; - chunkSize: number; - }; + video: ProjectVideo | null; + upload: VideoUploadSession; } -export interface HistoricalPromotionRequest { - sourceMediaId: string; +export interface CompleteVideoUploadRequest { + parts: Array<{partNumber: number; etag: string}>; } export interface PlaybackResponse { @@ -69,29 +85,3 @@ export interface PlaylistResponse { videos: PlaylistItem[]; streamMode: StreamMode; } - -export interface MeasurementQueueItem { - videoId: string; - projectId: string; - downloadUrl: string; -} - -export interface MeasurementQueueResponse { - videos: MeasurementQueueItem[]; -} - -export interface MeasurementResultRequest { - loudnessLufs: number; - durationSeconds: number; -} - -export interface ArchiveQueueItem { - videoId: string; - projectId: string; - fileName: string; - downloadUrl: string; -} - -export interface ArchiveQueueResponse { - videos: ArchiveQueueItem[]; -} diff --git a/src/worker/db/schema.ts b/src/worker/db/schema.ts index e4dfc73..84ac216 100644 --- a/src/worker/db/schema.ts +++ b/src/worker/db/schema.ts @@ -12,6 +12,8 @@ export const tableNames = [ 'awards', 'media', 'project_videos', + 'video_uploads', + 'video_upload_parts', + 'video_processing_attempts', 'screening_order', - 'stream_events', ] as const; diff --git a/src/worker/index.ts b/src/worker/index.ts index 7890e35..da008b0 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -11,13 +11,12 @@ import {groupsRoutes} from './routes/groups'; import {mediaRoutes} from './routes/media'; import {projectsRoutes} from './routes/projects'; import {sessionRoutes} from './routes/session'; -import {streamWebhookRoutes} from './routes/stream-webhook'; -import {videoJobRoutes} from './routes/video-jobs'; import {projectVideoRoutes, videosRoutes} from './routes/videos'; import {votesRoutes} from './routes/votes'; import {yearsRoutes} from './routes/years'; export interface VideoBindings { + VIDEOS: R2Bucket; STREAM_MODE?: string; STREAM_ACCOUNT_ID?: string; STREAM_API_TOKEN?: string; @@ -39,8 +38,6 @@ export type WorkerEnv = { const app = new Hono(); app.get('/api/health', (c) => c.json({ok: true})); -app.route('/api/stream-webhook', streamWebhookRoutes); -app.route('/api/video-jobs', videoJobRoutes); app.route('/api/auth', authRoutes); app.use('/api/*', authenticateRequest()); diff --git a/src/worker/repositories/administration.ts b/src/worker/repositories/administration.ts index 6ed6a52..52ea4f1 100644 --- a/src/worker/repositories/administration.ts +++ b/src/worker/repositories/administration.ts @@ -235,7 +235,7 @@ export async function getAdminYear( db .prepare( `SELECT p.id, p.name, pv.status video_status FROM projects p - LEFT JOIN project_videos pv ON pv.project_id = p.id + LEFT JOIN project_videos pv ON pv.project_id = p.id AND pv.retired_at IS NULL WHERE p.year_id = ? AND p.kind = 'project' AND p.status = 'active' ORDER BY p.name COLLATE NOCASE, p.id`, ) diff --git a/src/worker/routes/stream-webhook.ts b/src/worker/routes/stream-webhook.ts deleted file mode 100644 index 5977ce3..0000000 --- a/src/worker/routes/stream-webhook.ts +++ /dev/null @@ -1,119 +0,0 @@ -import {Hono} from 'hono'; - -import type {WorkerEnv} from '../index'; -import {processStreamWebhook} from '../services/videos'; - -const MAX_CLOCK_SKEW_SECONDS = 5 * 60; - -export const streamWebhookRoutes = new Hono(); - -streamWebhookRoutes.post('/', async (c) => { - const secret = c.env.STREAM_WEBHOOK_SECRET?.trim(); - if (!secret) return c.json({error: 'Webhook is not configured'}, 503); - const rawBody = await c.req.text(); - const signature = c.req.header('Webhook-Signature'); - if (!(await verifyStreamWebhook(rawBody, signature, secret))) { - return c.json({error: 'Invalid webhook signature'}, 401); - } - - const payload = parseWebhook(rawBody); - if (!payload) return c.json({error: 'Invalid webhook payload'}, 400); - const result = await processStreamWebhook(c.env.DB, payload); - return c.json(result); -}); - -export async function verifyStreamWebhook( - body: string, - header: string | undefined, - secret: string, - nowSeconds = Math.floor(Date.now() / 1000), -) { - if (!header) return false; - const fields = new Map( - header.split(',').map((part) => { - const separator = part.indexOf('='); - return separator < 1 - ? ['', ''] - : [part.slice(0, separator).trim(), part.slice(separator + 1).trim()]; - }), - ); - const timestampText = fields.get('time'); - const actualHex = fields.get('sig1'); - const timestamp = Number(timestampText); - if ( - !timestampText || - !Number.isInteger(timestamp) || - Math.abs(nowSeconds - timestamp) > MAX_CLOCK_SKEW_SECONDS || - !actualHex || - !/^[a-f0-9]{64}$/i.test(actualHex) - ) { - return false; - } - const key = await crypto.subtle.importKey( - 'raw', - new TextEncoder().encode(secret), - {name: 'HMAC', hash: 'SHA-256'}, - false, - ['sign'], - ); - const digest = await crypto.subtle.sign( - 'HMAC', - key, - new TextEncoder().encode(`${timestampText}.${body}`), - ); - return timingSafeHex(hex(digest), actualHex.toLowerCase()); -} - -function parseWebhook(body: string) { - try { - const value = JSON.parse(body) as Record; - const status = value.status as Record | undefined; - if ( - typeof value.uid !== 'string' || - !value.uid || - !status || - typeof status.state !== 'string' - ) { - return null; - } - const ready = - value.readyToStream === true && - status.state === 'ready' && - Number(status.pctComplete) === 100; - const modified = typeof value.modified === 'string' ? value.modified : ''; - const eventType = ready ? 'ready' : `failed:${status.state}`; - return { - eventId: `${value.uid}:${modified}:${eventType}`, - streamUid: value.uid, - eventType, - ready, - durationSeconds: - typeof value.duration === 'number' && Number.isFinite(value.duration) - ? value.duration - : null, - errorMessage: - typeof status.errorReasonText === 'string' && status.errorReasonText - ? status.errorReasonText - : typeof status.errorReasonCode === 'string' && status.errorReasonCode - ? status.errorReasonCode - : null, - }; - } catch { - return null; - } -} - -function hex(value: ArrayBuffer) { - return [...new Uint8Array(value)] - .map((byte) => byte.toString(16).padStart(2, '0')) - .join(''); -} - -function timingSafeHex(left: string, right: string) { - if (left.length !== right.length) return false; - let difference = 0; - for (let index = 0; index < left.length; index += 1) { - difference |= left.charCodeAt(index) ^ right.charCodeAt(index); - } - return difference === 0; -} diff --git a/src/worker/routes/video-jobs.ts b/src/worker/routes/video-jobs.ts deleted file mode 100644 index feaf9cf..0000000 --- a/src/worker/routes/video-jobs.ts +++ /dev/null @@ -1,152 +0,0 @@ -import {Hono, type Context} from 'hono'; - -import type { - ArchiveQueueResponse, - MeasurementQueueResponse, - MeasurementResultRequest, -} from '../../shared/videos'; -import {streamGateway} from '../integrations/stream'; -import type {WorkerEnv} from '../index'; -import {requireVideoService} from '../middleware/service-auth'; -import {errorResponse, ServiceError} from '../services/errors'; -import { - listArchiveQueue, - listMeasurementQueue, - markMeasurementFailure, - recordArchiveResult, - recordMeasurement, -} from '../services/videos'; - -export const videoJobRoutes = new Hono(); -videoJobRoutes.use('*', requireVideoService); - -videoJobRoutes.get('/measurements', async (c) => { - try { - const response: MeasurementQueueResponse = { - videos: await listMeasurementQueue( - c.env.DB, - streamGateway(c.env), - deliveryHost(c.env), - ), - }; - return c.json(response, 200, {'Cache-Control': 'private, no-store'}); - } catch (error) { - return respondError(c, error); - } -}); - -videoJobRoutes.post('/measurements/:videoId', async (c) => { - try { - const input = parseMeasurement(await c.req.json()); - return c.json({ - video: await recordMeasurement(c.env.DB, c.req.param('videoId'), input), - }); - } catch (error) { - return respondError(c, error); - } -}); - -videoJobRoutes.post('/measurements/:videoId/failure', async (c) => { - try { - const message = parseFailure(await c.req.json()); - await markMeasurementFailure(c.env.DB, c.req.param('videoId'), message); - return c.body(null, 204); - } catch (error) { - return respondError(c, error); - } -}); - -videoJobRoutes.get('/archives', async (c) => { - try { - const response: ArchiveQueueResponse = { - videos: await listArchiveQueue(c.env.DB, streamGateway(c.env), deliveryHost(c.env)), - }; - return c.json(response, 200, {'Cache-Control': 'private, no-store'}); - } catch (error) { - return respondError(c, error); - } -}); - -videoJobRoutes.post('/archives/:videoId', async (c) => { - try { - const input = parseArchive(await c.req.json()); - return c.json({ - video: await recordArchiveResult( - c.env.DB, - c.req.param('videoId'), - input.status, - input.error, - ), - }); - } catch (error) { - return respondError(c, error); - } -}); - -function parseMeasurement(value: unknown): MeasurementResultRequest { - if (!value || typeof value !== 'object') invalid('Request body must be an object'); - const input = value as Record; - if ( - typeof input.loudnessLufs !== 'number' || - !Number.isFinite(input.loudnessLufs) || - input.loudnessLufs < -100 || - input.loudnessLufs > 20 - ) { - invalid('Integrated loudness must be a finite LUFS value'); - } - if ( - typeof input.durationSeconds !== 'number' || - !Number.isFinite(input.durationSeconds) || - input.durationSeconds <= 0 || - input.durationSeconds > 24 * 60 * 60 - ) { - invalid('Duration must be a positive finite number'); - } - return { - loudnessLufs: input.loudnessLufs, - durationSeconds: input.durationSeconds, - }; -} - -function parseFailure(value: unknown) { - const message = - value && typeof value === 'object' ? (value as Record).error : null; - if (typeof message !== 'string' || !message.trim()) - invalid('Failure error is required'); - return message.trim().slice(0, 500); -} - -function parseArchive(value: unknown) { - if (!value || typeof value !== 'object') invalid('Request body must be an object'); - const input = value as Record; - if (input.status !== 'archived' && input.status !== 'failed') { - invalid('Archive status must be archived or failed'); - } - if ( - input.status === 'failed' && - (typeof input.error !== 'string' || !input.error.trim()) - ) { - invalid('Archive failure requires an error'); - } - return { - status: input.status, - error: typeof input.error === 'string' ? input.error.trim() : null, - } as const; -} - -function deliveryHost(env: WorkerEnv['Bindings']) { - const host = env.STREAM_DELIVERY_HOST?.trim(); - if (!host || host.includes('/') || host.includes('*')) { - throw new ServiceError('AUTH_CONFIG_INVALID', 'Stream delivery host is invalid', 500); - } - return host; -} - -function invalid(message: string): never { - throw new ServiceError('VALIDATION_FAILED', message, 400); -} - -function respondError(c: Context, error: unknown) { - const result = errorResponse(error); - return c.json(result.response, result.status); -} diff --git a/src/worker/routes/videos.ts b/src/worker/routes/videos.ts index c65dfaf..c50c99f 100644 --- a/src/worker/routes/videos.ts +++ b/src/worker/routes/videos.ts @@ -1,39 +1,32 @@ import {Hono, type Context} from 'hono'; import type { + CompleteVideoUploadRequest, DirectUploadRequest, DirectUploadResponse, - HistoricalPromotionRequest, - PlaybackResponse, - PlaylistResponse, + ProjectVideoResponse, } from '../../shared/videos'; -import { - FakeHistoricalVideoSource, - R2HistoricalVideoSource, -} from '../integrations/historical-source'; -import {streamGateway, streamMode} from '../integrations/stream'; +import {streamMode} from '../integrations/stream'; import type {WorkerEnv} from '../index'; import {errorResponse, ServiceError} from '../services/errors'; import { - createDirectUpload, - deleteProjectVideo, + abortVideoUpload, + completeVideoUpload, + createMultipartVideoUpload, getProjectVideo, - issuePlayback, - listPlaylist, + getVideoUpload, MAX_VIDEO_BYTES, - promoteHistoricalVideo, - retryVideo, - TUS_CHUNK_SIZE, + retireProjectVideo, + uploadVideoPart, } from '../services/videos'; export const videosRoutes = new Hono(); +export const projectVideoRoutes = new Hono(); -videosRoutes.get('/playlist', async (c) => { +projectVideoRoutes.get('/:projectId/video', async (c) => { try { - const year = c.req.query('year'); - if (!year) throw new ServiceError('VALIDATION_FAILED', 'Year is required', 400); - const response: PlaylistResponse = { - videos: await listPlaylist(c.env.DB, year), + const response: ProjectVideoResponse = { + video: await getProjectVideo(c.env.DB, c.req.param('projectId')), streamMode: streamMode(c.env), }; return c.json(response); @@ -42,82 +35,101 @@ videosRoutes.get('/playlist', async (c) => { } }); -videosRoutes.get('/:videoId/playback', async (c) => { +projectVideoRoutes.post('/:projectId/video/upload', async (c) => { try { - const response: PlaybackResponse = await issuePlayback( + const input = parseUpload(await c.req.json()); + const response: DirectUploadResponse = await createMultipartVideoUpload( c.env.DB, - streamGateway(c.env), - c.req.param('videoId'), - deliveryHost(c.env), + c.env.VIDEOS, + c.req.param('projectId'), + c.get('user'), + input, ); - return c.json(response, 200, {'Cache-Control': 'private, no-store'}); - } catch (error) { - return respondError(c, error); - } -}); - -videosRoutes.post('/:videoId/retry', async (c) => { - try { - return c.json({ - video: await retryVideo(c.env.DB, c.req.param('videoId'), c.get('user')), - }); + return c.json(response, 201, {'Cache-Control': 'private, no-store'}); } catch (error) { return respondError(c, error); } }); -export const projectVideoRoutes = new Hono(); - -projectVideoRoutes.get('/:projectId/video', async (c) => { +projectVideoRoutes.get('/:projectId/video/upload/:uploadId', async (c) => { try { - return c.json({ - video: await getProjectVideo(c.env.DB, c.req.param('projectId')), - streamMode: streamMode(c.env), - }); + const response: DirectUploadResponse = await getVideoUpload( + c.env.DB, + c.env.VIDEOS, + c.req.param('projectId'), + c.req.param('uploadId'), + c.get('user'), + ); + return c.json(response, 200, {'Cache-Control': 'private, no-store'}); } catch (error) { return respondError(c, error); } }); -projectVideoRoutes.post('/:projectId/video/upload', async (c) => { +projectVideoRoutes.put( + '/:projectId/video/upload/:uploadId/parts/:partNumber', + async (c) => { + try { + const partNumber = parsePartNumber(c.req.param('partNumber')); + const contentLength = parseContentLength(c.req.header('Content-Length')); + const body = c.req.raw.body; + if (!body) invalid('Video part body is required'); + const part = await uploadVideoPart( + c.env.DB, + c.env.VIDEOS, + c.req.param('projectId'), + c.req.param('uploadId'), + partNumber, + contentLength, + body, + c.get('user'), + ); + return c.json({part}, 200, {'Cache-Control': 'private, no-store'}); + } catch (error) { + return respondError(c, error); + } + }, +); + +projectVideoRoutes.post('/:projectId/video/upload/:uploadId/complete', async (c) => { try { - const input = parseUpload(await c.req.json()); - const result = await createDirectUpload( + const input = parseCompletion(await c.req.json()); + const video = await completeVideoUpload( c.env.DB, - streamGateway(c.env), + c.env.VIDEOS, c.req.param('projectId'), + c.req.param('uploadId'), + input.parts, c.get('user'), - input, - uploadOrigin(c.env), ); - const response: DirectUploadResponse = { - video: result.video, - upload: { - protocol: 'tus', - url: result.upload.uploadUrl, - expiresAt: result.upload.expiresAt.toISOString(), - chunkSize: TUS_CHUNK_SIZE, - }, - }; - return c.json(response, 201, {'Cache-Control': 'private, no-store'}); + return c.json({video}, 200, {'Cache-Control': 'private, no-store'}); } catch (error) { return respondError(c, error); } }); -projectVideoRoutes.post('/:projectId/video/promote', async (c) => { +projectVideoRoutes.post('/:projectId/video/promote', (c) => + c.json( + { + error: { + code: 'NOT_FOUND' as const, + message: 'Attachment promotion is not supported', + }, + }, + 404, + ), +); + +projectVideoRoutes.delete('/:projectId/video/upload/:uploadId', async (c) => { try { - const input = parsePromotion(await c.req.json()); - const video = await promoteHistoricalVideo( + await abortVideoUpload( c.env.DB, - streamGateway(c.env), - historicalSource(c.env), + c.env.VIDEOS, c.req.param('projectId'), - input.sourceMediaId, + c.req.param('uploadId'), c.get('user'), - uploadOrigin(c.env), ); - return c.json({video}, 201); + return c.body(null, 204); } catch (error) { return respondError(c, error); } @@ -125,11 +137,12 @@ projectVideoRoutes.post('/:projectId/video/promote', async (c) => { projectVideoRoutes.delete('/:projectId/video', async (c) => { try { - await deleteProjectVideo( + const confirmed = parseRetirement(await c.req.json()); + await retireProjectVideo( c.env.DB, - streamGateway(c.env), c.req.param('projectId'), c.get('user'), + confirmed, ); return c.body(null, 204); } catch (error) { @@ -159,66 +172,67 @@ function parseUpload(value: unknown): DirectUploadRequest { ) { invalid(`File size must be between 1 and ${MAX_VIDEO_BYTES} bytes`); } - return {fileName: input.fileName.trim(), fileSize: input.fileSize}; + if ( + input.contentType !== null && + (typeof input.contentType !== 'string' || + !/^video\/[a-zA-Z0-9!#$&^_.+-]{1,100}$/.test(input.contentType)) + ) { + invalid('Content type must be a valid video media type'); + } + return { + fileName: input.fileName.trim(), + fileSize: input.fileSize, + contentType: input.contentType, + }; } -function parsePromotion(value: unknown): HistoricalPromotionRequest { +function parseCompletion(value: unknown): CompleteVideoUploadRequest { if (!value || typeof value !== 'object') invalid('Request body must be an object'); - const sourceMediaId = (value as Record).sourceMediaId; - if (typeof sourceMediaId !== 'string' || !sourceMediaId.trim()) { - invalid('Source media id is required'); + const parts = (value as Record).parts; + if (!Array.isArray(parts) || parts.length === 0 || parts.length > 10_000) { + invalid('Completed parts are required'); + } + const parsed = parts.map((part) => { + if (!part || typeof part !== 'object') invalid('Completed part is invalid'); + const record = part as Record; + if ( + !Number.isInteger(record.partNumber) || + (record.partNumber as number) < 1 || + (record.partNumber as number) > 10_000 || + typeof record.etag !== 'string' || + !record.etag || + record.etag.length > 256 + ) { + invalid('Completed part is invalid'); + } + return {partNumber: record.partNumber as number, etag: record.etag}; + }); + parsed.sort((left, right) => left.partNumber - right.partNumber); + if (new Set(parsed.map((part) => part.partNumber)).size !== parsed.length) { + invalid('Completed part numbers must be unique'); } - return {sourceMediaId: sourceMediaId.trim()}; + return {parts: parsed}; } -function uploadOrigin(env: WorkerEnv['Bindings']) { - const origin = env.STREAM_ALLOWED_ORIGIN?.trim(); - if (!origin || origin.includes('/') || origin.includes('*')) { - throw new ServiceError( - 'AUTH_CONFIG_INVALID', - 'Stream allowed origin is invalid', - 500, - ); - } - return origin; +function parseRetirement(value: unknown) { + if (!value || typeof value !== 'object') invalid('Request body must be an object'); + return (value as Record).confirmed === true; } -function deliveryHost(env: WorkerEnv['Bindings']) { - const host = env.STREAM_DELIVERY_HOST?.trim(); - if (!host || host.includes('/') || host.includes('*')) { - throw new ServiceError('AUTH_CONFIG_INVALID', 'Stream delivery host is invalid', 500); +function parsePartNumber(value: string) { + if (!/^\d+$/.test(value)) invalid('Part number is invalid'); + const number = Number(value); + if (!Number.isInteger(number) || number < 1 || number > 10_000) { + invalid('Part number is invalid'); } - return host; + return number; } -function historicalSource(env: WorkerEnv['Bindings']) { - const mode = streamMode(env); - if (mode === 'disabled') { - throw new ServiceError( - 'SERVICE_UNAVAILABLE', - 'Video processing is temporarily unavailable', - 503, - ); - } - if (mode === 'fake') return new FakeHistoricalVideoSource(); - if ( - !env.R2_ACCOUNT_ID?.trim() || - !env.R2_BUCKET_NAME?.trim() || - !env.R2_ACCESS_KEY_ID?.trim() || - !env.R2_SECRET_ACCESS_KEY?.trim() - ) { - throw new ServiceError( - 'AUTH_CONFIG_INVALID', - 'Historical Stream promotion is not configured', - 500, - ); - } - return new R2HistoricalVideoSource( - env.R2_ACCOUNT_ID, - env.R2_BUCKET_NAME, - env.R2_ACCESS_KEY_ID, - env.R2_SECRET_ACCESS_KEY, - ); +function parseContentLength(value: string | undefined) { + if (!value || !/^\d+$/.test(value)) invalid('Content-Length is required'); + const length = Number(value); + if (!Number.isSafeInteger(length) || length <= 0) invalid('Content-Length is invalid'); + return length; } function invalid(message: string): never { diff --git a/src/worker/services/videos.ts b/src/worker/services/videos.ts index 3b2d617..4392b45 100644 --- a/src/worker/services/videos.ts +++ b/src/worker/services/videos.ts @@ -1,459 +1,433 @@ import type {SessionUser} from '../../shared/api'; import type { - ArchiveQueueItem, - ArchiveStatus, - MeasurementQueueItem, - PlaylistItem, ProjectVideo, - VideoFailureStage, - VideoStatus, + VideoUploadPart, + VideoUploadSession, } from '../../shared/videos'; -import type {HistoricalVideoSource} from '../integrations/historical-source'; -import type {StreamGateway} from '../integrations/stream'; +import {currentYearIdSql, effectiveYearFlags} from '../repositories/years'; import {ServiceError} from './errors'; export const MAX_VIDEO_BYTES = 5 * 1024 * 1024 * 1024; -export const MAX_VIDEO_DURATION_SECONDS = 10 * 60; -export const TUS_CHUNK_SIZE = 50 * 1024 * 1024; -export const UPLOAD_EXPIRY_MINUTES = 30; -export const PLAYBACK_EXPIRY_MINUTES = 15; -export const SERVICE_DOWNLOAD_EXPIRY_MINUTES = 15; +export const VIDEO_PART_SIZE = 50 * 1024 * 1024; +export const UPLOAD_EXPIRY_MINUTES = 24 * 60; interface VideoRow { id: string; project_id: string; - stream_uid: string | null; - source_media_id: string | null; + original_name: string; + content_type: string | null; + size_bytes: number; status: string; + processing_attempt: number; duration_seconds: number | null; loudness_lufs: number | null; gain_db: number | null; error_message: string | null; - failure_stage: string | null; - archive_status: string; - archive_error: string | null; + created_at: string; +} + +interface UploadRow { + id: string; + video_id: string; + project_id: string; + creator_id: string; + r2_upload_id: string | null; + original_r2_key: string; + original_name: string; + content_type: string | null; + expected_size_bytes: number; + part_size_bytes: number; + status: VideoUploadSession['status']; + expires_at: string; } interface ProjectAuthorizationRow { id: string; - name: string; + year_id: string; creator_id: string; kind: string; status: string; + voting_enabled: number; + submissions_closed: number; + current_year_id: string; is_member: number; } export async function getProjectVideo(db: D1Database, projectId: string) { - const row = await videoByProject(db, projectId); + const row = await db + .prepare(`${videoSelect()} WHERE project_id = ? AND retired_at IS NULL`) + .bind(projectId) + .first(); return row ? mapVideo(row) : null; } -export async function createDirectUpload( +export async function createMultipartVideoUpload( db: D1Database, - gateway: StreamGateway, + bucket: R2Bucket, projectId: string, user: SessionUser, - input: {fileName: string; fileSize: number}, - allowedOrigin: string, + input: {fileName: string; fileSize: number; contentType: string | null}, now = new Date(), ) { await authorizeVideoWrite(db, projectId, user); - const existing = await videoByProject(db, projectId); - if (existing && !(await canReplaceUpload(db, existing, now))) { - throw new ServiceError( - 'CONFLICT', - 'The primary video must fail, expire, or be deleted before it can be replaced', - 409, - ); - } - + const uploadId = crypto.randomUUID(); + const videoId = crypto.randomUUID(); + const originalKey = videoOriginalKey(projectId, videoId, input.fileName); const expiresAt = new Date(now.getTime() + UPLOAD_EXPIRY_MINUTES * 60_000); - const upload = await gateway.createDirectUpload({ - creator: user.id, - fileName: input.fileName, - fileSize: input.fileSize, - maxDurationSeconds: MAX_VIDEO_DURATION_SECONDS, - allowedOrigin, - expiresAt, - }); - const id = existing?.id ?? crypto.randomUUID(); + try { await db .prepare( - `INSERT INTO project_videos - (id, project_id, stream_uid, status, upload_expires_at, - error_message, failure_stage, duration_seconds, loudness_lufs, gain_db, - archive_status, archive_error) - VALUES (?, ?, ?, 'uploading', ?, NULL, NULL, NULL, NULL, NULL, 'pending', NULL) - ON CONFLICT(project_id) DO UPDATE SET - stream_uid = excluded.stream_uid, source_media_id = NULL, status = 'uploading', - upload_expires_at = excluded.upload_expires_at, duration_seconds = NULL, - loudness_lufs = NULL, gain_db = NULL, - error_message = NULL, failure_stage = NULL, archive_status = 'pending', - archive_error = NULL, archived_at = NULL, updated_at = CURRENT_TIMESTAMP`, + `INSERT INTO video_uploads ( + id, video_id, project_id, creator_id, original_r2_key, original_name, + content_type, expected_size_bytes, part_size_bytes, status, expires_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'creating', ?)`, + ) + .bind( + uploadId, + videoId, + projectId, + user.id, + originalKey, + input.fileName, + input.contentType, + input.fileSize, + VIDEO_PART_SIZE, + expiresAt.toISOString(), ) - .bind(id, projectId, upload.uid, expiresAt.toISOString()) .run(); } catch (error) { - await gateway.deleteVideo(upload.uid).catch(() => undefined); + if (isVideoSlotConflict(error)) { + throw new ServiceError( + 'CONFLICT', + 'This project already has an active video or upload', + 409, + ); + } throw error; } - const row = await requireVideoById(db, id); - return {video: mapVideo(row), upload}; -} -export async function promoteHistoricalVideo( - db: D1Database, - gateway: StreamGateway, - historicalSource: HistoricalVideoSource, - projectId: string, - sourceMediaId: string, - user: SessionUser, - allowedOrigin: string, -) { - await authorizeVideoWrite(db, projectId, user); - const existing = await videoByProject(db, projectId); - if (existing && !canReplace(existing.status)) { - throw new ServiceError('CONFLICT', 'This project already has a primary video', 409); - } - const media = await db - .prepare( - `SELECT id, original_name, r2_key, media_type, status - FROM media WHERE id = ? AND project_id = ?`, - ) - .bind(sourceMediaId, projectId) - .first<{ - id: string; - original_name: string; - r2_key: string; - media_type: string | null; - status: string; - }>(); - if (!media || media.status !== 'available' || !media.media_type?.startsWith('video/')) { - throw new ServiceError( - 'VALIDATION_FAILED', - 'Historical promotion requires an available video attachment from this project', - 400, - ); - } - const sourceUrl = await historicalSource.createReadUrl(media.r2_key, 15 * 60); - const streamUid = await gateway.promoteHistoricalVideo({ - creator: user.id, - sourceUrl, - fileName: media.original_name, - allowedOrigin, - }); - const id = existing?.id ?? crypto.randomUUID(); try { + const multipart = await bucket.createMultipartUpload(originalKey, { + httpMetadata: {contentType: input.contentType || 'application/octet-stream'}, + customMetadata: {projectId, videoId, uploadId}, + }); await db .prepare( - `INSERT INTO project_videos - (id, project_id, stream_uid, source_media_id, status, archive_status) - VALUES (?, ?, ?, ?, 'processing', 'pending') - ON CONFLICT(project_id) DO UPDATE SET - stream_uid = excluded.stream_uid, source_media_id = excluded.source_media_id, - status = 'processing', upload_expires_at = NULL, duration_seconds = NULL, - gain_db = NULL, error_message = NULL, failure_stage = NULL, - archive_status = 'pending', archive_error = NULL, archived_at = NULL, - updated_at = CURRENT_TIMESTAMP`, + `UPDATE video_uploads SET r2_upload_id = ?, status = 'uploading', + updated_at = CURRENT_TIMESTAMP WHERE id = ? AND status = 'creating'`, ) - .bind(id, projectId, streamUid, media.id) + .bind(multipart.uploadId, uploadId) .run(); - } catch (error) { - await gateway.deleteVideo(streamUid).catch(() => undefined); - throw error; - } - return mapVideo(await requireVideoById(db, id)); -} - -export async function deleteProjectVideo( - db: D1Database, - gateway: StreamGateway, - projectId: string, - user: SessionUser, -) { - await authorizeVideoWrite(db, projectId, user); - const video = await videoByProject(db, projectId); - if (!video) throw new ServiceError('NOT_FOUND', 'Video not found', 404); - if (video.stream_uid) await gateway.deleteVideo(video.stream_uid); - await db.prepare('DELETE FROM project_videos WHERE id = ?').bind(video.id).run(); -} - -export async function processStreamWebhook( - db: D1Database, - event: { - eventId: string; - streamUid: string; - eventType: string; - ready: boolean; - durationSeconds: number | null; - errorMessage: string | null; - }, -) { - const video = await db - .prepare(`${videoSelect()} WHERE stream_uid = ?`) - .bind(event.streamUid) - .first(); - if (!video) return {handled: false, duplicate: false}; - - const status: VideoStatus = event.ready ? 'measuring' : 'failed'; - const failureStage: VideoFailureStage | null = event.ready ? null : 'stream'; - const inserted = await db - .prepare( - `INSERT INTO stream_events (event_id, stream_uid, event_type) - VALUES (?, ?, ?) ON CONFLICT(event_id) DO NOTHING`, - ) - .bind(event.eventId, event.streamUid, event.eventType) - .run(); - if (!inserted.meta.changes) return {handled: true, duplicate: true}; - try { + } catch { await db .prepare( - `UPDATE project_videos SET status = ?, duration_seconds = COALESCE(?, duration_seconds), - error_message = ?, failure_stage = ?, upload_expires_at = NULL, - updated_at = CURRENT_TIMESTAMP WHERE id = ?`, + `UPDATE video_uploads SET status = 'aborted', updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status = 'creating'`, ) - .bind( - status, - event.durationSeconds, - event.ready ? null : event.errorMessage || 'Stream processing failed', - failureStage, - video.id, - ) - .run(); - } catch (error) { - await db - .prepare('DELETE FROM stream_events WHERE event_id = ?') - .bind(event.eventId) + .bind(uploadId) .run(); - throw error; + throw new ServiceError('STORAGE_FAILED', 'Video upload could not be started', 500); } - return {handled: true, duplicate: false}; -} -export async function listPlaylist(db: D1Database, yearId: string) { - const {results} = await db - .prepare( - `SELECT pv.id video_id, p.id project_id, p.name project_name, - pv.duration_seconds, pv.gain_db, so.position - FROM screening_order so - JOIN projects p ON p.id = so.project_id AND p.status = 'active' - JOIN project_videos pv ON pv.project_id = p.id - WHERE so.year_id = ? AND pv.status = 'ready' - AND pv.duration_seconds IS NOT NULL AND pv.gain_db IS NOT NULL - ORDER BY so.position, p.id`, - ) - .bind(yearId) - .all<{ - video_id: string; - project_id: string; - project_name: string; - duration_seconds: number; - gain_db: number; - position: number; - }>(); - return results.map((row) => ({ - videoId: row.video_id, - projectId: row.project_id, - projectName: row.project_name, - durationSeconds: row.duration_seconds, - gainDb: row.gain_db, - position: row.position, - })); + return getVideoUpload(db, bucket, projectId, uploadId, user, now); } -export async function issuePlayback( +export async function getVideoUpload( db: D1Database, - gateway: StreamGateway, - videoId: string, - deliveryHost: string, + bucket: R2Bucket, + projectId: string, + uploadId: string, + user: SessionUser, now = new Date(), ) { - const video = await requireVideoById(db, videoId); - if (video.status !== 'ready' || !video.stream_uid) { - throw new ServiceError('CONFLICT', 'Video is not ready for playback', 409); + await authorizeVideoWrite(db, projectId, user); + const upload = await requireUpload(db, projectId, uploadId); + if (isExpired(upload, now)) { + if (upload.r2_upload_id) { + await bucket + .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) + .abort() + .catch(() => undefined); + } + await markExpired(db, upload.id); + upload.status = 'expired'; } - const expiresAt = new Date(now.getTime() + PLAYBACK_EXPIRY_MINUTES * 60_000); - const token = await gateway.createPlaybackToken(video.stream_uid, expiresAt); + const video = + upload.status === 'completed' ? await requireVideoById(db, upload.video_id) : null; return { - mode: token.startsWith('fake.') ? ('fake' as const) : ('stream' as const), - manifestUrl: token.startsWith('fake.') - ? null - : `https://${deliveryHost}/${encodeURIComponent(token)}/manifest/video.m3u8`, - expiresAt: expiresAt.toISOString(), + video: video ? mapVideo(video) : null, + upload: await mapUpload(db, upload), }; } -export async function listMeasurementQueue( +export async function uploadVideoPart( db: D1Database, - gateway: StreamGateway, - deliveryHost: string, + bucket: R2Bucket, + projectId: string, + uploadId: string, + partNumber: number, + contentLength: number, + body: ReadableStream, + user: SessionUser, now = new Date(), -) { - const {results} = await db +): Promise { + await authorizeVideoWrite(db, projectId, user); + const upload = await requireUpload(db, projectId, uploadId); + await assertUploadIsWritable(db, bucket, upload, now); + if (upload.status !== 'uploading' || !upload.r2_upload_id) { + throw new ServiceError('CONFLICT', 'Upload is not accepting parts', 409); + } + + const expectedSize = expectedPartSize(upload, partNumber); + if (expectedSize === null || contentLength !== expectedSize) { + throw new ServiceError( + 'VALIDATION_FAILED', + `Part ${partNumber} must contain exactly ${expectedSize ?? 0} bytes`, + 400, + ); + } + + const existing = await db .prepare( - `${videoSelect()} WHERE status = 'measuring' ORDER BY updated_at, id LIMIT 20`, + `SELECT part_number, etag, size_bytes FROM video_upload_parts + WHERE upload_id = ? AND part_number = ?`, ) - .all(); - const queue: MeasurementQueueItem[] = []; - for (const video of results) { - if (!video.stream_uid) continue; - const download = await gateway.ensureDownload(video.stream_uid); - if (download.status === 'error') { - await markMeasurementFailure(db, video.id, 'Stream MP4 generation failed'); - continue; - } - if (download.status !== 'ready') continue; - const expiresAt = new Date(now.getTime() + SERVICE_DOWNLOAD_EXPIRY_MINUTES * 60_000); - const token = await gateway.createDownloadToken(video.stream_uid, expiresAt); - queue.push({ - videoId: video.id, - projectId: video.project_id, - downloadUrl: token.startsWith('fake.') - ? download.url! - : `https://${deliveryHost}/${encodeURIComponent(token)}/downloads/default.mp4`, - }); + .bind(upload.id, partNumber) + .first<{part_number: number; etag: string; size_bytes: number}>(); + if (existing) return mapPart(existing); + + let uploaded: R2UploadedPart; + try { + uploaded = await bucket + .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) + .uploadPart(partNumber, body); + } catch { + throw new ServiceError('STORAGE_FAILED', 'Video part upload failed', 500); } - return queue; -} -export async function recordMeasurement( - db: D1Database, - videoId: string, - input: {loudnessLufs: number; durationSeconds: number}, -) { - const result = await db + await db .prepare( - `UPDATE project_videos SET status = 'ready', duration_seconds = ?, - loudness_lufs = ?, gain_db = ?, error_message = NULL, failure_stage = NULL, - measurement_attempts = measurement_attempts + 1, updated_at = CURRENT_TIMESTAMP - WHERE id = ? AND status = 'measuring'`, - ) - .bind( - input.durationSeconds, - input.loudnessLufs, - loudnessGain(input.loudnessLufs), - videoId, + `INSERT INTO video_upload_parts (upload_id, part_number, etag, size_bytes) + VALUES (?, ?, ?, ?) + ON CONFLICT(upload_id, part_number) DO UPDATE SET + etag = excluded.etag, size_bytes = excluded.size_bytes`, ) + .bind(upload.id, uploaded.partNumber, uploaded.etag, contentLength) .run(); - if (!result.meta.changes) { - throw new ServiceError('CONFLICT', 'Video is not awaiting measurement', 409); - } - return mapVideo(await requireVideoById(db, videoId)); + return {partNumber: uploaded.partNumber, etag: uploaded.etag, sizeBytes: contentLength}; } -export async function markMeasurementFailure( +export async function completeVideoUpload( db: D1Database, - videoId: string, - message: string, + bucket: R2Bucket, + projectId: string, + uploadId: string, + suppliedParts: Array<{partNumber: number; etag: string}>, + user: SessionUser, + now = new Date(), ) { - const result = await db - .prepare( - `UPDATE project_videos SET status = 'failed', failure_stage = 'measurement', - error_message = ?, measurement_attempts = measurement_attempts + 1, - updated_at = CURRENT_TIMESTAMP WHERE id = ? AND status = 'measuring'`, - ) - .bind(message.slice(0, 500), videoId) - .run(); - if (!result.meta.changes) { - throw new ServiceError('CONFLICT', 'Video is not awaiting measurement', 409); + await authorizeVideoWrite(db, projectId, user); + const upload = await requireUpload(db, projectId, uploadId); + if (upload.status === 'completed') { + return mapVideo(await requireVideoById(db, upload.video_id)); } -} - -export async function retryVideo(db: D1Database, videoId: string, user: SessionUser) { - const video = await requireVideoById(db, videoId); - await authorizeVideoWrite(db, video.project_id, user); - if (video.status !== 'failed') { - throw new ServiceError('CONFLICT', 'Only failed videos can be retried', 409); + await assertUploadIsWritable(db, bucket, upload, now); + if (!upload.r2_upload_id || !['uploading', 'completing'].includes(upload.status)) { + throw new ServiceError('CONFLICT', 'Upload cannot be completed', 409); } - if (video.failure_stage === 'measurement' && video.stream_uid) { - await db + + const storedParts = await listStoredParts(db, upload.id); + validateCompletionParts(upload, storedParts, suppliedParts); + + if (upload.status === 'uploading') { + const claimed = await db .prepare( - `UPDATE project_videos SET status = 'measuring', error_message = NULL, - failure_stage = NULL, updated_at = CURRENT_TIMESTAMP WHERE id = ?`, + `UPDATE video_uploads SET status = 'completing', updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status = 'uploading'`, ) - .bind(videoId) + .bind(upload.id) .run(); - return mapVideo(await requireVideoById(db, videoId)); + if (!claimed.meta.changes) { + throw new ServiceError('CONFLICT', 'Upload completion is already in progress', 409); + } + upload.status = 'completing'; } - throw new ServiceError( - 'CONFLICT', - 'Upload and Stream processing failures require a replacement upload', - 409, - ); + + let object = await bucket.head(upload.original_r2_key); + if (!object) { + try { + object = await bucket + .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) + .complete(storedParts.map(({partNumber, etag}) => ({partNumber, etag}))); + } catch { + object = await bucket.head(upload.original_r2_key); + if (!object) { + await db + .prepare( + `UPDATE video_uploads SET status = 'uploading', updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status = 'completing'`, + ) + .bind(upload.id) + .run(); + throw new ServiceError( + 'STORAGE_FAILED', + 'Video upload could not be completed', + 500, + ); + } + } + } + if (object.size !== upload.expected_size_bytes) { + throw new ServiceError('STORAGE_FAILED', 'Completed video size does not match', 500); + } + + try { + await db.batch([ + db + .prepare( + `UPDATE video_uploads SET status = 'completed', completed_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP WHERE id = ? AND status = 'completing'`, + ) + .bind(upload.id), + db + .prepare( + `INSERT INTO project_videos ( + id, project_id, original_name, content_type, size_bytes, original_r2_key, + status, processing_attempt + ) VALUES (?, ?, ?, ?, ?, ?, 'queued', 1)`, + ) + .bind( + upload.video_id, + upload.project_id, + upload.original_name, + upload.content_type, + upload.expected_size_bytes, + upload.original_r2_key, + ), + db + .prepare( + `INSERT INTO video_processing_attempts (video_id, attempt, status) + VALUES (?, 1, 'queued')`, + ) + .bind(upload.video_id), + ]); + } catch (error) { + const existing = await db + .prepare(`${videoSelect()} WHERE id = ?`) + .bind(upload.video_id) + .first(); + if (!existing) throw error; + } + return mapVideo(await requireVideoById(db, upload.video_id)); } -export async function listArchiveQueue( +export async function abortVideoUpload( db: D1Database, - gateway: StreamGateway, - deliveryHost: string, - now = new Date(), + bucket: R2Bucket, + projectId: string, + uploadId: string, + user: SessionUser, ) { - const {results} = await db + await authorizeVideoWrite(db, projectId, user); + const upload = await requireUpload(db, projectId, uploadId); + if (upload.status === 'aborted') return; + if (upload.status === 'expired') { + if (upload.r2_upload_id) { + await bucket + .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) + .abort() + .catch(() => undefined); + } + return; + } + if (upload.status === 'completed') { + throw new ServiceError('CONFLICT', 'Completed video objects cannot be aborted', 409); + } + if (upload.r2_upload_id) { + try { + await bucket + .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) + .abort(); + } catch { + throw new ServiceError('STORAGE_FAILED', 'Video upload could not be aborted', 500); + } + } + await db .prepare( - `SELECT pv.id, pv.project_id, pv.stream_uid, pv.source_media_id, pv.status, - pv.duration_seconds, pv.loudness_lufs, pv.gain_db, pv.error_message, - pv.failure_stage, pv.archive_status, pv.archive_error, p.name project_name - FROM project_videos pv JOIN projects p ON p.id = pv.project_id - WHERE pv.status = 'ready' AND pv.archive_status IN ('pending', 'failed') - ORDER BY pv.updated_at, pv.id LIMIT 20`, + `UPDATE video_uploads SET status = 'aborted', updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status IN ('creating', 'uploading', 'completing')`, ) - .all(); - const queue: ArchiveQueueItem[] = []; - for (const video of results) { - if (!video.stream_uid) continue; - const download = await gateway.ensureDownload(video.stream_uid); - if (download.status !== 'ready') continue; - const expiresAt = new Date(now.getTime() + SERVICE_DOWNLOAD_EXPIRY_MINUTES * 60_000); - const token = await gateway.createDownloadToken(video.stream_uid, expiresAt); - queue.push({ - videoId: video.id, - projectId: video.project_id, - fileName: safeFileName(video.project_name, video.project_id), - downloadUrl: token.startsWith('fake.') - ? download.url! - : `https://${deliveryHost}/${encodeURIComponent(token)}/downloads/default.mp4`, - }); - } - return queue; + .bind(upload.id) + .run(); } -export async function recordArchiveResult( +export async function retireProjectVideo( db: D1Database, - videoId: string, - status: Extract, - error: string | null, + projectId: string, + user: SessionUser, + confirmed: boolean, ) { - const result = await db - .prepare( - `UPDATE project_videos SET archive_status = ?, archive_error = ?, - archived_at = CASE WHEN ? = 'archived' THEN CURRENT_TIMESTAMP ELSE archived_at END, - archive_attempts = archive_attempts + 1, updated_at = CURRENT_TIMESTAMP - WHERE id = ? AND status = 'ready'`, - ) - .bind( - status, - status === 'failed' ? error?.slice(0, 500) || 'Archive failed' : null, - status, - videoId, - ) - .run(); - if (!result.meta.changes) { - throw new ServiceError('CONFLICT', 'Only ready videos can be archived', 409); + if (!confirmed) { + throw new ServiceError( + 'VALIDATION_FAILED', + 'Video retirement must be confirmed', + 400, + ); } - return mapVideo(await requireVideoById(db, videoId)); + await authorizeVideoWrite(db, projectId, user); + const video = await db + .prepare(`${videoSelect()} WHERE project_id = ? AND retired_at IS NULL`) + .bind(projectId) + .first(); + if (!video) throw new ServiceError('NOT_FOUND', 'Video not found', 404); + await db.batch([ + db + .prepare( + `UPDATE project_videos SET status = 'retired', retired_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP WHERE id = ? AND retired_at IS NULL`, + ) + .bind(video.id), + db + .prepare( + `UPDATE video_processing_attempts SET status = 'cancelled', + finished_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP + WHERE video_id = ? AND status IN ('queued', 'running')`, + ) + .bind(video.id), + ]); } -export function loudnessGain(loudnessLufs: number) { - return Math.max(-12, Math.min(12, -16 - loudnessLufs)); +async function assertUploadIsWritable( + db: D1Database, + bucket: R2Bucket, + upload: UploadRow, + now: Date, +) { + if (isExpired(upload, now)) { + if (upload.r2_upload_id) { + await bucket + .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) + .abort() + .catch(() => undefined); + } + await markExpired(db, upload.id); + throw new ServiceError('CONFLICT', 'Upload session has expired', 409); + } + if (upload.status === 'aborted' || upload.status === 'expired') { + throw new ServiceError('CONFLICT', 'Upload session is no longer active', 409); + } } async function authorizeVideoWrite(db: D1Database, projectId: string, user: SessionUser) { const project = await db .prepare( - `SELECT p.id, p.name, p.creator_id, p.kind, p.status, - EXISTS(SELECT 1 FROM project_members pm WHERE pm.project_id = p.id AND pm.user_id = ?) is_member - FROM projects p WHERE p.id = ?`, + `SELECT p.id, p.year_id, p.creator_id, p.kind, p.status, + y.voting_enabled, y.submissions_closed, + ${currentYearIdSql} current_year_id, + EXISTS(SELECT 1 FROM project_members pm + WHERE pm.project_id = p.id AND pm.user_id = ?) is_member + FROM projects p JOIN years y ON y.id = p.year_id WHERE p.id = ?`, ) .bind(user.id, projectId) .first(); @@ -461,7 +435,10 @@ async function authorizeVideoWrite(db: D1Database, projectId: string, user: Sess throw new ServiceError('NOT_FOUND', 'Project not found', 404); } if (project.kind !== 'project') { - throw new ServiceError('VALIDATION_FAILED', 'Ideas cannot have primary videos', 400); + throw new ServiceError('VALIDATION_FAILED', 'Ideas cannot have project videos', 400); + } + if (effectiveYearFlags(project.year_id, project).submissionsClosed) { + throw new ServiceError('AUTH_FORBIDDEN', 'Submissions are closed', 403); } if (user.role !== 'admin' && project.creator_id !== user.id && !project.is_member) { throw new ServiceError( @@ -472,11 +449,18 @@ async function authorizeVideoWrite(db: D1Database, projectId: string, user: Sess } } -function videoByProject(db: D1Database, projectId: string) { - return db - .prepare(`${videoSelect()} WHERE project_id = ?`) - .bind(projectId) - .first(); +async function requireUpload(db: D1Database, projectId: string, uploadId: string) { + const upload = await db + .prepare( + `SELECT id, video_id, project_id, creator_id, r2_upload_id, original_r2_key, + original_name, content_type, expected_size_bytes, part_size_bytes, + status, expires_at + FROM video_uploads WHERE id = ? AND project_id = ?`, + ) + .bind(uploadId, projectId) + .first(); + if (!upload) throw new ServiceError('NOT_FOUND', 'Video upload not found', 404); + return upload; } async function requireVideoById(db: D1Database, videoId: string) { @@ -488,50 +472,126 @@ async function requireVideoById(db: D1Database, videoId: string) { return video; } -function videoSelect() { - return `SELECT id, project_id, stream_uid, source_media_id, status, - duration_seconds, loudness_lufs, gain_db, error_message, failure_stage, - archive_status, archive_error FROM project_videos`; +async function mapUpload(db: D1Database, upload: UploadRow): Promise { + return { + uploadId: upload.id, + videoId: upload.video_id, + projectId: upload.project_id, + fileName: upload.original_name, + contentType: upload.content_type, + fileSize: upload.expected_size_bytes, + partSize: upload.part_size_bytes, + expiresAt: upload.expires_at, + status: upload.status, + completedParts: await listStoredParts(db, upload.id), + }; } function mapVideo(row: VideoRow): ProjectVideo { return { id: row.id, projectId: row.project_id, - streamUid: row.stream_uid, - sourceMediaId: row.source_media_id, - status: row.status as VideoStatus, + status: row.status as ProjectVideo['status'], + originalName: row.original_name, + contentType: row.content_type, + sizeBytes: row.size_bytes, durationSeconds: row.duration_seconds, loudnessLufs: row.loudness_lufs, gainDb: row.gain_db, errorMessage: row.error_message, - failureStage: row.failure_stage as VideoFailureStage | null, - archiveStatus: row.archive_status as ArchiveStatus, - archiveError: row.archive_error, + failureStage: row.status === 'failed' ? 'processing' : null, + processingAttempt: row.processing_attempt, + createdAt: row.created_at, }; } -function canReplace(status: string) { - return status === 'failed' || status === 'pending_upload'; +function videoSelect() { + return `SELECT id, project_id, original_name, content_type, size_bytes, status, + processing_attempt, duration_seconds, loudness_lufs, gain_db, error_message, + created_at FROM project_videos`; } -async function canReplaceUpload(db: D1Database, video: VideoRow, now: Date) { - if (canReplace(video.status)) return true; - if (video.status !== 'uploading') return false; - const row = await db - .prepare('SELECT upload_expires_at FROM project_videos WHERE id = ?') - .bind(video.id) - .first<{upload_expires_at: string | null}>(); - return Boolean( - row?.upload_expires_at && Date.parse(row.upload_expires_at) <= now.getTime(), +async function listStoredParts(db: D1Database, uploadId: string) { + const {results} = await db + .prepare( + `SELECT part_number, etag, size_bytes FROM video_upload_parts + WHERE upload_id = ? ORDER BY part_number`, + ) + .bind(uploadId) + .all<{part_number: number; etag: string; size_bytes: number}>(); + return results.map(mapPart); +} + +function mapPart(row: {part_number: number; etag: string; size_bytes: number}) { + return {partNumber: row.part_number, etag: row.etag, sizeBytes: row.size_bytes}; +} + +function validateCompletionParts( + upload: UploadRow, + stored: VideoUploadPart[], + supplied: Array<{partNumber: number; etag: string}>, +) { + const count = Math.ceil(upload.expected_size_bytes / upload.part_size_bytes); + if (stored.length !== count || supplied.length !== count) { + throw new ServiceError('VALIDATION_FAILED', 'Every video part must be uploaded', 400); + } + for (let index = 0; index < count; index += 1) { + const expectedNumber = index + 1; + const saved = stored[index]; + const provided = supplied[index]; + if ( + saved.partNumber !== expectedNumber || + provided?.partNumber !== expectedNumber || + provided.etag !== saved.etag || + saved.sizeBytes !== expectedPartSize(upload, expectedNumber) + ) { + throw new ServiceError( + 'VALIDATION_FAILED', + 'Completed video parts are invalid', + 400, + ); + } + } +} + +function expectedPartSize(upload: UploadRow, partNumber: number) { + const count = Math.ceil(upload.expected_size_bytes / upload.part_size_bytes); + if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > count) return null; + if (partNumber < count) return upload.part_size_bytes; + return upload.expected_size_bytes - upload.part_size_bytes * (count - 1); +} + +function isExpired(upload: UploadRow, now: Date) { + return ( + ['creating', 'uploading', 'completing'].includes(upload.status) && + Date.parse(upload.expires_at) <= now.getTime() + ); +} + +function markExpired(db: D1Database, uploadId: string) { + return db + .prepare( + `UPDATE video_uploads SET status = 'expired', updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status IN ('creating', 'uploading', 'completing')`, + ) + .bind(uploadId) + .run(); +} + +function isVideoSlotConflict(error: unknown) { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes('video_uploads_active_project_idx') || + message.includes('active project video exists') || + message.includes('UNIQUE constraint failed: video_uploads.project_id') ); } -function safeFileName(projectName: string, projectId: string) { - const slug = projectName - .normalize('NFKD') - .replace(/[^a-zA-Z0-9_-]+/g, '-') +function videoOriginalKey(projectId: string, videoId: string, fileName: string) { + const safeName = fileName + .normalize('NFKC') + .replace(/[^a-zA-Z0-9._-]+/g, '-') .replace(/^-+|-+$/g, '') - .slice(0, 80); - return `${slug || projectId}.mp4`; + .slice(0, 100); + return `projects/${encodeURIComponent(projectId)}/videos/${videoId}/original/${safeName || 'video'}`; } diff --git a/test/env.d.ts b/test/env.d.ts index feab673..821862e 100644 --- a/test/env.d.ts +++ b/test/env.d.ts @@ -2,6 +2,7 @@ declare global { namespace Cloudflare { interface Env { TEST_MIGRATIONS: import('cloudflare:test').D1Migration[]; + VIDEOS: R2Bucket; } } } diff --git a/test/migration/migration.test.ts b/test/migration/migration.test.ts index 4feea0b..22f5928 100644 --- a/test/migration/migration.test.ts +++ b/test/migration/migration.test.ts @@ -23,6 +23,20 @@ async function fixture(name: string) { } describe('Firebase migration transformation', () => { + it('adds the forward R2 video history and multipart constraints', async () => { + const sql = await readFile( + path.resolve('migrations/0007_r2_video_lifecycle.sql'), + 'utf8', + ); + + expect(sql).toContain('ALTER TABLE project_videos RENAME TO legacy_project_videos'); + expect(sql).toContain('CREATE TABLE video_uploads'); + expect(sql).toContain('CREATE TABLE video_upload_parts'); + expect(sql).toContain('CREATE TABLE video_processing_attempts'); + expect(sql).toContain('WHERE retired_at IS NULL'); + expect(sql).toContain("status IN ('creating', 'uploading', 'completing')"); + }); + it('preserves deterministic IDs, relationships, and storage keys', async () => { const database = await fixture('database.json'); const manifest = await readStorageManifest( diff --git a/test/video-ui/video-ui.test.tsx b/test/video-ui/video-ui.test.tsx index 808d553..7c502a3 100644 --- a/test/video-ui/video-ui.test.tsx +++ b/test/video-ui/video-ui.test.tsx @@ -11,8 +11,18 @@ import { } from '../../src/app/player/ScreeningPlayer'; import {WatchPage} from '../../src/app/routes/WatchPage'; import {ProjectVideoPanel} from '../../src/app/video/ProjectVideoPanel'; -import type {ResumableUpload, UploadSnapshot} from '../../src/app/video/upload'; -import type {PlaylistItem, ProjectVideo} from '../../src/shared/videos'; +import { + createMultipartUpload, + persistResumeRecord, + readResumeRecord, + type ResumableUpload, + type UploadSnapshot, +} from '../../src/app/video/upload'; +import type { + PlaylistItem, + ProjectVideo, + VideoUploadSession, +} from '../../src/shared/videos'; const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); @@ -21,31 +31,22 @@ vi.stubGlobal( vi.fn(() => true), ); -afterEach(() => fetchMock.mockReset()); +afterEach(() => { + fetchMock.mockReset(); + localStorage.clear(); +}); describe('video user experience', () => { - it('shows resumable upload progress controls through a local fake tus event adapter', async () => { + it('shows resumable multipart progress controls through a local event adapter', async () => { fetchMock.mockImplementation(async (_input, init) => { if (init?.method === 'POST') { - return json( - { - video: {...baseVideo, status: 'uploading'}, - upload: { - protocol: 'tus', - url: 'https://upload.test/files/one', - expiresAt: 'later', - chunkSize: 1024, - }, - }, - 201, - ); + return json({video: null, upload: uploadSession}, 201); } return json({}); }); const uploadFactory = ( file: File, - _url: string, - _chunkSize: number, + _session: VideoUploadSession, onChange: (snapshot: UploadSnapshot) => void, ): ResumableUpload => ({ start: () => @@ -80,13 +81,46 @@ describe('video user experience', () => { ); }); + it('persists multipart resume identity and skips server-confirmed parts', async () => { + const file = new File(['abcde'], 'resume.mp4', { + type: 'video/mp4', + lastModified: 42, + }); + const session: VideoUploadSession = { + ...uploadSession, + fileName: file.name, + fileSize: file.size, + partSize: 3, + completedParts: [{partNumber: 1, etag: 'first', sizeBytes: 3}], + }; + persistResumeRecord(file, session); + expect(readResumeRecord('project', file)).toMatchObject({ + uploadId: 'upload-1', + completedParts: session.completedParts, + }); + + fetchMock + .mockResolvedValueOnce(json({part: {partNumber: 2, etag: 'second', sizeBytes: 2}})) + .mockResolvedValueOnce(json({video: {...baseVideo, status: 'queued'}})); + const snapshots: UploadSnapshot[] = []; + createMultipartUpload(file, session, (snapshot) => snapshots.push(snapshot)).start(); + + await vi.waitFor(() => expect(snapshots.at(-1)?.phase).toBe('complete')); + const firstRequest = fetchMock.mock.calls[0][0]; + if (typeof firstRequest !== 'string') + throw new Error('Expected a string request URL'); + expect(firstRequest).toContain('/parts/2'); + expect(firstRequest).not.toContain('/parts/1'); + expect(readResumeRecord('project', file)).toBeNull(); + }); + it.each([ - {streamMode: 'fake', expectedHref: null}, - {streamMode: 'disabled', expectedHref: null}, - {streamMode: 'real', expectedHref: '/years/2026/projects/project/video'}, + {streamMode: 'fake'}, + {streamMode: 'disabled'}, + {streamMode: 'real'}, ] as const)( - 'handles project video playback in $streamMode stream mode', - ({streamMode, expectedHref}) => { + 'keeps R2 lifecycle state independent of $streamMode Stream configuration', + ({streamMode}) => { renderQuery( { />, ); - expect( - screen.queryByRole('link', {name: 'watch video'})?.getAttribute('href') ?? null, - ).toBe(expectedHref); + expect(screen.getByRole('link', {name: 'watch video'}).getAttribute('href')).toBe( + '/years/2026/projects/project/video', + ); }, ); - it('shows disabled-video UX without upload or lifecycle actions', () => { + it('keeps uploads available without depending on Stream configuration', () => { renderQuery( { />, ); - expect(screen.getByText(/video processing is temporarily unavailable/i)).toBeTruthy(); - expect(screen.queryByLabelText('select project video')).toBeNull(); - expect(screen.queryByRole('button', {name: 'delete video'})).toBeNull(); + expect(screen.getByLabelText('select project video')).toBeTruthy(); + expect(screen.getByText(/private R2 storage/i)).toBeTruthy(); }); - it('keeps failed owner state visible with retry/replacement/delete actions', () => { + it('requires retirement before a failed video can be replaced', () => { renderQuery( { video={{ ...baseVideo, status: 'failed', - failureStage: 'measurement', + failureStage: 'processing', errorMessage: 'audio decode failed', }} canManage />, ); expect(screen.getByText('audio decode failed')).toBeTruthy(); - expect(screen.getByLabelText('choose replacement video')).toBeTruthy(); - expect(screen.getByRole('button', {name: 'retry measurement'})).toBeTruthy(); - expect(screen.getByRole('button', {name: 'delete video'})).toBeTruthy(); + expect(screen.queryByLabelText('select project video')).toBeNull(); + expect(screen.getByRole('button', {name: 'retire video'})).toBeTruthy(); }); it('renders accessible empty reel and individual ready-video permalinks', async () => { @@ -224,16 +256,29 @@ function json(value: unknown, status = 200) { const baseVideo: ProjectVideo = { id: 'video-1', projectId: 'project', - streamUid: 'stream', - sourceMediaId: null, status: 'ready', + originalName: 'demo.mp4', + contentType: 'video/mp4', + sizeBytes: 5, durationSeconds: 30, loudnessLufs: -16, gainDb: 0, errorMessage: null, failureStage: null, - archiveStatus: 'pending', - archiveError: null, + processingAttempt: 1, + createdAt: '2030-01-01T00:00:00.000Z', +}; +const uploadSession: VideoUploadSession = { + uploadId: 'upload-1', + videoId: 'video-1', + projectId: 'project', + fileName: 'demo.mp4', + contentType: 'video/mp4', + fileSize: 5, + partSize: 50 * 1024 * 1024, + expiresAt: '2030-01-02T00:00:00.000Z', + status: 'uploading', + completedParts: [], }; const playlist: PlaylistItem[] = [ { diff --git a/test/video/video.test.ts b/test/video/video.test.ts index 273fbe4..172a24d 100644 --- a/test/video/video.test.ts +++ b/test/video/video.test.ts @@ -2,18 +2,16 @@ import {env, SELF} from 'cloudflare:test'; import {beforeEach, describe, expect, it} from 'vitest'; import type {ProjectWriteRequest} from '../../src/shared/projects'; -import worker from '../../src/worker'; -import {streamGateway, streamMode} from '../../src/worker/integrations/stream'; -import {FakeStreamGateway} from '../../src/worker/integrations/stream/fake'; -import {RealStreamGateway} from '../../src/worker/integrations/stream/real'; -import {loudnessGain} from '../../src/worker/services/videos'; +import {MAX_VIDEO_BYTES, VIDEO_PART_SIZE} from '../../src/worker/services/videos'; import {createSessionCookie} from '../auth/fixture'; const base = 'https://hackweek.test/api'; -const webhookSecret = 'test-webhook-secret'; let suffix = 0; let ownerToken: string; +let memberToken: string; let outsiderToken: string; +let ownerId: string; +let memberId: string; let projectId: string; let yearId: string; let groupId: string; @@ -24,351 +22,332 @@ beforeEach(async () => { sub: `video-owner-${suffix}`, email: `video-owner-${suffix}@sentry.io`, }); + memberToken = await createSessionCookie({ + sub: `video-member-${suffix}`, + email: `video-member-${suffix}@sentry.io`, + }); outsiderToken = await createSessionCookie({ sub: `video-outsider-${suffix}`, email: `video-outsider-${suffix}@sentry.io`, }); - await session(ownerToken); - await session(outsiderToken); - yearId = `video-year-${suffix}`; - groupId = `video-group-${suffix}`; + await Promise.all([session(ownerToken), session(memberToken), session(outsiderToken)]); const owner = await env.DB.prepare('SELECT id FROM users WHERE google_subject = ?') .bind(`video-owner-${suffix}`) .first<{id: string}>(); + const member = await env.DB.prepare('SELECT id FROM users WHERE google_subject = ?') + .bind(`video-member-${suffix}`) + .first<{id: string}>(); + ownerId = owner!.id; + memberId = member!.id; + yearId = `zzzz-video-year-${String(suffix).padStart(4, '0')}`; + groupId = `video-group-${suffix}`; await env.DB.batch([ env.DB.prepare('INSERT INTO years (id) VALUES (?)').bind(yearId), env.DB.prepare( `INSERT INTO groups (id, source_id, year_id, name, creator_id) VALUES (?, ?, ?, 'Video group', ?)`, - ).bind(groupId, groupId, yearId, owner!.id), + ).bind(groupId, groupId, yearId, ownerId), ]); - const created = await api('/projects', ownerToken, { - method: 'POST', - body: projectPayload(), - }); - expect(created.status, JSON.stringify(created.body)).toBe(201); - projectId = created.body.project.id; + projectId = await createProject('Video project'); + await env.DB.prepare('INSERT INTO project_members (project_id, user_id) VALUES (?, ?)') + .bind(projectId, memberId) + .run(); }); -describe('Cloudflare Stream gateways', () => { - it('fails closed without creating fake records when Stream is disabled', async () => { - const disabledEnv = new Proxy(env, { - get(target, property, receiver) { - return property === 'STREAM_MODE' - ? 'disabled' - : Reflect.get(target, property, receiver); - }, +describe('R2 multipart video lifecycle', () => { + it('streams resumable parts, completes idempotently, and records the queued handoff', async () => { + const forbidden = await createUpload(projectId, outsiderToken, 11); + expect(forbidden.status).toBe(403); + + const created = await createUpload(projectId, memberToken, 11); + expect(created.status).toBe(201); + expect(created.body.video).toBeNull(); + expect(created.body.upload).toMatchObject({ + projectId, + fileSize: 11, + partSize: VIDEO_PART_SIZE, + status: 'uploading', + completedParts: [], }); - expect(streamMode(disabledEnv)).toBe('disabled'); - expect(() => streamGateway(disabledEnv)).toThrow( - 'Video processing is temporarily unavailable', + + const uploadId = created.body.upload.uploadId as string; + const firstPart = await putPart( + projectId, + uploadId, + 1, + new TextEncoder().encode('hello video'), + memberToken, + ); + const duplicatePart = await putPart( + projectId, + uploadId, + 1, + new TextEncoder().encode('hello video'), + memberToken, + ); + expect(firstPart.status).toBe(200); + expect(duplicatePart.body.part.etag).toBe(firstPart.body.part.etag); + + const resumed = await api( + `/projects/${projectId}/video/upload/${uploadId}`, + memberToken, ); + expect(resumed.body.upload.completedParts).toEqual([firstPart.body.part]); - const response = await worker.request( - `${base}/projects/${projectId}/video/upload`, + const parts = [ { - method: 'POST', - headers: { - Cookie: ownerToken, - Origin: 'https://hackweek.test', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({fileName: 'demo.mp4', fileSize: 300_000_000}), + partNumber: firstPart.body.part.partNumber, + etag: firstPart.body.part.etag, }, - disabledEnv, + ]; + const completed = await api( + `/projects/${projectId}/video/upload/${uploadId}/complete`, + memberToken, + {method: 'POST', body: {parts}}, + ); + const duplicateCompletion = await api( + `/projects/${projectId}/video/upload/${uploadId}/complete`, + memberToken, + {method: 'POST', body: {parts}}, ); - const stored = await env.DB.prepare( - 'SELECT COUNT(*) count FROM project_videos WHERE project_id = ?', - ) - .bind(projectId) - .first<{count: number}>(); - expect(response.status).toBe(503); - expect(await response.json()).toEqual({ - error: { - code: 'SERVICE_UNAVAILABLE', - message: 'Video processing is temporarily unavailable', - }, + expect(completed.status).toBe(200); + expect(completed.body.video).toMatchObject({ + projectId, + status: 'queued', + sizeBytes: 11, + originalName: 'demo.mp4', + processingAttempt: 1, }); - expect(stored?.count).toBe(0); - }); + expect(duplicateCompletion.body.video.id).toBe(completed.body.video.id); - it('creates constrained direct tus requests without exposing the API token', async () => { - const requests: Request[] = []; - const originalFetch = globalThis.fetch; - globalThis.fetch = async (input, init) => { - const request = new Request(input, init); - requests.push(request); - return new Response(null, { - status: 201, - headers: { - Location: 'https://upload.videodelivery.net/tus-once', - 'stream-media-id': 'stream-real-uid', - }, - }); - }; - try { - const gateway = new RealStreamGateway('account-1', 'super-secret-token'); - const upload = await gateway.createDirectUpload({ - creator: 'user-1', - fileName: 'demo reel.mp4', - fileSize: 300_000_000, - maxDurationSeconds: 600, - allowedOrigin: 'hackweek.example.com', - expiresAt: new Date('2030-01-01T00:30:00.000Z'), - }); - expect(upload).toMatchObject({uid: 'stream-real-uid', protocol: 'tus'}); - expect(requests[0].url).toContain('/stream?direct_user=true'); - expect(requests[0].headers.get('Authorization')).toBe('Bearer super-secret-token'); - expect(requests[0].headers.get('Upload-Length')).toBe('300000000'); - expect(requests[0].headers.get('Tus-Resumable')).toBe('1.0.0'); - const metadata = requests[0].headers.get('Upload-Metadata')!; - expect(metadata).toContain('requiresignedurls'); - expect(metadata).toContain(`maxdurationseconds ${btoa('600')}`); - expect(metadata).toContain(`allowedorigins ${btoa('hackweek.example.com')}`); - } finally { - globalThis.fetch = originalFetch; - } + const stored = await env.DB.prepare( + `SELECT original_r2_key FROM project_videos WHERE id = ?`, + ) + .bind(completed.body.video.id) + .first<{original_r2_key: string}>(); + const attempt = await env.DB.prepare( + `SELECT attempt, status FROM video_processing_attempts WHERE video_id = ?`, + ) + .bind(completed.body.video.id) + .first<{attempt: number; status: string}>(); + expect((await env.VIDEOS.head(stored!.original_r2_key))?.size).toBe(11); + expect(attempt).toEqual({attempt: 1, status: 'queued'}); }); - it('keeps the local fake explicit and does not fabricate HLS', async () => { - const gateway = new FakeStreamGateway(); - const upload = await gateway.createDirectUpload({ - creator: 'user', - fileName: 'demo.mp4', - fileSize: 42, - maxDurationSeconds: 600, - allowedOrigin: 'hackweek.test', - expiresAt: new Date('2030-01-01T00:30:00Z'), - }); - const token = await gateway.createPlaybackToken( - upload.uid, - new Date('2030-01-01T00:15:00Z'), - ); - expect(upload.uploadUrl).toMatch(/^https:\/\/upload\.videodelivery\.net\/fake\//); - expect(token).toMatch(/^fake\.playback\./); - expect(token).not.toContain('m3u8'); - }); -}); + it('enforces one active project slot in D1 while different projects stay independent', async () => { + const sameProject = await Promise.all([ + createUpload(projectId, ownerToken, 20), + createUpload(projectId, ownerToken, 20), + ]); + expect(sameProject.map(({status}) => status).sort((a, b) => a - b)).toEqual([ + 201, 409, + ]); -describe('video lifecycle APIs', () => { - it('authorizes one direct primary upload and returns no secrets or bytes', async () => { - const forbidden = await api(`/projects/${projectId}/video/upload`, outsiderToken, { - method: 'POST', - body: {fileName: 'demo.mp4', fileSize: 300_000_000}, - }); - const created = await api(`/projects/${projectId}/video/upload`, ownerToken, { - method: 'POST', - body: {fileName: 'demo.mp4', fileSize: 300_000_000}, - }); - const duplicate = await api(`/projects/${projectId}/video/upload`, ownerToken, { - method: 'POST', - body: {fileName: 'another.mp4', fileSize: 10}, - }); + const left = await createProject('Independent left'); + const right = await createProject('Independent right'); + const independent = await Promise.all([ + createUpload(left, ownerToken, 20), + createUpload(right, ownerToken, 20), + ]); + expect(independent.map(({status}) => status)).toEqual([201, 201]); - expect(forbidden.status).toBe(403); - expect(created.status).toBe(201); - expect(created.body.upload).toMatchObject({protocol: 'tus', chunkSize: 52_428_800}); - expect(created.body.video.status).toBe('uploading'); - const serialized = JSON.stringify(created.body); - expect(serialized).not.toMatch(/api.?token|secret|fileSize|fileName/i); - expect(duplicate.status).toBe(409); + const activeIndex = await env.DB.prepare( + `SELECT sql FROM sqlite_master WHERE type = 'index' + AND name = 'project_videos_active_project_idx'`, + ).first<{sql: string}>(); + expect(activeIndex?.sql).toContain('WHERE retired_at IS NULL'); }); - it('verifies webhooks and deduplicates lifecycle events', async () => { - const created = await createUpload(); - const payload = JSON.stringify({ - uid: created.video.streamUid, - readyToStream: true, - modified: '2030-01-01T00:00:00Z', - duration: 92.5, - status: {state: 'ready', pctComplete: '100.000000'}, - }); - const invalid = await SELF.fetch(`${base}/stream-webhook`, { - method: 'POST', - headers: {'Webhook-Signature': 'time=1,sig1=bad'}, - body: payload, - }); - const signature = await webhookSignature(payload); - const first = await SELF.fetch(`${base}/stream-webhook`, { - method: 'POST', - headers: {'Webhook-Signature': signature}, - body: payload, + it('retires only with confirmation, retains the original, and requires a fresh replacement', async () => { + const {video, key} = await completeSmallUpload(projectId, ownerToken, 'first video'); + const unconfirmed = await api(`/projects/${projectId}/video`, ownerToken, { + method: 'DELETE', + body: {confirmed: false}, }); - const duplicate = await SELF.fetch(`${base}/stream-webhook`, { - method: 'POST', - headers: {'Webhook-Signature': signature}, - body: payload, + expect(unconfirmed.status).toBe(400); + + const retired = await api(`/projects/${projectId}/video`, ownerToken, { + method: 'DELETE', + body: {confirmed: true}, }); - const stored = await env.DB.prepare( - 'SELECT status, duration_seconds FROM project_videos WHERE id = ?', + expect(retired.status).toBe(204); + expect(await env.VIDEOS.head(key)).not.toBeNull(); + const retiredRow = await env.DB.prepare( + 'SELECT status, original_r2_key FROM project_videos WHERE id = ?', ) - .bind(created.video.id) - .first<{status: string; duration_seconds: number}>(); - const eventCount = await env.DB.prepare( - 'SELECT COUNT(*) count FROM stream_events WHERE stream_uid = ?', + .bind(video.id) + .first<{status: string; original_r2_key: string}>(); + expect(retiredRow).toEqual({status: 'retired', original_r2_key: key}); + const attempt = await env.DB.prepare( + 'SELECT status FROM video_processing_attempts WHERE video_id = ?', ) - .bind(created.video.streamUid) - .first<{count: number}>(); + .bind(video.id) + .first<{status: string}>(); + expect(attempt?.status).toBe('cancelled'); + + const replacement = await createUpload(projectId, ownerToken, 7); + expect(replacement.status).toBe(201); + expect(replacement.body.upload.videoId).not.toBe(video.id); + expect(replacement.body.upload.uploadId).not.toBe(video.id); - expect(invalid.status).toBe(401); - expect(first.status).toBe(200); - expect(await first.json()).toMatchObject({handled: true, duplicate: false}); - expect(await duplicate.json()).toMatchObject({handled: true, duplicate: true}); - expect(stored).toMatchObject({status: 'measuring', duration_seconds: 92.5}); - expect(eventCount?.count).toBe(1); + const promoted = await api(`/projects/${projectId}/video/promote`, ownerToken, { + method: 'POST', + body: {sourceMediaId: 'attachment-id'}, + }); + expect(promoted.status).toBe(404); }); - it('clamps exact -16 LUFS gain and only playlists measured ready videos', async () => { - const created = await createUpload(); - await moveToMeasuring(created.video.streamUid); + it('rejects malformed, oversized, closed, idea, stale, and incomplete requests deterministically', async () => { + const malformed = await api(`/projects/${projectId}/video/upload`, ownerToken, { + method: 'POST', + body: {fileName: 'demo.mp4', fileSize: 10, contentType: 'text/plain'}, + }); + const oversized = await createUpload(projectId, ownerToken, MAX_VIDEO_BYTES + 1); + expect(malformed.status).toBe(400); + expect(oversized.status).toBe(400); + const maximumProject = await createProject('Maximum declaration'); + const maximum = await createUpload(maximumProject, ownerToken, MAX_VIDEO_BYTES); + expect(maximum.status).toBe(201); + await api( + `/projects/${maximumProject}/video/upload/${maximum.body.upload.uploadId}`, + ownerToken, + {method: 'DELETE'}, + ); + + const ideaId = `video-idea-${suffix}`; await env.DB.prepare( - 'INSERT INTO screening_order (year_id, project_id, position) VALUES (?, ?, 0)', + `INSERT INTO projects + (id, source_id, year_id, creator_id, group_id, name, kind) + VALUES (?, ?, ?, ?, ?, 'Video idea', 'idea')`, ) - .bind(yearId, projectId) + .bind(ideaId, ideaId, yearId, ownerId, groupId) .run(); + expect((await createUpload(ideaId, ownerToken, 10)).status).toBe(400); - const before = await api(`/videos/playlist?year=${yearId}`, ownerToken); - const measured = await serviceApi(`/video-jobs/measurements/${created.video.id}`, { - method: 'POST', - body: {loudnessLufs: -31, durationSeconds: 92.5}, - }); - const after = await api(`/videos/playlist?year=${yearId}`, ownerToken); - const playback = await api(`/videos/${created.video.id}/playback`, ownerToken); + await env.DB.prepare('UPDATE years SET submissions_closed = 1 WHERE id = ?') + .bind(yearId) + .run(); + expect((await createUpload(projectId, ownerToken, 10)).status).toBe(403); + await env.DB.prepare('UPDATE years SET submissions_closed = 0 WHERE id = ?') + .bind(yearId) + .run(); - expect(loudnessGain(-31)).toBe(12); - expect(loudnessGain(0)).toBe(-12); - expect(loudnessGain(-18.5)).toBe(2.5); - expect(before.body.videos).toEqual([]); - expect(measured.body.video).toMatchObject({status: 'ready', gainDb: 12}); - expect(after.body.videos).toEqual([ - expect.objectContaining({videoId: created.video.id, gainDb: 12}), - ]); - expect(playback.body).toMatchObject({mode: 'fake', manifestUrl: null}); - expect(playback.headers.get('Cache-Control')).toBe('private, no-store'); - }); + const created = await createUpload(projectId, ownerToken, 10); + const uploadId = created.body.upload.uploadId as string; + const incomplete = await api( + `/projects/${projectId}/video/upload/${uploadId}/complete`, + ownerToken, + {method: 'POST', body: {parts: [{partNumber: 1, etag: 'missing'}]}}, + ); + expect(incomplete.status).toBe(400); - it('promotes selected historical R2 video media through the same record', async () => { - const mediaId = `video-media-${suffix}`; await env.DB.prepare( - `INSERT INTO media - (id, source_id, project_id, original_name, r2_key, media_type, status) - VALUES (?, ?, ?, 'old-demo.mp4', ?, 'video/mp4', 'available')`, + `UPDATE video_uploads SET expires_at = '2000-01-01T00:00:00.000Z' WHERE id = ?`, ) - .bind(mediaId, mediaId, projectId, `media/${mediaId}.mp4`) + .bind(uploadId) .run(); - const promoted = await api(`/projects/${projectId}/video/promote`, ownerToken, { - method: 'POST', - body: {sourceMediaId: mediaId}, - }); - - expect(promoted.status).toBe(201); - expect(promoted.body.video).toMatchObject({ - projectId, - sourceMediaId: mediaId, - status: 'processing', - }); + const expired = await putPart(projectId, uploadId, 1, new Uint8Array(10), ownerToken); + expect(expired.status).toBe(409); + expect(expired.body.error.message).toBe('Upload session has expired'); }); - it('uses distinct service auth and keeps archive off readiness', async () => { - const created = await createUpload(); - await moveToMeasuring(created.video.streamUid); - await serviceApi(`/video-jobs/measurements/${created.video.id}`, { - method: 'POST', - body: {loudnessLufs: -16, durationSeconds: 20}, - }); - const unauthenticated = await SELF.fetch(`${base}/video-jobs/archives`); - const queue = await serviceApi('/video-jobs/archives'); - const failed = await serviceApi(`/video-jobs/archives/${created.video.id}`, { - method: 'POST', - body: {status: 'failed', error: 'Drive quota exhausted'}, - }); - const stillReady = await env.DB.prepare( - 'SELECT status, archive_status FROM project_videos WHERE id = ?', - ) - .bind(created.video.id) - .first<{status: string; archive_status: string}>(); - - expect(unauthenticated.status).toBe(401); - expect(queue.body.videos).toContainEqual( - expect.objectContaining({videoId: created.video.id}), + it('allows aborting incomplete uploads idempotently but never completed objects', async () => { + const created = await createUpload(projectId, ownerToken, 8); + const uploadId = created.body.upload.uploadId as string; + const first = await api( + `/projects/${projectId}/video/upload/${uploadId}`, + ownerToken, + {method: 'DELETE'}, ); - expect(failed.body.video).toMatchObject({status: 'ready', archiveStatus: 'failed'}); - expect(stillReady).toEqual({status: 'ready', archive_status: 'failed'}); - }); - - it('exposes measurement failures as retryable state', async () => { - const created = await createUpload(); - await moveToMeasuring(created.video.streamUid); - const failed = await serviceApi( - `/video-jobs/measurements/${created.video.id}/failure`, - {method: 'POST', body: {error: 'No audio stream'}}, + const duplicate = await api( + `/projects/${projectId}/video/upload/${uploadId}`, + ownerToken, + {method: 'DELETE'}, ); - const stored = await env.DB.prepare( - 'SELECT status, failure_stage FROM project_videos WHERE id = ?', - ) - .bind(created.video.id) - .first<{status: string; failure_stage: string}>(); - const retried = await api(`/videos/${created.video.id}/retry`, ownerToken, { - method: 'POST', - }); - - expect(failed.status).toBe(204); - expect(stored).toEqual({status: 'failed', failure_stage: 'measurement'}); - expect(retried.body.video.status).toBe('measuring'); + expect(first.status).toBe(204); + expect(duplicate.status).toBe(204); }); }); -async function createUpload() { - const response = await api(`/projects/${projectId}/video/upload`, ownerToken, { - method: 'POST', - body: {fileName: 'demo.mp4', fileSize: 300_000_000}, - }); - expect(response.status).toBe(201); - return response.body; +async function completeSmallUpload(project: string, token: string, bytes: string) { + const created = await createUpload(project, token, bytes.length); + expect(created.status).toBe(201); + const uploadId = created.body.upload.uploadId as string; + const part = await putPart( + project, + uploadId, + 1, + new TextEncoder().encode(bytes), + token, + ); + expect(part.status).toBe(200); + const completed = await api( + `/projects/${project}/video/upload/${uploadId}/complete`, + token, + { + method: 'POST', + body: { + parts: [{partNumber: 1, etag: part.body.part.etag}], + }, + }, + ); + expect(completed.status).toBe(200); + const stored = await env.DB.prepare( + 'SELECT original_r2_key FROM project_videos WHERE id = ?', + ) + .bind(completed.body.video.id) + .first<{original_r2_key: string}>(); + return {video: completed.body.video, key: stored!.original_r2_key}; } -async function moveToMeasuring(streamUid: string) { - const payload = JSON.stringify({ - uid: streamUid, - readyToStream: true, - modified: `2030-01-01T00:00:0${suffix % 10}Z`, - duration: 20, - status: {state: 'ready', pctComplete: '100'}, - }); - const response = await SELF.fetch(`${base}/stream-webhook`, { +function createUpload(project: string, token: string, fileSize: number) { + return api(`/projects/${project}/video/upload`, token, { method: 'POST', - headers: {'Webhook-Signature': await webhookSignature(payload)}, - body: payload, + body: {fileName: 'demo.mp4', fileSize, contentType: 'video/mp4'}, }); - expect(response.status).toBe(200); } -async function webhookSignature(body: string) { - const timestamp = Math.floor(Date.now() / 1000); - const key = await crypto.subtle.importKey( - 'raw', - new TextEncoder().encode(webhookSecret), - {name: 'HMAC', hash: 'SHA-256'}, - false, - ['sign'], - ); - const digest = await crypto.subtle.sign( - 'HMAC', - key, - new TextEncoder().encode(`${timestamp}.${body}`), +async function putPart( + project: string, + uploadId: string, + partNumber: number, + body: Uint8Array, + token: string, +) { + const response = await SELF.fetch( + `${base}/projects/${project}/video/upload/${uploadId}/parts/${partNumber}`, + { + method: 'PUT', + headers: { + Cookie: token, + Origin: 'https://hackweek.test', + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(body.byteLength), + }, + body: body.buffer.slice( + body.byteOffset, + body.byteOffset + body.byteLength, + ) as ArrayBuffer, + }, ); - const signature = [...new Uint8Array(digest)] - .map((byte) => byte.toString(16).padStart(2, '0')) - .join(''); - return `time=${timestamp},sig1=${signature}`; + return parseResponse(response); } -function projectPayload(): ProjectWriteRequest { +async function createProject(name: string) { + const response = await api('/projects', ownerToken, { + method: 'POST', + body: projectPayload(name), + }); + expect(response.status, JSON.stringify(response.body)).toBe(201); + return response.body.project.id as string; +} + +function projectPayload(name: string): ProjectWriteRequest { return { yearId, - name: 'Video project', - summary: 'Project with a directly uploaded primary demo video.', + name, + summary: 'Project with a multipart R2 video.', repository: null, kind: 'project', groupId, @@ -379,9 +358,7 @@ function projectPayload(): ProjectWriteRequest { } function session(token: string) { - return SELF.fetch(`${base}/session`, { - headers: {Cookie: token}, - }); + return SELF.fetch(`${base}/session`, {headers: {Cookie: token}}); } async function api( @@ -403,18 +380,6 @@ async function api( return parseResponse(response); } -async function serviceApi(path: string, options: {method?: string; body?: unknown} = {}) { - const response = await SELF.fetch(`${base}${path}`, { - method: options.method, - headers: { - Authorization: 'Bearer test-video-service-token', - ...(options.body === undefined ? {} : {'Content-Type': 'application/json'}), - }, - body: options.body === undefined ? undefined : JSON.stringify(options.body), - }); - return parseResponse(response); -} - async function parseResponse(response: Response) { const body = response.status === 204 ? null : await response.json(); return {status: response.status, body, headers: response.headers}; diff --git a/wrangler.jsonc b/wrangler.jsonc index 591bfdb..39d8b13 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -21,6 +21,10 @@ "binding": "ATTACHMENTS", "bucket_name": "hackweek-attachments-local", }, + { + "binding": "VIDEOS", + "bucket_name": "hackweek-videos-local", + }, ], "observability": { "enabled": true, From ffda277518189867ef1f4c8850a8ce229d53d7f5 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 14:04:44 +0200 Subject: [PATCH 02/18] feat(video): process R2 uploads with Workflow and FFmpeg Start one deterministic, attempt-fenced Cloudflare Workflow when an R2 multipart upload completes, and route scoped source/output access through a Container outbound handler without exposing bucket credentials. Add a digest-pinned multi-architecture FFmpeg image that probes, rejects overlong or malformed input, performs two-pass -16 LUFS normalization, supplies deterministic silence, and emits immutable H.264/AAC fast-start MP4 derivatives. Configure local concurrency one and prepared production concurrency two, with real Docker and local Workflow smoke coverage. --- .dev.vars.example | 4 +- .dockerignore | 3 + Dockerfile.video-processor | 13 + package-lock.json | 7 + package.json | 6 +- processor/video-processor.mjs | 384 +++++++++++++++++++ scripts/local-video-workflow.ts | 469 +++++++++++++++++++++++ scripts/measure-loudness.ts | 91 ----- scripts/test-video-processor.ts | 265 +++++++++++++ src/worker/containers/video-processor.ts | 105 +++++ src/worker/index.ts | 15 +- src/worker/routes/videos.ts | 20 + src/worker/services/videos.ts | 259 ++++++++++++- src/worker/video-processing.ts | 27 ++ src/worker/workflows/video-processing.ts | 165 ++++++++ test/video/video.test.ts | 146 ++++++- vitest.config.ts | 1 + worker-configuration.d.ts | 8 +- wrangler.jsonc | 35 ++ wrangler.production.json | 37 ++ 20 files changed, 1958 insertions(+), 102 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile.video-processor create mode 100644 processor/video-processor.mjs create mode 100644 scripts/local-video-workflow.ts delete mode 100644 scripts/measure-loudness.ts create mode 100644 scripts/test-video-processor.ts create mode 100644 src/worker/containers/video-processor.ts create mode 100644 src/worker/video-processing.ts create mode 100644 src/worker/workflows/video-processing.ts diff --git a/.dev.vars.example b/.dev.vars.example index 43d3ead..7d6dfa6 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -6,8 +6,8 @@ GOOGLE_CLIENT_SECRET= GOOGLE_REDIRECT_URI=http://localhost:5173/api/auth/callback ALLOWED_EMAIL_DOMAIN="sentry.io" -# The local adapter issues direct-upload fixtures and protected-playback contracts. -# It does not transcode, generate HLS, or move video bytes. +# R2 uploads are processed by the local Workflow and pinned FFmpeg Container. +# Stream fake mode remains only for the playback surface pending its R2 cutover. STREAM_MODE="fake" STREAM_ALLOWED_ORIGIN="localhost" STREAM_DELIVERY_HOST="customer-fake.cloudflarestream.com" diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9ed695b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile.video-processor +!processor/video-processor.mjs diff --git a/Dockerfile.video-processor b/Dockerfile.video-processor new file mode 100644 index 0000000..b485dde --- /dev/null +++ b/Dockerfile.video-processor @@ -0,0 +1,13 @@ +FROM mwader/static-ffmpeg:8.0.1@sha256:252705ff88532fa338e7065c21792756552f8fe7c212f84bc503d3c340689594 AS ffmpeg + +FROM node:24.11.0-bookworm-slim@sha256:76d0ed0ed93bed4f4376211e9d8fddac4d8b3fbdb54cc45955696001a3c91152 +COPY --from=ffmpeg /ffmpeg /usr/local/bin/ffmpeg +COPY --from=ffmpeg /ffprobe /usr/local/bin/ffprobe +COPY processor/video-processor.mjs /app/video-processor.mjs +RUN useradd --create-home --uid 10001 processor \ + && chmod 0555 /usr/local/bin/ffmpeg /usr/local/bin/ffprobe /app/video-processor.mjs +USER processor +WORKDIR /app +ENV PORT=8080 +EXPOSE 8080 +ENTRYPOINT ["node", "/app/video-processor.mjs"] diff --git a/package-lock.json b/package-lock.json index 661d1c6..9c229e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "hackweek", "version": "0.1.0", "dependencies": { + "@cloudflare/containers": "0.3.7", "@fontsource/rubik": "^5.3.0", "@tanstack/react-query": "^5.101.4", "hls.js": "^1.6.16", @@ -97,6 +98,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@cloudflare/containers": { + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/@cloudflare/containers/-/containers-0.3.7.tgz", + "integrity": "sha512-DM9dm3FnIBSyiSJ1FLavKwl/lk3oAmTaynCzZQ9pZR0ncRPquSxkxd8Nu2MFILxmDDsPkxKsSNEh9mHHMty4Fw==", + "license": "MIT OR Apache-2.0" + }, "node_modules/@cloudflare/kv-asset-handler": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", diff --git a/package.json b/package.json index 8eb1349..da4d50b 100644 --- a/package.json +++ b/package.json @@ -20,15 +20,19 @@ "preview": "vp preview", "cf-typegen": "wrangler types worker-configuration.d.ts --config wrangler.production.json", "db:migrate:local": "wrangler d1 migrations apply hackweek-db --local", + "video:dev": "npm run db:migrate:local && vp dev", + "video:processor:build": "docker build --file Dockerfile.video-processor --tag hackweek-video-processor:local .", + "test:video-processor": "tsx scripts/test-video-processor.ts", + "test:video-workflow": "tsx scripts/local-video-workflow.ts", "migrate:validate": "tsx scripts/migrate/cli.ts validate", "migrate:dry-run": "tsx scripts/migrate/cli.ts dry-run", "migrate:local": "tsx scripts/migrate/cli.ts import --target local", "migrate:cloudflare": "tsx scripts/migrate/cli.ts import --target cloudflare", "migrate:reconcile": "tsx scripts/migrate/cli.ts reconcile", - "video:measure": "tsx scripts/measure-loudness.ts", "video:archive": "tsx scripts/archive-to-drive.ts" }, "dependencies": { + "@cloudflare/containers": "0.3.7", "@fontsource/rubik": "^5.3.0", "@tanstack/react-query": "^5.101.4", "hls.js": "^1.6.16", diff --git a/processor/video-processor.mjs b/processor/video-processor.mjs new file mode 100644 index 0000000..1bf34ec --- /dev/null +++ b/processor/video-processor.mjs @@ -0,0 +1,384 @@ +import {createHash} from 'node:crypto'; +import {createReadStream, createWriteStream} from 'node:fs'; +import {mkdtemp, open, rm, stat} from 'node:fs/promises'; +import {createServer} from 'node:http'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +import {spawn} from 'node:child_process'; +import {Readable} from 'node:stream'; +import {pipeline} from 'node:stream/promises'; + +const TARGET_LUFS = -16; +const LOUDNESS_TOLERANCE_LU = 0.7; +const MAX_DURATION_SECONDS = 600; +const PORT = Number(process.env.PORT ?? 8080); +const R2_ORIGIN = process.env.VIDEO_R2_ORIGIN ?? 'http://video-r2'; +const SCALE_FILTER = + "scale=w='min(1920,iw)':h='min(1080,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2"; + +class ProcessorError extends Error {} + +export async function processFile(inputPath, outputPath) { + const source = await probe(inputPath); + const sourceVideo = source.streams.find((stream) => stream.codec_type === 'video'); + if (!sourceVideo || !positive(sourceVideo.width) || !positive(sourceVideo.height)) { + throw new ProcessorError('Input does not contain a valid video stream'); + } + const durationSeconds = mediaDuration(source); + if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) { + throw new ProcessorError('Input duration is invalid'); + } + if (durationSeconds > MAX_DURATION_SECONDS + 0.01) { + throw new ProcessorError( + `Input duration ${durationSeconds.toFixed(3)}s exceeds ${MAX_DURATION_SECONDS}s`, + ); + } + + const hasAudio = source.streams.some((stream) => stream.codec_type === 'audio'); + const firstPass = hasAudio ? await analyzeLoudness(inputPath) : null; + const normalizeAudio = firstPass !== null && firstPass.inputI > -70; + await transcode(inputPath, outputPath, firstPass, normalizeAudio); + + const output = await probe(outputPath); + const video = output.streams.find((stream) => stream.codec_type === 'video'); + const audio = output.streams.find((stream) => stream.codec_type === 'audio'); + const outputDuration = mediaDuration(output); + if ( + !video || + !audio || + video.codec_name !== 'h264' || + audio.codec_name !== 'aac' || + video.pix_fmt !== 'yuv420p' || + !positive(video.width) || + !positive(video.height) || + video.width > 1920 || + video.height > 1080 || + outputDuration > MAX_DURATION_SECONDS + 0.05 + ) { + throw new ProcessorError( + 'Canonical output failed codec, pixel, size, or duration checks', + ); + } + + const rotation = sourceRotation(sourceVideo); + const sourceWidth = + Math.abs(rotation) % 180 === 90 ? sourceVideo.height : sourceVideo.width; + const sourceHeight = + Math.abs(rotation) % 180 === 90 ? sourceVideo.width : sourceVideo.height; + if (video.width > sourceWidth || video.height > sourceHeight) { + throw new ProcessorError('Canonical output unexpectedly upscaled the source'); + } + + const fastStart = await hasFastStart(outputPath); + if (!fastStart) throw new ProcessorError('Canonical MP4 is not fast-start enabled'); + const measured = await analyzeLoudness(outputPath); + const loudnessLufs = measured?.inputI ?? null; + if ( + normalizeAudio && + (loudnessLufs === null || + Math.abs(loudnessLufs - TARGET_LUFS) > LOUDNESS_TOLERANCE_LU) + ) { + throw new ProcessorError( + `Output loudness ${String(loudnessLufs)} LUFS is outside ${LOUDNESS_TOLERANCE_LU} LU of ${TARGET_LUFS}`, + ); + } + + return { + durationSeconds: outputDuration, + width: video.width, + height: video.height, + videoCodec: video.codec_name, + audioCodec: audio.codec_name, + pixelFormat: video.pix_fmt, + loudnessLufs, + loudnessTargetLufs: TARGET_LUFS, + loudnessToleranceLu: LOUDNESS_TOLERANCE_LU, + audioMode: normalizeAudio ? 'normalized' : 'generated-silence', + fastStart, + sha256: await sha256(outputPath), + }; +} + +async function transcode(inputPath, outputPath, firstPass, normalizeAudio) { + const args = ['-hide_banner', '-nostdin', '-y', '-i', inputPath]; + if (!normalizeAudio) { + args.push('-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=48000'); + } + args.push('-map', '0:v:0', '-map', normalizeAudio ? '0:a:0' : '1:a:0'); + args.push('-vf', SCALE_FILTER); + if (normalizeAudio) { + args.push( + '-af', + [ + `loudnorm=I=${TARGET_LUFS}`, + 'LRA=11', + 'TP=-1.5', + `measured_I=${firstPass.inputI}`, + `measured_LRA=${firstPass.inputLra}`, + `measured_TP=${firstPass.inputTp}`, + `measured_thresh=${firstPass.inputThresh}`, + `offset=${firstPass.targetOffset}`, + 'linear=true', + 'print_format=summary', + ].join(':'), + ); + } + args.push( + '-c:v', + 'libx264', + '-preset', + 'medium', + '-crf', + '20', + '-profile:v', + 'high', + '-level:v', + '4.1', + '-pix_fmt', + 'yuv420p', + '-c:a', + 'aac', + '-b:a', + '192k', + '-ar', + '48000', + '-ac', + '2', + '-map_metadata', + '-1', + '-metadata:s:v:0', + 'rotate=0', + '-sn', + '-dn', + '-movflags', + '+faststart', + '-max_muxing_queue_size', + '4096', + '-shortest', + outputPath, + ); + await run('ffmpeg', args); +} + +async function analyzeLoudness(file) { + const output = await run('ffmpeg', [ + '-hide_banner', + '-nostdin', + '-i', + file, + '-map', + '0:a:0', + '-af', + `loudnorm=I=${TARGET_LUFS}:LRA=11:TP=-1.5:print_format=json`, + '-f', + 'null', + '-', + ]); + const blocks = [...output.matchAll(/\{[\s\S]*?"input_i"[\s\S]*?\}/g)]; + const json = blocks.at(-1)?.[0]; + if (!json) throw new ProcessorError('FFmpeg loudness analysis did not return data'); + const value = JSON.parse(json); + const parsed = { + inputI: finite(value.input_i), + inputTp: finite(value.input_tp), + inputLra: finite(value.input_lra), + inputThresh: finite(value.input_thresh), + targetOffset: finite(value.target_offset), + }; + return Object.values(parsed).every((item) => item !== null) ? parsed : null; +} + +async function probe(file) { + const output = await run('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration:stream=codec_type,codec_name,width,height,pix_fmt,duration:stream_tags=rotate:stream_side_data=rotation', + '-of', + 'json', + file, + ]); + try { + const parsed = JSON.parse(output); + if (!Array.isArray(parsed.streams)) throw new Error('streams missing'); + return parsed; + } catch { + throw new ProcessorError('Input is malformed or could not be probed'); + } +} + +function mediaDuration(probeResult) { + const formatDuration = Number(probeResult.format?.duration); + if (Number.isFinite(formatDuration)) return formatDuration; + return Math.max(...probeResult.streams.map((stream) => Number(stream.duration) || 0)); +} + +function sourceRotation(stream) { + const sideData = Array.isArray(stream.side_data_list) + ? stream.side_data_list.find((entry) => Number.isFinite(Number(entry.rotation))) + : null; + return Number(sideData?.rotation ?? stream.tags?.rotate ?? 0); +} + +async function hasFastStart(file) { + const handle = await open(file, 'r'); + try { + const {size} = await stat(file); + const buffer = Buffer.alloc(Math.min(size, 2 * 1024 * 1024)); + await handle.read(buffer, 0, buffer.length, 0); + const moov = buffer.indexOf(Buffer.from('moov')); + const mdat = buffer.indexOf(Buffer.from('mdat')); + return moov >= 0 && mdat >= 0 && moov < mdat; + } finally { + await handle.close(); + } +} + +function sha256(file) { + return new Promise((resolve, reject) => { + const hash = createHash('sha256'); + const input = createReadStream(file); + input.on('error', reject); + input.on('data', (chunk) => hash.update(chunk)); + input.on('end', () => resolve(hash.digest('hex'))); + }); +} + +function run(command, args) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, {stdio: ['ignore', 'pipe', 'pipe']}); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk) => (stdout += chunk)); + child.stderr.on('data', (chunk) => { + stderr += chunk; + if (stderr.length > 2_000_000) stderr = stderr.slice(-1_000_000); + }); + child.once('error', reject); + child.once('close', (code) => { + if (code === 0) resolve(stdout || stderr); + else + reject(new ProcessorError(`${command} exited ${code}: ${stderr.slice(-2000)}`)); + }); + }); +} + +async function processRequest(payload) { + const {videoId, attempt} = validatePayload(payload); + const directory = await mkdtemp(path.join(tmpdir(), 'hackweek-video-')); + const input = path.join(directory, 'original'); + const output = path.join(directory, 'canonical.mp4'); + try { + const source = await fetch(`${R2_ORIGIN}/source`, { + headers: {'x-video-id': videoId, 'x-video-attempt': String(attempt)}, + }); + if (!source.ok || !source.body) { + throw new ProcessorError(`Scoped original download failed with ${source.status}`); + } + await pipeline( + Readable.fromWeb(source.body), + createWriteStream(input, {flags: 'wx'}), + ); + const result = await processFile(input, output); + const outputSize = (await stat(output)).size; + const uploaded = await fetch(`${R2_ORIGIN}/output`, { + method: 'PUT', + headers: { + 'content-type': 'video/mp4', + 'content-length': String(outputSize), + 'x-video-id': videoId, + 'x-video-attempt': String(attempt), + 'x-content-sha256': result.sha256, + }, + body: Readable.toWeb(createReadStream(output)), + duplex: 'half', + }); + if (!uploaded.ok) { + throw new ProcessorError(`Scoped derivative upload failed with ${uploaded.status}`); + } + return result; + } finally { + await rm(directory, {recursive: true, force: true}); + } +} + +function validatePayload(value) { + if ( + !value || + typeof value !== 'object' || + typeof value.videoId !== 'string' || + !/^[a-zA-Z0-9-]{1,128}$/.test(value.videoId) || + !Number.isInteger(value.attempt) || + value.attempt < 1 + ) { + throw new ProcessorError('Processor request is invalid'); + } + return {videoId: value.videoId, attempt: value.attempt}; +} + +async function readJson(request) { + let body = ''; + for await (const chunk of request) { + body += chunk; + if (body.length > 16_384) throw new ProcessorError('Processor request is too large'); + } + try { + return JSON.parse(body); + } catch { + throw new ProcessorError('Processor request must contain JSON'); + } +} + +let processingTail = Promise.resolve(); +const server = createServer((request, response) => { + if (request.method === 'GET' && request.url === '/ping') { + response.writeHead(200, {'content-type': 'text/plain'}).end('ok'); + return; + } + if (request.method !== 'POST' || request.url !== '/process') { + response.writeHead(404).end(); + return; + } + const task = processingTail.then(async () => { + try { + const result = await processRequest(await readJson(request)); + response + .writeHead(200, {'content-type': 'application/json'}) + .end(JSON.stringify(result)); + } catch (error) { + const message = + error instanceof Error ? error.message.slice(0, 2000) : 'Unknown error'; + response + .writeHead(error instanceof ProcessorError ? 422 : 500, { + 'content-type': 'application/json', + }) + .end(JSON.stringify({error: message})); + } + }); + processingTail = task.catch(() => undefined); +}); + +if (process.argv[1] === new URL(import.meta.url).pathname) { + if (process.argv[2] === 'process-file') { + const [, , , input, output] = process.argv; + if (!input || !output) throw new Error('Usage: process-file '); + console.log(JSON.stringify(await processFile(input, output))); + } else if (process.argv.length === 2) { + server.listen(PORT, '0.0.0.0', () => + console.log(`video processor listening on ${PORT}`), + ); + } else { + throw new Error('Unknown video processor command'); + } +} + +function finite(value) { + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +function positive(value) { + return Number.isInteger(value) && value > 0; +} diff --git a/scripts/local-video-workflow.ts b/scripts/local-video-workflow.ts new file mode 100644 index 0000000..b8acf36 --- /dev/null +++ b/scripts/local-video-workflow.ts @@ -0,0 +1,469 @@ +#!/usr/bin/env node +import {execFileSync, spawn, type ChildProcess} from 'node:child_process'; +import {createHash} from 'node:crypto'; +import {mkdtemp, readFile, rename, rm, stat, writeFile} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; + +const root = process.cwd(); +const state = await mkdtemp(path.join(tmpdir(), 'hackweek-workflow-smoke-')); +const port = Number(process.env.VIDEO_WORKFLOW_PORT ?? 5201); +const origin = `http://127.0.0.1:${port}`; +const config = path.join(state, 'wrangler.video-workflow.json'); +const source = path.join(state, 'workflow-source.mp4'); +const derivative = path.join(state, 'workflow-derivative.mp4'); +const token = createHash('sha256').update('local-video-workflow').digest('base64url'); +const tokenHash = createHash('sha256').update(token).digest('hex'); +const rootDevVars = path.join(root, '.dev.vars'); +const savedDevVars = path.join(state, '.dev.vars.saved'); +const hadDevVars = await exists(rootDevVars); +let server: ChildProcess | undefined; +const logs: string[] = []; + +try { + await writeFile(config, JSON.stringify(localConfig()), {mode: 0o600}); + const devVars = `APP_ORIGIN="${origin}"\nGOOGLE_CLIENT_ID="local.apps.googleusercontent.com"\nGOOGLE_CLIENT_SECRET="local-secret"\nGOOGLE_REDIRECT_URI="${origin}/api/auth/callback"\nALLOWED_EMAIL_DOMAIN="sentry.io"\nSTREAM_MODE="fake"\nVIDEO_PROCESSOR_CONCURRENCY="1"\nVIDEO_PROCESSING_AUTOSTART="true"\n`; + await writeFile(path.join(state, '.dev.vars'), devVars, {mode: 0o600}); + if (hadDevVars) await rename(rootDevVars, savedDevVars); + await writeFile(rootDevVars, devVars, {mode: 0o600}); + + run('npx', [ + 'wrangler', + 'd1', + 'migrations', + 'apply', + 'hackweek-db', + '--local', + '--persist-to', + state, + '--config', + config, + ]); + const now = Math.floor(Date.now() / 1000); + sql(` + INSERT INTO users + (id, source_uid, google_subject, email, display_name, is_admin) + VALUES + ('workflow-user', 'workflow-user', 'workflow-google-user', + 'workflow@sentry.io', 'Workflow User', 1); + INSERT INTO user_sessions + (token_hash, user_id, expires_at, created_at, last_used_at) + VALUES ('${tokenHash}', 'workflow-user', ${now + 3600}, ${now}, ${now}); + INSERT INTO years (id) VALUES ('9999'); + INSERT INTO groups (id, source_id, year_id, name, creator_id) + VALUES ('workflow-group', 'workflow-group', '9999', 'Workflow Group', 'workflow-user'); + INSERT INTO projects + (id, source_id, year_id, creator_id, group_id, name, summary, kind) + VALUES + ('workflow-project', 'workflow-project', '9999', 'workflow-user', + 'workflow-group', 'Workflow Project', 'Local real Workflow smoke', 'project'); + `); + run('ffmpeg', [ + '-hide_banner', + '-loglevel', + 'error', + '-nostdin', + '-y', + '-f', + 'lavfi', + '-i', + 'testsrc2=size=640x360:rate=24', + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:sample_rate=48000', + '-t', + '2', + '-af', + 'volume=0.05', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a', + 'aac', + '-shortest', + source, + ]); + + server = spawn( + process.execPath, + [ + path.join(root, 'node_modules/vite-plus/bin/vp'), + 'dev', + '--host', + '127.0.0.1', + '--port', + String(port), + '--strictPort', + ], + { + cwd: root, + detached: true, + env: localEnvironment(), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + server.stdout?.on('data', (chunk) => logs.push(String(chunk))); + server.stderr?.on('data', (chunk) => logs.push(String(chunk))); + await waitForServer(); + + const bytes = await readFile(source); + const created = await api('/api/projects/workflow-project/video/upload', { + method: 'POST', + body: JSON.stringify({ + fileName: 'workflow-source.mp4', + fileSize: bytes.byteLength, + contentType: 'video/mp4', + }), + }); + assert( + created.response.status === 201, + 'multipart upload is created through the Worker', + ); + const uploadId = created.body.upload.uploadId as string; + const partResponse = await fetch( + `${origin}/api/projects/workflow-project/video/upload/${uploadId}/parts/1`, + { + method: 'PUT', + headers: authenticatedHeaders({ + Origin: origin, + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(bytes.byteLength), + }), + body: bytes, + }, + ); + const part = (await partResponse.json()) as {part: {partNumber: number; etag: string}}; + assert(partResponse.status === 200, 'real source bytes stream into local R2'); + const completed = await api( + `/api/projects/workflow-project/video/upload/${uploadId}/complete`, + { + method: 'POST', + body: JSON.stringify({parts: [part.part]}), + }, + ); + assert(completed.response.status === 200, 'multipart completion starts processing'); + const videoId = completed.body.video.id as string; + assert( + completed.body.video.status === 'queued', + 'completed upload is initially queued', + ); + const duplicateCompletion = await api( + `/api/projects/workflow-project/video/upload/${uploadId}/complete`, + { + method: 'POST', + body: JSON.stringify({parts: [part.part]}), + }, + ); + assert( + duplicateCompletion.body.video.id === videoId, + 'duplicate completion reuses the deterministic Workflow attempt', + ); + + const ready = await waitForReady(); + assert(ready.status === 'ready', 'real local Workflow conditionally marks video ready'); + assert( + Math.abs(ready.loudnessLufs + 16) <= 0.7, + 'Workflow records normalized loudness', + ); + + await new Promise((resolve) => setTimeout(resolve, 500)); + const workflowEvidence = output('npx', [ + 'wrangler', + 'workflows', + 'instances', + 'describe', + 'hackweek-video-processing-local-smoke', + `video-${videoId}-attempt-1`, + '--local', + '--port', + String(port), + '--config', + config, + ]); + assert( + workflowEvidence.includes('run pinned ffmpeg processor'), + 'local Workflow records FFmpeg step', + ); + assert( + workflowEvidence.toLowerCase().includes('complete'), + 'local Workflow instance completes', + ); + + await stopServer(); + const row = query<{original_r2_key: string; processed_r2_key: string}>( + `SELECT original_r2_key, processed_r2_key FROM project_videos WHERE id = '${videoId}'`, + ); + assert( + row.original_r2_key !== row.processed_r2_key, + 'original and derivative R2 keys differ', + ); + run('npx', [ + 'wrangler', + 'r2', + 'object', + 'get', + `hackweek-videos-local-smoke/${row.processed_r2_key}`, + '--file', + derivative, + '--local', + '--persist-to', + state, + '--config', + config, + ]); + const probe = JSON.parse( + output('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration:stream=codec_type,codec_name,width,height,pix_fmt', + '-of', + 'json', + derivative, + ]), + ) as { + streams: Array<{ + codec_type: string; + codec_name: string; + width?: number; + height?: number; + pix_fmt?: string; + }>; + format: {duration: string}; + }; + const video = probe.streams.find((stream) => stream.codec_type === 'video'); + const audio = probe.streams.find((stream) => stream.codec_type === 'audio'); + assert( + video?.codec_name === 'h264' && video.pix_fmt === 'yuv420p', + 'R2 derivative is H.264 yuv420p', + ); + assert(audio?.codec_name === 'aac', 'R2 derivative contains AAC audio'); + assert( + (video?.width ?? 0) <= 1920 && (video?.height ?? 0) <= 1080, + 'R2 derivative is <=1080p', + ); + assert( + Number(probe.format.duration) <= 600, + 'R2 derivative is within the duration limit', + ); + + console.log('Local Workflow + Container: 14 checks passed'); +} finally { + await stopServer(); + await rm(rootDevVars, {force: true}); + if (hadDevVars) await rename(savedDevVars, rootDevVars); + await rm(state, {recursive: true, force: true}); +} + +function localConfig() { + return { + name: 'hackweek-video-workflow-smoke', + main: path.join(root, 'src/worker/index.ts'), + compatibility_date: '2026-08-03', + assets: { + directory: path.join(root, 'public'), + not_found_handling: 'single-page-application', + binding: 'ASSETS', + run_worker_first: true, + }, + d1_databases: [ + { + binding: 'DB', + database_name: 'hackweek-db', + database_id: 'local', + migrations_dir: path.join(root, 'migrations'), + }, + ], + r2_buckets: [ + {binding: 'ATTACHMENTS', bucket_name: 'hackweek-attachments-local-smoke'}, + {binding: 'VIDEOS', bucket_name: 'hackweek-videos-local-smoke'}, + ], + workflows: [ + { + binding: 'VIDEO_PROCESSING_WORKFLOW', + name: 'hackweek-video-processing-local-smoke', + class_name: 'VideoProcessingWorkflow', + }, + ], + containers: [ + { + name: 'hackweek-video-processor-local-smoke', + class_name: 'VideoProcessorContainer', + image: path.join(root, 'Dockerfile.video-processor'), + image_build_context: root, + max_instances: 1, + instance_type: 'standard-2', + }, + ], + durable_objects: { + bindings: [{name: 'VIDEO_PROCESSOR', class_name: 'VideoProcessorContainer'}], + }, + migrations: [ + {tag: 'video-processor-v1', new_sqlite_classes: ['VideoProcessorContainer']}, + ], + vars: { + APP_ORIGIN: origin, + GOOGLE_CLIENT_ID: 'local.apps.googleusercontent.com', + GOOGLE_CLIENT_SECRET: 'local-secret', + GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, + ALLOWED_EMAIL_DOMAIN: 'sentry.io', + STREAM_MODE: 'fake', + VIDEO_PROCESSOR_CONCURRENCY: '1', + VIDEO_PROCESSING_AUTOSTART: 'true', + }, + }; +} + +function localEnvironment() { + return { + ...process.env, + CLOUDFLARE_VITE_DEV_VARS_PATH: '/dev/null', + HACKWEEK_LOCAL_STATE_PATH: state, + HACKWEEK_WRANGLER_CONFIG: config, + APP_ORIGIN: origin, + GOOGLE_CLIENT_ID: 'local.apps.googleusercontent.com', + GOOGLE_CLIENT_SECRET: 'local-secret', + GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, + ALLOWED_EMAIL_DOMAIN: 'sentry.io', + STREAM_MODE: 'fake', + VIDEO_PROCESSOR_CONCURRENCY: '1', + VIDEO_PROCESSING_AUTOSTART: 'true', + }; +} + +async function waitForServer() { + for (let index = 0; index < 600; index += 1) { + if (server?.exitCode !== null) + throw new Error(`Local server exited:\n${logs.join('')}`); + try { + if ((await fetch(`${origin}/api/health`)).ok) return; + } catch { + // Worker and Container image are still starting. + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`Timed out waiting for local Worker:\n${logs.join('')}`); +} + +async function waitForReady() { + for (let index = 0; index < 300; index += 1) { + const result = await api('/api/projects/workflow-project/video'); + const video = result.body.video as { + status: string; + loudnessLufs: number; + errorMessage: string | null; + }; + if (video.status === 'ready') return video; + if (video.status === 'failed') { + throw new Error( + `Local video processing failed: ${video.errorMessage}\n${logs.join('')}`, + ); + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error(`Timed out waiting for Workflow:\n${logs.join('')}`); +} + +async function api(pathname: string, init: RequestInit = {}) { + const headers = authenticatedHeaders(init.headers); + if (init.body) headers.set('Content-Type', 'application/json'); + if (init.method && init.method !== 'GET') headers.set('Origin', origin); + const response = await fetch(`${origin}${pathname}`, {...init, headers}); + const body = (await response.json()) as any; + if (!response.ok) { + throw new Error( + `${pathname} returned ${response.status}: ${JSON.stringify(body)}\n${logs.join('')}`, + ); + } + return {response, body}; +} + +function authenticatedHeaders(init?: ConstructorParameters[0]) { + const headers = new Headers(init); + headers.set('Cookie', `sentry-hackweek-session=${token}`); + return headers; +} + +function sql(command: string) { + run('npx', [ + 'wrangler', + 'd1', + 'execute', + 'hackweek-db', + '--local', + '--persist-to', + state, + '--config', + config, + '--command', + command, + ]); +} + +function query(command: string) { + const json = output('npx', [ + 'wrangler', + 'd1', + 'execute', + 'hackweek-db', + '--local', + '--persist-to', + state, + '--config', + config, + '--command', + command, + '--json', + ]); + const parsed = JSON.parse(json) as Array<{results: T[]}>; + const row = parsed[0]?.results[0]; + if (!row) throw new Error(`D1 query returned no rows: ${command}`); + return row; +} + +function run(command: string, args: string[]) { + execFileSync(command, args, {cwd: root, env: localEnvironment(), stdio: 'inherit'}); +} + +function output(command: string, args: string[]) { + return execFileSync(command, args, { + cwd: root, + env: localEnvironment(), + encoding: 'utf8', + }); +} + +async function stopServer() { + if (!server?.pid || server.exitCode !== null) return; + try { + process.kill(-server.pid, 'SIGTERM'); + } catch { + return; + } + const exited = await Promise.race([ + new Promise((resolve) => server!.once('exit', () => resolve(true))), + new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), + ]); + if (!exited) { + try { + process.kill(-server.pid, 'SIGKILL'); + } catch { + // The detached process group exited between checks. + } + } +} + +function assert(value: unknown, message: string): asserts value { + if (!value) + throw new Error(`Local Workflow check failed: ${message}\n${logs.join('')}`); + console.log(`✓ ${message}`); +} + +function exists(file: string) { + return stat(file).then( + () => true, + () => false, + ); +} diff --git a/scripts/measure-loudness.ts b/scripts/measure-loudness.ts deleted file mode 100644 index b8d4903..0000000 --- a/scripts/measure-loudness.ts +++ /dev/null @@ -1,91 +0,0 @@ -import {spawn} from 'node:child_process'; -import process from 'node:process'; - -interface QueueItem { - videoId: string; - downloadUrl: string; -} - -const apiUrl = required('VIDEO_API_URL').replace(/\/$/, ''); -const serviceToken = required('VIDEO_SERVICE_TOKEN'); -const queue = await request<{videos: QueueItem[]}>('/api/video-jobs/measurements'); - -for (const video of queue.videos) { - try { - const measurement = await measure(video.downloadUrl); - await request(`/api/video-jobs/measurements/${encodeURIComponent(video.videoId)}`, { - method: 'POST', - body: JSON.stringify(measurement), - }); - console.log(`Measured ${video.videoId}: ${measurement.loudnessLufs} LUFS`); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - await request( - `/api/video-jobs/measurements/${encodeURIComponent(video.videoId)}/failure`, - { - method: 'POST', - body: JSON.stringify({error: message}), - }, - ); - console.error(`Measurement failed for ${video.videoId}: ${message}`); - } -} - -async function measure(url: string) { - const output = await run('ffmpeg', [ - '-hide_banner', - '-nostdin', - '-i', - url, - '-af', - 'loudnorm=print_format=json', - '-f', - 'null', - '-', - ]); - const blocks = [...output.matchAll(/\{[\s\S]*?"input_i"[\s\S]*?\}/g)]; - const json = blocks.at(-1)?.[0]; - if (!json) throw new Error('ffmpeg loudnorm output did not contain JSON'); - const parsed = JSON.parse(json) as {input_i?: string; input_duration?: string}; - const loudnessLufs = Number(parsed.input_i); - const durationSeconds = Number(parsed.input_duration); - if (!Number.isFinite(loudnessLufs) || !Number.isFinite(durationSeconds)) { - throw new Error('ffmpeg returned invalid loudness or duration'); - } - return {loudnessLufs, durationSeconds}; -} - -function run(command: string, args: string[]) { - return new Promise((resolve, reject) => { - const child = spawn(command, args, {stdio: ['ignore', 'ignore', 'pipe']}); - let stderr = ''; - child.stderr.setEncoding('utf8'); - child.stderr.on('data', (chunk: string) => (stderr += chunk)); - child.once('error', reject); - child.once('close', (code) => - code === 0 - ? resolve(stderr) - : reject(new Error(`ffmpeg exited ${code}: ${stderr.slice(-500)}`)), - ); - }); -} - -async function request(path: string, init: RequestInit = {}) { - const response = await fetch(`${apiUrl}${path}`, { - ...init, - headers: new Headers({ - Authorization: `Bearer ${serviceToken}`, - ...(init.body ? {'Content-Type': 'application/json'} : {}), - ...Object.fromEntries(new Headers(init.headers).entries()), - }), - }); - if (!response.ok) - throw new Error(`Video API returned ${response.status}: ${await response.text()}`); - return (response.status === 204 ? undefined : await response.json()) as T; -} - -function required(name: string) { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -} diff --git a/scripts/test-video-processor.ts b/scripts/test-video-processor.ts new file mode 100644 index 0000000..f2bf23f --- /dev/null +++ b/scripts/test-video-processor.ts @@ -0,0 +1,265 @@ +#!/usr/bin/env node +import {execFileSync, spawnSync} from 'node:child_process'; +import {chmod, mkdtemp, readFile, rm, writeFile} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; + +const root = process.cwd(); +const image = 'hackweek-video-processor:local'; +const work = await mkdtemp(path.join(tmpdir(), 'hackweek-processor-test-')); +const uid = process.getuid?.() ?? 1000; +const gid = process.getgid?.() ?? 1000; + +try { + run('docker', ['build', '--file', 'Dockerfile.video-processor', '--tag', image, '.']); + const version = output('docker', [ + 'run', + '--rm', + '--entrypoint', + 'ffmpeg', + image, + '-version', + ]); + assert(version.startsWith('ffmpeg version 8.0.1 '), 'pinned FFmpeg 8.0.1 runs'); + + await chmod(work, 0o777); + const audible = path.join(work, 'audible.mp4'); + const lowSilent = path.join(work, 'low-silent.mp4'); + const rotationBase = path.join(work, 'rotation-base.mp4'); + const rotated = path.join(work, 'rotated.mp4'); + const overDuration = path.join(work, 'over-duration.mp4'); + const malformed = path.join(work, 'malformed.mp4'); + + ffmpeg([ + '-f', + 'lavfi', + '-i', + 'testsrc2=size=1280x720:rate=24', + '-f', + 'lavfi', + '-i', + 'sine=frequency=1000:sample_rate=48000', + '-t', + '2', + '-af', + 'volume=0.05', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a', + 'aac', + '-shortest', + audible, + ]); + ffmpeg([ + '-f', + 'lavfi', + '-i', + 'testsrc2=size=320x180:rate=15', + '-t', + '1.5', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-an', + lowSilent, + ]); + ffmpeg([ + '-f', + 'lavfi', + '-i', + 'testsrc2=size=320x180:rate=15', + '-t', + '1', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-an', + rotationBase, + ]); + ffmpeg(['-display_rotation:v:0', '90', '-i', rotationBase, '-c', 'copy', rotated]); + ffmpeg([ + '-f', + 'lavfi', + '-i', + 'color=size=64x64:rate=1:color=black', + '-t', + '601', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-an', + overDuration, + ]); + await writeFile(malformed, 'not a media file'); + await Promise.all( + [audible, lowSilent, rotationBase, rotated, overDuration, malformed].map((file) => + chmod(file, 0o644), + ), + ); + + const audibleResult = processFixture('audible.mp4', 'audible-output.mp4'); + const audibleProbe = probe(path.join(work, 'audible-output.mp4')); + assertCanonical(audibleProbe); + assert( + audibleResult.audioMode === 'normalized', + 'audible input uses two-pass loudnorm', + ); + assert( + typeof audibleResult.loudnessLufs === 'number' && + Math.abs(audibleResult.loudnessLufs + 16) <= 0.7, + `audible output is ${String(audibleResult.loudnessLufs)} LUFS within ±0.7 LU`, + ); + assert( + await fastStart(path.join(work, 'audible-output.mp4')), + 'MP4 moov precedes mdat', + ); + + const silentResult = processFixture('low-silent.mp4', 'low-silent-output.mp4'); + const silentProbe = probe(path.join(work, 'low-silent-output.mp4')); + assertCanonical(silentProbe); + const silentVideo = silentProbe.streams.find( + (stream) => stream.codec_type === 'video', + )!; + assert( + silentVideo.width === 320 && silentVideo.height === 180, + 'low-resolution input is not upscaled', + ); + assert( + silentResult.audioMode === 'generated-silence', + 'input without audio receives deterministic AAC silence', + ); + + const rotatedResult = processFixture('rotated.mp4', 'rotated-output.mp4'); + assert( + rotatedResult.width === 180 && rotatedResult.height === 320, + 'rotation metadata is applied without upscaling', + ); + assertCanonical(probe(path.join(work, 'rotated-output.mp4'))); + + expectFixtureFailure('malformed.mp4', 'malformed-output.mp4', 'ffprobe exited'); + expectFixtureFailure('over-duration.mp4', 'over-output.mp4', 'exceeds 600s'); + expectFixtureFailure('audible.mp4', 'missing/output.mp4', 'No such file'); + + console.log('Video processor: 28 checks passed'); +} finally { + await rm(work, {recursive: true, force: true}); +} + +function processFixture(input: string, outputFile: string) { + const result = spawnSync( + 'docker', + [ + 'run', + '--rm', + '--user', + `${uid}:${gid}`, + '--volume', + `${work}:/work`, + image, + 'process-file', + `/work/${input}`, + `/work/${outputFile}`, + ], + {cwd: root, encoding: 'utf8'}, + ); + if (result.status !== 0) { + throw new Error(`Processor failed:\n${result.stdout}\n${result.stderr}`); + } + return JSON.parse(result.stdout.trim().split('\n').at(-1)!) as { + width: number; + height: number; + loudnessLufs: number | null; + audioMode: string; + }; +} + +function expectFixtureFailure(input: string, outputFile: string, message: string) { + const result = spawnSync( + 'docker', + [ + 'run', + '--rm', + '--user', + `${uid}:${gid}`, + '--volume', + `${work}:/work`, + image, + 'process-file', + `/work/${input}`, + `/work/${outputFile}`, + ], + {cwd: root, encoding: 'utf8'}, + ); + assert(result.status !== 0, `${input} is rejected deterministically`); + assert( + `${result.stdout}\n${result.stderr}`.includes(message), + `${input} reports ${message}`, + ); +} + +function assertCanonical(result: Probe) { + const video = result.streams.find((stream) => stream.codec_type === 'video'); + const audio = result.streams.find((stream) => stream.codec_type === 'audio'); + assert(video?.codec_name === 'h264', 'output video codec is H.264'); + assert(video?.pix_fmt === 'yuv420p', 'output pixel format is yuv420p'); + assert( + (video?.width ?? 0) <= 1920 && (video?.height ?? 0) <= 1080, + 'output is <=1080p', + ); + assert(audio?.codec_name === 'aac', 'output audio codec is AAC'); + assert(Number(result.format.duration) <= 600.05, 'output duration is <=600 seconds'); +} + +function probe(file: string) { + return JSON.parse( + output('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration:stream=codec_type,codec_name,width,height,pix_fmt', + '-of', + 'json', + file, + ]), + ) as Probe; +} + +interface Probe { + streams: Array<{ + codec_type: string; + codec_name?: string; + width?: number; + height?: number; + pix_fmt?: string; + }>; + format: {duration: string}; +} + +async function fastStart(file: string) { + const bytes = await readFile(file); + const moov = bytes.indexOf(Buffer.from('moov')); + const mdat = bytes.indexOf(Buffer.from('mdat')); + return moov >= 0 && mdat >= 0 && moov < mdat; +} + +function ffmpeg(args: string[]) { + run('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-nostdin', '-y', ...args]); +} + +function run(command: string, args: string[]) { + execFileSync(command, args, {cwd: root, stdio: 'inherit'}); +} + +function output(command: string, args: string[]) { + return execFileSync(command, args, {cwd: root, encoding: 'utf8'}); +} + +function assert(value: unknown, message: string): asserts value { + if (!value) throw new Error(`Video processor check failed: ${message}`); + console.log(`✓ ${message}`); +} diff --git a/src/worker/containers/video-processor.ts b/src/worker/containers/video-processor.ts new file mode 100644 index 0000000..0d7dfbb --- /dev/null +++ b/src/worker/containers/video-processor.ts @@ -0,0 +1,105 @@ +import { + Container, + type OutboundHandler, + type OutboundHandlerContext, +} from '@cloudflare/containers'; + +import type {VideoProcessingParams} from '../video-processing'; + +interface ProcessorContainerEnv { + DB: D1Database; + VIDEOS: R2Bucket; +} + +interface ProcessingStorageRow { + original_r2_key: string; + output_r2_key: string; +} + +export class VideoProcessorContainer extends Container { + defaultPort = 8080; + requiredPorts = [8080]; + sleepAfter = '5m'; + enableInternet = false; + allowedHosts = ['video-r2']; +} + +const videoR2Handler: OutboundHandler< + ProcessorContainerEnv, + VideoProcessingParams +> = async (request, env, ctx) => { + const scope = requireScope(request, ctx); + const storage = await currentStorage(env.DB, scope); + if (!storage || storage.original_r2_key === storage.output_r2_key) { + return new Response('Processing attempt is no longer current', {status: 409}); + } + + const pathname = new URL(request.url).pathname; + if (request.method === 'GET' && pathname === '/source') { + const object = await env.VIDEOS.get(storage.original_r2_key); + if (!object) return new Response('Immutable original is missing', {status: 404}); + const headers = new Headers({'content-length': String(object.size)}); + object.writeHttpMetadata(headers); + return new Response(object.body, {headers}); + } + if (request.method === 'PUT' && pathname === '/output') { + if (!request.body) return new Response('Derivative body is required', {status: 400}); + const checksum = request.headers.get('x-content-sha256'); + if (!checksum || !/^[a-f0-9]{64}$/.test(checksum)) { + return new Response('Derivative checksum is invalid', {status: 400}); + } + const existing = await env.VIDEOS.head(storage.output_r2_key); + if (existing) { + return existing.customMetadata?.sha256 === checksum + ? new Response(null, {status: 204}) + : new Response('Immutable derivative already exists', {status: 409}); + } + try { + await env.VIDEOS.put(storage.output_r2_key, request.body, { + httpMetadata: {contentType: 'video/mp4'}, + customMetadata: { + videoId: scope.videoId, + attempt: String(scope.attempt), + sha256: checksum, + }, + }); + return new Response(null, {status: 201}); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return new Response(`Derivative R2 write failed: ${message}`, {status: 500}); + } + } + return new Response('Scoped video storage route not found', {status: 404}); +}; + +VideoProcessorContainer.outboundHandlers = {videoR2: videoR2Handler}; + +async function currentStorage(db: D1Database, scope: VideoProcessingParams) { + return db + .prepare( + `SELECT pv.original_r2_key, vpa.output_r2_key + FROM project_videos pv + JOIN video_processing_attempts vpa + ON vpa.video_id = pv.id AND vpa.attempt = ? + WHERE pv.id = ? AND pv.processing_attempt = ? + AND pv.status = 'processing' AND pv.retired_at IS NULL + AND vpa.status = 'running' AND vpa.output_r2_key IS NOT NULL`, + ) + .bind(scope.attempt, scope.videoId, scope.attempt) + .first(); +} + +function requireScope( + request: Request, + ctx: OutboundHandlerContext, +) { + const scope = ctx.params; + if ( + !scope || + request.headers.get('x-video-id') !== scope.videoId || + request.headers.get('x-video-attempt') !== String(scope.attempt) + ) { + throw new Error('Container request escaped its processing-attempt scope'); + } + return scope; +} diff --git a/src/worker/index.ts b/src/worker/index.ts index da008b0..3920f86 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -1,5 +1,7 @@ +import {ContainerProxy} from '@cloudflare/containers'; import {Hono} from 'hono'; +import type {VideoProcessorContainer} from './containers/video-processor'; import type {AuthBindings, AuthVariables} from './middleware/auth'; import {authenticateRequest, protectMutationOrigin} from './middleware/auth'; import {requireRole} from './middleware/user'; @@ -14,6 +16,11 @@ import {sessionRoutes} from './routes/session'; import {projectVideoRoutes, videosRoutes} from './routes/videos'; import {votesRoutes} from './routes/votes'; import {yearsRoutes} from './routes/years'; +import type {VideoProcessingParams} from './video-processing'; + +export {ContainerProxy}; +export {VideoProcessorContainer} from './containers/video-processor'; +export {VideoProcessingWorkflow} from './workflows/video-processing'; export interface VideoBindings { VIDEOS: R2Bucket; @@ -24,10 +31,10 @@ export interface VideoBindings { STREAM_ALLOWED_ORIGIN?: string; STREAM_DELIVERY_HOST?: string; VIDEO_SERVICE_TOKEN?: string; - R2_ACCOUNT_ID?: string; - R2_BUCKET_NAME?: string; - R2_ACCESS_KEY_ID?: string; - R2_SECRET_ACCESS_KEY?: string; + VIDEO_PROCESSING_WORKFLOW: Workflow; + VIDEO_PROCESSOR: DurableObjectNamespace; + VIDEO_PROCESSOR_CONCURRENCY: string; + VIDEO_PROCESSING_AUTOSTART: string; } export type WorkerEnv = { diff --git a/src/worker/routes/videos.ts b/src/worker/routes/videos.ts index c50c99f..520d3fb 100644 --- a/src/worker/routes/videos.ts +++ b/src/worker/routes/videos.ts @@ -17,6 +17,7 @@ import { getVideoUpload, MAX_VIDEO_BYTES, retireProjectVideo, + retryProjectVideo, uploadVideoPart, } from '../services/videos'; @@ -97,6 +98,9 @@ projectVideoRoutes.post('/:projectId/video/upload/:uploadId/complete', async (c) const video = await completeVideoUpload( c.env.DB, c.env.VIDEOS, + String(c.env.VIDEO_PROCESSING_AUTOSTART) === 'false' + ? null + : c.env.VIDEO_PROCESSING_WORKFLOW, c.req.param('projectId'), c.req.param('uploadId'), input.parts, @@ -108,6 +112,22 @@ projectVideoRoutes.post('/:projectId/video/upload/:uploadId/complete', async (c) } }); +projectVideoRoutes.post('/:projectId/video/retry', async (c) => { + try { + const video = await retryProjectVideo( + c.env.DB, + String(c.env.VIDEO_PROCESSING_AUTOSTART) === 'false' + ? null + : c.env.VIDEO_PROCESSING_WORKFLOW, + c.req.param('projectId'), + c.get('user'), + ); + return c.json({video}, 202, {'Cache-Control': 'private, no-store'}); + } catch (error) { + return respondError(c, error); + } +}); + projectVideoRoutes.post('/:projectId/video/promote', (c) => c.json( { diff --git a/src/worker/services/videos.ts b/src/worker/services/videos.ts index 4392b45..feaa38c 100644 --- a/src/worker/services/videos.ts +++ b/src/worker/services/videos.ts @@ -5,6 +5,8 @@ import type { VideoUploadSession, } from '../../shared/videos'; import {currentYearIdSql, effectiveYearFlags} from '../repositories/years'; +import type {VideoProcessingParams, VideoProcessorResult} from '../video-processing'; +import {videoWorkflowInstanceId} from '../video-processing'; import {ServiceError} from './errors'; export const MAX_VIDEO_BYTES = 5 * 1024 * 1024 * 1024; @@ -26,6 +28,17 @@ interface VideoRow { created_at: string; } +interface ProcessingAttemptRow { + video_id: string; + project_id: string; + original_r2_key: string; + processing_attempt: number; + video_status: string; + attempt_status: string; +} + +export class ProcessingCapacityError extends Error {} + interface UploadRow { id: string; video_id: string; @@ -221,6 +234,7 @@ export async function uploadVideoPart( export async function completeVideoUpload( db: D1Database, bucket: R2Bucket, + workflow: Workflow | null, projectId: string, uploadId: string, suppliedParts: Array<{partNumber: number; etag: string}>, @@ -230,7 +244,11 @@ export async function completeVideoUpload( await authorizeVideoWrite(db, projectId, user); const upload = await requireUpload(db, projectId, uploadId); if (upload.status === 'completed') { - return mapVideo(await requireVideoById(db, upload.video_id)); + const video = await requireVideoById(db, upload.video_id); + if (workflow) { + await ensureVideoProcessingWorkflow(workflow, video.id, video.processing_attempt); + } + return mapVideo(video); } await assertUploadIsWritable(db, bucket, upload, now); if (!upload.r2_upload_id || !['uploading', 'completing'].includes(upload.status)) { @@ -319,7 +337,11 @@ export async function completeVideoUpload( .first(); if (!existing) throw error; } - return mapVideo(await requireVideoById(db, upload.video_id)); + const video = await requireVideoById(db, upload.video_id); + if (workflow) { + await ensureVideoProcessingWorkflow(workflow, video.id, video.processing_attempt); + } + return mapVideo(video); } export async function abortVideoUpload( @@ -362,6 +384,192 @@ export async function abortVideoUpload( .run(); } +export async function retryProjectVideo( + db: D1Database, + workflow: Workflow | null, + projectId: string, + user: SessionUser, +) { + await authorizeVideoWrite(db, projectId, user); + const video = await db + .prepare(`${videoSelect()} WHERE project_id = ? AND retired_at IS NULL`) + .bind(projectId) + .first(); + if (!video) throw new ServiceError('NOT_FOUND', 'Video not found', 404); + if (video.status !== 'failed') { + throw new ServiceError('CONFLICT', 'Only a failed video can be retried', 409); + } + const attempt = video.processing_attempt + 1; + const results = await db.batch([ + db + .prepare( + `UPDATE project_videos SET status = 'queued', processing_attempt = ?, + duration_seconds = NULL, loudness_lufs = NULL, gain_db = NULL, + error_message = NULL, processed_r2_key = NULL, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND processing_attempt = ? AND status = 'failed' + AND retired_at IS NULL`, + ) + .bind(attempt, video.id, video.processing_attempt), + db + .prepare( + `INSERT INTO video_processing_attempts (video_id, attempt, status) + SELECT ?, ?, 'queued' WHERE EXISTS ( + SELECT 1 FROM project_videos WHERE id = ? + AND processing_attempt = ? AND status = 'queued' AND retired_at IS NULL + )`, + ) + .bind(video.id, attempt, video.id, attempt), + ]); + if (results.some((result) => result.meta.changes !== 1)) { + throw new ServiceError('CONFLICT', 'Video retry was superseded', 409); + } + if (workflow) await ensureVideoProcessingWorkflow(workflow, video.id, attempt); + return mapVideo(await requireVideoById(db, video.id)); +} + +export async function claimVideoProcessingAttempt( + db: D1Database, + videoId: string, + attempt: number, + concurrency: number, +): Promise<{status: 'claimed'; outputKey: string} | {status: 'stale'}> { + const row = await processingAttempt(db, videoId, attempt); + if ( + !row || + row.processing_attempt !== attempt || + row.video_status !== 'queued' || + row.attempt_status !== 'queued' + ) { + return {status: 'stale'}; + } + const running = await db + .prepare( + `SELECT COUNT(*) count FROM video_processing_attempts WHERE status = 'running'`, + ) + .first<{count: number}>(); + if ((running?.count ?? 0) >= concurrency) { + throw new ProcessingCapacityError('Video processor concurrency is currently full'); + } + const outputKey = videoProcessedKey(row.project_id, videoId, attempt); + const results = await db.batch([ + db + .prepare( + `UPDATE video_processing_attempts + SET status = 'running', output_r2_key = ?, started_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE video_id = ? AND attempt = ? AND status = 'queued' + AND (SELECT COUNT(*) FROM video_processing_attempts WHERE status = 'running') < ? + AND EXISTS ( + SELECT 1 FROM project_videos WHERE id = ? AND processing_attempt = ? + AND status = 'queued' AND retired_at IS NULL + )`, + ) + .bind(outputKey, videoId, attempt, concurrency, videoId, attempt), + db + .prepare( + `UPDATE project_videos SET status = 'processing', error_message = NULL, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND processing_attempt = ? AND status = 'queued' + AND retired_at IS NULL AND EXISTS ( + SELECT 1 FROM video_processing_attempts + WHERE video_id = ? AND attempt = ? AND status = 'running' + AND output_r2_key = ? + )`, + ) + .bind(videoId, attempt, videoId, attempt, outputKey), + ]); + if (results.every((result) => result.meta.changes === 1)) { + return {status: 'claimed', outputKey}; + } + const current = await processingAttempt(db, videoId, attempt); + if (current?.video_status === 'queued' && current.attempt_status === 'queued') { + throw new ProcessingCapacityError('Video processor concurrency is currently full'); + } + return {status: 'stale'}; +} + +export async function publishVideoProcessingAttempt( + db: D1Database, + videoId: string, + attempt: number, + outputKey: string, + result: VideoProcessorResult, +) { + const updates = await db.batch([ + db + .prepare( + `UPDATE project_videos SET status = 'ready', processed_r2_key = ?, + duration_seconds = ?, loudness_lufs = ?, gain_db = 0, + error_message = NULL, updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND processing_attempt = ? AND status = 'processing' + AND retired_at IS NULL AND processed_r2_key IS NULL + AND EXISTS ( + SELECT 1 FROM video_processing_attempts + WHERE video_id = ? AND attempt = ? AND status = 'running' + AND output_r2_key = ? + )`, + ) + .bind( + outputKey, + result.durationSeconds, + result.loudnessLufs, + videoId, + attempt, + videoId, + attempt, + outputKey, + ), + db + .prepare( + `UPDATE video_processing_attempts SET status = 'succeeded', + finished_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP + WHERE video_id = ? AND attempt = ? AND status = 'running' + AND output_r2_key = ? AND EXISTS ( + SELECT 1 FROM project_videos WHERE id = ? AND processing_attempt = ? + AND status = 'ready' AND retired_at IS NULL AND processed_r2_key = ? + )`, + ) + .bind(videoId, attempt, outputKey, videoId, attempt, outputKey), + ]); + return updates.every((update) => update.meta.changes === 1); +} + +export async function failVideoProcessingAttempt( + db: D1Database, + videoId: string, + attempt: number, + error: string, +) { + const message = boundedProcessingError(error); + const updates = await db.batch([ + db + .prepare( + `UPDATE project_videos SET status = 'failed', error_message = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND processing_attempt = ? AND retired_at IS NULL + AND status IN ('queued', 'processing') + AND EXISTS ( + SELECT 1 FROM video_processing_attempts + WHERE video_id = ? AND attempt = ? AND status IN ('queued', 'running') + )`, + ) + .bind(message, videoId, attempt, videoId, attempt), + db + .prepare( + `UPDATE video_processing_attempts SET status = 'failed', error_message = ?, + finished_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP + WHERE video_id = ? AND attempt = ? AND status IN ('queued', 'running') + AND EXISTS ( + SELECT 1 FROM project_videos WHERE id = ? AND processing_attempt = ? + AND status = 'failed' AND retired_at IS NULL + )`, + ) + .bind(message, videoId, attempt, videoId, attempt), + ]); + return updates.every((update) => update.meta.changes === 1); +} + export async function retireProjectVideo( db: D1Database, projectId: string, @@ -587,6 +795,53 @@ function isVideoSlotConflict(error: unknown) { ); } +async function processingAttempt(db: D1Database, videoId: string, attempt: number) { + return db + .prepare( + `SELECT pv.id video_id, pv.project_id, pv.original_r2_key, + pv.processing_attempt, pv.status video_status, vpa.status attempt_status + FROM project_videos pv + JOIN video_processing_attempts vpa + ON vpa.video_id = pv.id AND vpa.attempt = ? + WHERE pv.id = ?`, + ) + .bind(attempt, videoId) + .first(); +} + +async function ensureVideoProcessingWorkflow( + workflow: Workflow, + videoId: string, + attempt: number, +) { + const id = videoWorkflowInstanceId(videoId, attempt); + try { + await workflow.create({ + id, + params: {videoId, attempt}, + retention: {successRetention: '30 days', errorRetention: '30 days'}, + }); + } catch (error) { + try { + const existing = await workflow.get(id); + const status = await existing.status(); + if (status.status !== 'unknown') return; + } catch { + // Preserve the original create failure when no deterministic instance exists. + } + throw error; + } +} + +function boundedProcessingError(error: string) { + const normalized = error.replace(/\s+/g, ' ').trim(); + return (normalized || 'Video processing failed').slice(0, 500); +} + +function videoProcessedKey(projectId: string, videoId: string, attempt: number) { + return `projects/${encodeURIComponent(projectId)}/videos/${videoId}/processed/attempt-${attempt}.mp4`; +} + function videoOriginalKey(projectId: string, videoId: string, fileName: string) { const safeName = fileName .normalize('NFKC') diff --git a/src/worker/video-processing.ts b/src/worker/video-processing.ts new file mode 100644 index 0000000..970de76 --- /dev/null +++ b/src/worker/video-processing.ts @@ -0,0 +1,27 @@ +export const VIDEO_PROCESSOR_TARGET_LUFS = -16; +export const VIDEO_PROCESSOR_LOUDNESS_TOLERANCE_LU = 0.7; +export const VIDEO_PROCESSOR_MAX_DURATION_SECONDS = 600; + +export interface VideoProcessingParams { + videoId: string; + attempt: number; +} + +export interface VideoProcessorResult { + durationSeconds: number; + width: number; + height: number; + videoCodec: 'h264'; + audioCodec: 'aac'; + pixelFormat: 'yuv420p'; + loudnessLufs: number | null; + loudnessTargetLufs: number; + loudnessToleranceLu: number; + audioMode: 'normalized' | 'generated-silence'; + fastStart: true; + sha256: string; +} + +export function videoWorkflowInstanceId(videoId: string, attempt: number) { + return `video-${videoId}-attempt-${attempt}`; +} diff --git a/src/worker/workflows/video-processing.ts b/src/worker/workflows/video-processing.ts new file mode 100644 index 0000000..b7cd392 --- /dev/null +++ b/src/worker/workflows/video-processing.ts @@ -0,0 +1,165 @@ +import {getContainer} from '@cloudflare/containers'; +import { + WorkflowEntrypoint, + type WorkflowEvent, + type WorkflowStep, +} from 'cloudflare:workers'; + +import {VideoProcessorContainer} from '../containers/video-processor'; +import { + claimVideoProcessingAttempt, + failVideoProcessingAttempt, + publishVideoProcessingAttempt, +} from '../services/videos'; +import { + VIDEO_PROCESSOR_LOUDNESS_TOLERANCE_LU, + VIDEO_PROCESSOR_MAX_DURATION_SECONDS, + VIDEO_PROCESSOR_TARGET_LUFS, + type VideoProcessingParams, + type VideoProcessorResult, +} from '../video-processing'; + +export interface VideoProcessingEnvironment { + DB: D1Database; + VIDEOS: R2Bucket; + VIDEO_PROCESSOR: DurableObjectNamespace; + VIDEO_PROCESSOR_CONCURRENCY: string; +} + +export class VideoProcessingWorkflow extends WorkflowEntrypoint< + VideoProcessingEnvironment, + VideoProcessingParams +> { + async run(event: WorkflowEvent, step: WorkflowStep) { + const {videoId, attempt} = event.payload; + let claim: Awaited>; + try { + claim = await step.do( + 'claim current processing attempt', + { + retries: {limit: 360, delay: '5 seconds', backoff: 'constant'}, + timeout: '30 seconds', + }, + () => + claimVideoProcessingAttempt( + this.env.DB, + videoId, + attempt, + processingConcurrency(this.env.VIDEO_PROCESSOR_CONCURRENCY), + ), + ); + } catch (error) { + await step.do('record claim failure', () => + failVideoProcessingAttempt(this.env.DB, videoId, attempt, errorMessage(error)), + ); + return {status: 'failed', stage: 'claim'}; + } + if (claim.status === 'stale') return {status: 'stale'}; + + let result: VideoProcessorResult; + try { + result = await step.do( + 'run pinned ffmpeg processor', + { + retries: {limit: 1, delay: '2 seconds', backoff: 'constant'}, + timeout: '30 minutes', + }, + async () => { + const container = getContainer( + this.env.VIDEO_PROCESSOR, + `${videoId}-attempt-${attempt}`, + ); + await container.setOutboundByHost('video-r2', 'videoR2', {videoId, attempt}); + const response = await container.fetch('http://container/process', { + method: 'POST', + headers: {'content-type': 'application/json'}, + body: JSON.stringify({videoId, attempt}), + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`FFmpeg processor returned ${response.status}: ${body}`); + } + return parseProcessorResult(await response.json()); + }, + ); + } catch (error) { + await step.do('record processor failure', () => + failVideoProcessingAttempt(this.env.DB, videoId, attempt, errorMessage(error)), + ); + return {status: 'failed', stage: 'processor'}; + } + + const published = await step.do('publish only if attempt is current', () => + publishVideoProcessingAttempt( + this.env.DB, + videoId, + attempt, + claim.outputKey, + result, + ), + ); + return {status: published ? 'ready' : 'stale', result}; + } +} + +function processingConcurrency(value: string) { + const concurrency = Number(value); + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 2) { + throw new Error('VIDEO_PROCESSOR_CONCURRENCY must be 1 or 2'); + } + return concurrency; +} + +function parseProcessorResult(value: unknown): VideoProcessorResult { + if (!value || typeof value !== 'object') throw invalidProcessorResult(); + const result = value as Record; + const loudness = result.loudnessLufs; + if ( + !finitePositive(result.durationSeconds) || + (result.durationSeconds as number) > VIDEO_PROCESSOR_MAX_DURATION_SECONDS + 0.05 || + !integerBetween(result.width, 1, 1920) || + !integerBetween(result.height, 1, 1080) || + result.videoCodec !== 'h264' || + result.audioCodec !== 'aac' || + result.pixelFormat !== 'yuv420p' || + result.fastStart !== true || + result.loudnessTargetLufs !== VIDEO_PROCESSOR_TARGET_LUFS || + result.loudnessToleranceLu !== VIDEO_PROCESSOR_LOUDNESS_TOLERANCE_LU || + !['normalized', 'generated-silence'].includes(String(result.audioMode)) || + typeof result.sha256 !== 'string' || + !/^[a-f0-9]{64}$/.test(result.sha256) || + (loudness !== null && typeof loudness !== 'number') + ) { + throw invalidProcessorResult(); + } + if ( + result.audioMode === 'normalized' && + (typeof loudness !== 'number' || + !Number.isFinite(loudness) || + Math.abs(loudness - VIDEO_PROCESSOR_TARGET_LUFS) > + VIDEO_PROCESSOR_LOUDNESS_TOLERANCE_LU) + ) { + throw invalidProcessorResult(); + } + return result as unknown as VideoProcessorResult; +} + +function finitePositive(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) && value > 0; +} + +function integerBetween(value: unknown, minimum: number, maximum: number) { + return ( + Number.isInteger(value) && + (value as number) >= minimum && + (value as number) <= maximum + ); +} + +function invalidProcessorResult() { + return new Error('FFmpeg processor returned invalid canonical metadata'); +} + +function errorMessage(error: unknown) { + return error instanceof Error ? error.message : String(error); +} diff --git a/test/video/video.test.ts b/test/video/video.test.ts index 172a24d..6e803f0 100644 --- a/test/video/video.test.ts +++ b/test/video/video.test.ts @@ -2,7 +2,15 @@ import {env, SELF} from 'cloudflare:test'; import {beforeEach, describe, expect, it} from 'vitest'; import type {ProjectWriteRequest} from '../../src/shared/projects'; -import {MAX_VIDEO_BYTES, VIDEO_PART_SIZE} from '../../src/worker/services/videos'; +import { + claimVideoProcessingAttempt, + failVideoProcessingAttempt, + MAX_VIDEO_BYTES, + ProcessingCapacityError, + publishVideoProcessingAttempt, + VIDEO_PART_SIZE, +} from '../../src/worker/services/videos'; +import type {VideoProcessorResult} from '../../src/worker/video-processing'; import {createSessionCookie} from '../auth/fixture'; const base = 'https://hackweek.test/api'; @@ -268,8 +276,144 @@ describe('R2 multipart video lifecycle', () => { expect(first.status).toBe(204); expect(duplicate.status).toBe(204); }); + + it('conditionally publishes canonical metadata exactly once for the current attempt', async () => { + const {video, key: originalKey} = await completeSmallUpload( + projectId, + ownerToken, + 'canonical source', + ); + const claim = await claimVideoProcessingAttempt(env.DB, video.id, 1, 1); + expect(claim.status).toBe('claimed'); + if (claim.status !== 'claimed') throw new Error('attempt was not claimed'); + expect(claim.outputKey).not.toBe(originalKey); + + expect( + await publishVideoProcessingAttempt( + env.DB, + video.id, + 1, + claim.outputKey, + canonicalResult, + ), + ).toBe(true); + expect( + await publishVideoProcessingAttempt( + env.DB, + video.id, + 1, + claim.outputKey, + canonicalResult, + ), + ).toBe(false); + const stored = await env.DB.prepare( + `SELECT status, original_r2_key, processed_r2_key, duration_seconds, + loudness_lufs, processing_attempt FROM project_videos WHERE id = ?`, + ) + .bind(video.id) + .first(); + expect(stored).toEqual({ + status: 'ready', + original_r2_key: originalKey, + processed_r2_key: claim.outputKey, + duration_seconds: canonicalResult.durationSeconds, + loudness_lufs: canonicalResult.loudnessLufs, + processing_attempt: 1, + }); + }); + + it('fences retirement, failure, retry, and late attempt completion while retaining bytes', async () => { + const {video, key: originalKey} = await completeSmallUpload( + projectId, + ownerToken, + 'retry source', + ); + const first = await claimVideoProcessingAttempt(env.DB, video.id, 1, 1); + if (first.status !== 'claimed') throw new Error('attempt was not claimed'); + await env.VIDEOS.put(first.outputKey, 'retained stale derivative'); + expect(await failVideoProcessingAttempt(env.DB, video.id, 1, 'ffmpeg failed')).toBe( + true, + ); + + const retried = await api(`/projects/${projectId}/video/retry`, ownerToken, { + method: 'POST', + }); + expect(retried.status).toBe(202); + expect(retried.body.video).toMatchObject({status: 'queued', processingAttempt: 2}); + const second = await claimVideoProcessingAttempt(env.DB, video.id, 2, 1); + if (second.status !== 'claimed') throw new Error('retry was not claimed'); + expect(second.outputKey).not.toBe(first.outputKey); + expect( + await publishVideoProcessingAttempt( + env.DB, + video.id, + 1, + first.outputKey, + canonicalResult, + ), + ).toBe(false); + + await env.VIDEOS.put(second.outputKey, 'retained current derivative'); + const retired = await api(`/projects/${projectId}/video`, ownerToken, { + method: 'DELETE', + body: {confirmed: true}, + }); + expect(retired.status).toBe(204); + expect( + await publishVideoProcessingAttempt( + env.DB, + video.id, + 2, + second.outputKey, + canonicalResult, + ), + ).toBe(false); + expect(await env.VIDEOS.head(originalKey)).not.toBeNull(); + expect(await env.VIDEOS.head(first.outputKey)).not.toBeNull(); + expect(await env.VIDEOS.head(second.outputKey)).not.toBeNull(); + }); + + it('limits local processing to one while independent projects remain queued', async () => { + const leftProject = await createProject('Processor left'); + const rightProject = await createProject('Processor right'); + const left = await completeSmallUpload(leftProject, ownerToken, 'left source'); + const right = await completeSmallUpload(rightProject, ownerToken, 'right source'); + + const claimed = await claimVideoProcessingAttempt(env.DB, left.video.id, 1, 1); + expect(claimed.status).toBe('claimed'); + await expect( + claimVideoProcessingAttempt(env.DB, right.video.id, 1, 1), + ).rejects.toBeInstanceOf(ProcessingCapacityError); + expect( + await env.DB.prepare('SELECT status FROM project_videos WHERE id = ?') + .bind(right.video.id) + .first('status'), + ).toBe('queued'); + + await failVideoProcessingAttempt(env.DB, left.video.id, 1, 'fixture release'); + expect(await claimVideoProcessingAttempt(env.DB, right.video.id, 1, 1)).toMatchObject( + { + status: 'claimed', + }, + ); + }); }); +const canonicalResult: VideoProcessorResult = { + durationSeconds: 2, + width: 1280, + height: 720, + videoCodec: 'h264', + audioCodec: 'aac', + pixelFormat: 'yuv420p', + loudnessLufs: -16, + loudnessTargetLufs: -16, + loudnessToleranceLu: 0.7, + audioMode: 'normalized', + fastStart: true, + sha256: 'a'.repeat(64), +}; + async function completeSmallUpload(project: string, token: string, bytes: string) { const created = await createUpload(project, token, bytes.length); expect(created.status).toBe(201); diff --git a/vitest.config.ts b/vitest.config.ts index 05871a7..4af6ade 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -29,6 +29,7 @@ export default defineConfig({ STREAM_DELIVERY_HOST: 'customer-fake.cloudflarestream.com', STREAM_WEBHOOK_SECRET: 'test-webhook-secret', VIDEO_SERVICE_TOKEN: 'test-video-service-token', + VIDEO_PROCESSING_AUTOSTART: 'false', GOOGLE_JWKS_JSON: '{"keys":[{"kty":"RSA","n":"3SSum9jtxKTheDwctdDnp80Mv5_hAQzcKJJcxpw3wShOU0LyEpt23riO3ncaOC4iVm5xseM9PJmFjYMQJcplKi6I3nDC7tToFWrFqrn7LSjdvJS3WqUjn20CUiUxYZ3QLZcYyERU6M39M8nE1zFHQ3tHz7YkjoNQTPMUXMRydeL8yuBizdsrGQosgpGJceTAFIHJkKtdCipbSBZA3qrrE-HDJa9nZSYloywLVsaxzKJG2SiJzvVBydZbCQ2ZQeR44qdpCIibU2IMyVelKqiCqHwoBwzYybGx4Tcx4N_1UrNZQnECbcN7jSzxRp1agrK6p2w-svyYYXmt7ymqa3kjdQ","e":"AQAB","kid":"google-test","alg":"RS256","use":"sig"}]}', }, diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 11e3975..1f76b38 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,19 +1,25 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --config=wrangler.production.json` (hash: 4fd31437dfec74b4bde52f8eb582094e) +// Generated by Wrangler by running `wrangler types --config=wrangler.production.json` (hash: 1e7f342b27ec67431d48ecd80cae37c9) // Runtime types generated with workerd@1.20260730.1 2026-08-03 interface __BaseEnv_Env { ATTACHMENTS: R2Bucket; + VIDEOS: R2Bucket; DB: D1Database; ASSETS: Fetcher; + VIDEO_PROCESSOR_CONCURRENCY: "2"; + VIDEO_PROCESSING_AUTOSTART: "true"; APP_ORIGIN: "https://hackweek.sentry.new"; GOOGLE_REDIRECT_URI: "https://hackweek.sentry.new/api/auth/callback"; GOOGLE_CLIENT_ID: "694837489680-25m2umkr51lofdads5uvocgtcdqcs6c4.apps.googleusercontent.com"; ALLOWED_EMAIL_DOMAIN: "sentry.io"; STREAM_MODE: "disabled"; + VIDEO_PROCESSOR: DurableObjectNamespace; + VIDEO_PROCESSING_WORKFLOW: Workflow[0]['payload']>; } declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/worker/index"); + durableNamespaces: "VideoProcessorContainer"; } interface Env extends __BaseEnv_Env {} } diff --git a/wrangler.jsonc b/wrangler.jsonc index 39d8b13..c15b96b 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -26,6 +26,41 @@ "bucket_name": "hackweek-videos-local", }, ], + "workflows": [ + { + "binding": "VIDEO_PROCESSING_WORKFLOW", + "name": "hackweek-video-processing-local", + "class_name": "VideoProcessingWorkflow", + }, + ], + "containers": [ + { + "name": "hackweek-video-processor-local", + "class_name": "VideoProcessorContainer", + "image": "./Dockerfile.video-processor", + "image_build_context": ".", + "max_instances": 1, + "instance_type": "standard-2", + }, + ], + "durable_objects": { + "bindings": [ + { + "name": "VIDEO_PROCESSOR", + "class_name": "VideoProcessorContainer", + }, + ], + }, + "migrations": [ + { + "tag": "video-processor-v1", + "new_sqlite_classes": ["VideoProcessorContainer"], + }, + ], + "vars": { + "VIDEO_PROCESSOR_CONCURRENCY": "1", + "VIDEO_PROCESSING_AUTOSTART": "true", + }, "observability": { "enabled": true, }, diff --git a/wrangler.production.json b/wrangler.production.json index dbbaa03..12f78be 100644 --- a/wrangler.production.json +++ b/wrangler.production.json @@ -22,9 +22,46 @@ { "binding": "ATTACHMENTS", "bucket_name": "hackweek-attachments" + }, + { + "binding": "VIDEOS", + "bucket_name": "hackweek-videos" + } + ], + "workflows": [ + { + "binding": "VIDEO_PROCESSING_WORKFLOW", + "name": "hackweek-video-processing", + "class_name": "VideoProcessingWorkflow" + } + ], + "containers": [ + { + "name": "hackweek-video-processor", + "class_name": "VideoProcessorContainer", + "image": "./Dockerfile.video-processor", + "image_build_context": ".", + "max_instances": 2, + "instance_type": "standard-2" + } + ], + "durable_objects": { + "bindings": [ + { + "name": "VIDEO_PROCESSOR", + "class_name": "VideoProcessorContainer" + } + ] + }, + "migrations": [ + { + "tag": "video-processor-v1", + "new_sqlite_classes": ["VideoProcessorContainer"] } ], "vars": { + "VIDEO_PROCESSOR_CONCURRENCY": "2", + "VIDEO_PROCESSING_AUTOSTART": "true", "APP_ORIGIN": "https://hackweek.sentry.new", "GOOGLE_REDIRECT_URI": "https://hackweek.sentry.new/api/auth/callback", "GOOGLE_CLIENT_ID": "694837489680-25m2umkr51lofdads5uvocgtcdqcs6c4.apps.googleusercontent.com", From 695a625f4660b1f6934597ba38ae813a871c8e34 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 14:27:23 +0200 Subject: [PATCH 03/18] feat(video): add private MP4 playback and curated reel Serve current canonical R2 derivatives through authenticated same-origin endpoints with exact single-range responses and storage-neutral playback descriptors. Keep curated ordering while adding project team overlay data without exposing unrelated user fields.\n\nReplace HLS attachment with direct MP4 playback, embed ready videos on project pages, and harden the double-buffered reel with one-clip preloading, active-slot advancement, recoverable errors, pause/skip, and fullscreen controls. --- .dev.vars.example | 6 +- package-lock.json | 7 -- package.json | 1 - src/app/player/IndividualPlayer.tsx | 23 +--- src/app/player/ScreeningPlayer.tsx | 25 ++++- src/app/player/controller.ts | 97 ++++++++++++----- src/app/player/media.ts | 48 ++++----- src/app/queries/videos.ts | 10 +- src/app/routes/ProjectDetailsPage.tsx | 11 +- src/app/routes/ProjectsPage.tsx | 8 +- src/app/routes/WatchPage.tsx | 20 +--- src/app/styles.css | 22 ++++ src/app/video/ProjectVideoPanel.tsx | 17 ++- src/shared/projects.ts | 2 - src/shared/videos.ts | 8 +- src/worker/routes/videos.ts | 84 ++++++++++++++- src/worker/routes/years.ts | 2 - src/worker/services/videos.ts | 146 ++++++++++++++++++++++++- test/app/routes.test.tsx | 66 +++++------- test/player/controller.test.tsx | 34 +++--- test/video-ui/video-ui.test.tsx | 86 ++++++++------- test/video/video.test.ts | 149 ++++++++++++++++++++++++++ 22 files changed, 643 insertions(+), 229 deletions(-) diff --git a/.dev.vars.example b/.dev.vars.example index 7d6dfa6..cad93a0 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -7,9 +7,5 @@ GOOGLE_REDIRECT_URI=http://localhost:5173/api/auth/callback ALLOWED_EMAIL_DOMAIN="sentry.io" # R2 uploads are processed by the local Workflow and pinned FFmpeg Container. -# Stream fake mode remains only for the playback surface pending its R2 cutover. -STREAM_MODE="fake" -STREAM_ALLOWED_ORIGIN="localhost" -STREAM_DELIVERY_HOST="customer-fake.cloudflarestream.com" -STREAM_WEBHOOK_SECRET="replace-with-a-local-signing-secret" +# Ready derivatives play through the authenticated same-origin MP4 endpoint. VIDEO_SERVICE_TOKEN="replace-with-a-local-job-token" diff --git a/package-lock.json b/package-lock.json index 9c229e0..62307fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,6 @@ "@cloudflare/containers": "0.3.7", "@fontsource/rubik": "^5.3.0", "@tanstack/react-query": "^5.101.4", - "hls.js": "^1.6.16", "hono": "^4.12.34", "jose": "^6.2.8", "react": "^19.2.4", @@ -4359,12 +4358,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/hls.js": { - "version": "1.6.16", - "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.16.tgz", - "integrity": "sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==", - "license": "Apache-2.0" - }, "node_modules/hono": { "version": "4.12.34", "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.34.tgz", diff --git a/package.json b/package.json index da4d50b..7d3c698 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "@cloudflare/containers": "0.3.7", "@fontsource/rubik": "^5.3.0", "@tanstack/react-query": "^5.101.4", - "hls.js": "^1.6.16", "hono": "^4.12.34", "jose": "^6.2.8", "react": "^19.2.4", diff --git a/src/app/player/IndividualPlayer.tsx b/src/app/player/IndividualPlayer.tsx index 74af830..d39083d 100644 --- a/src/app/player/IndividualPlayer.tsx +++ b/src/app/player/IndividualPlayer.tsx @@ -1,7 +1,7 @@ import {useEffect, useRef, useState} from 'react'; import type {PlaybackResponse} from '../../shared/videos'; -import {attachProtectedHls} from './media'; +import {attachMp4} from './media'; export function IndividualPlayer({ playback, @@ -14,24 +14,11 @@ export function IndividualPlayer({ const [error, setError] = useState(null); useEffect(() => { - if (!video.current || !playback.manifestUrl) return; - const attachment = attachProtectedHls( - video.current, - playback.manifestUrl, - undefined, - setError, - ); + if (!video.current || playback.source.kind !== 'mp4') return; + setError(null); + const attachment = attachMp4(video.current, playback.source.url, undefined, setError); return () => attachment.destroy(); - }, [playback.manifestUrl]); - - if (!playback.manifestUrl) { - return ( -

- local fake Stream has no HLS manifest. protected playback must be validated in the - Cloudflare environment. -

- ); - } + }, [playback.source]); return (
diff --git a/src/app/player/ScreeningPlayer.tsx b/src/app/player/ScreeningPlayer.tsx index 119bbed..48baf87 100644 --- a/src/app/player/ScreeningPlayer.tsx +++ b/src/app/player/ScreeningPlayer.tsx @@ -49,7 +49,7 @@ export function ScreeningPlayer({ handleScreeningShortcut(event, { togglePause: () => void controller.current?.togglePause(), skip: () => void controller.current?.skip(), - fullscreen: () => void shell.current?.requestFullscreen(), + fullscreen: () => void shell.current?.requestFullscreen().catch(() => undefined), }); } window.addEventListener('keydown', onKeyDown); @@ -68,6 +68,7 @@ export function ScreeningPlayer({ const activeSlot = state.index % 2; const showingTitle = state.phase === 'title'; + const team = clip.teamMembers.join(' · ') || 'Hackweek team'; return (
@@ -87,13 +88,19 @@ export function ScreeningPlayer({

#{String(state.index + 1).padStart(2, '0')} / Hackweek

{clip.projectName}

- up next + {team}
+ {['playing', 'paused'].includes(state.phase) && ( +
+ {clip.projectName} + {team} +
+ )} {state.phase === 'idle' && (
#H

the Hackweek reel

-

{playlist.length} ready project videos · normalized audio

+

{playlist.length} ready project videos · private progressive MP4

)}
-
diff --git a/src/app/player/controller.ts b/src/app/player/controller.ts index e16c65f..52361e2 100644 --- a/src/app/player/controller.ts +++ b/src/app/player/controller.ts @@ -1,6 +1,6 @@ import type {PlaylistItem, PlaybackResponse} from '../../shared/videos'; import type {PlayerAudioGraph} from './audio'; -import {attachProtectedHls, type MediaAttachment} from './media'; +import {attachMp4, type MediaAttachment} from './media'; export type PlayerPhase = 'idle' | 'title' | 'playing' | 'paused' | 'complete' | 'error'; @@ -24,7 +24,7 @@ export function createScreeningController({ getPlayback, onState, titleDurationMs = 1_800, - attach = attachProtectedHls, + attach = attachMp4, }: { playlist: PlaylistItem[]; elements: [HTMLVideoElement, HTMLVideoElement]; @@ -32,49 +32,73 @@ export function createScreeningController({ getPlayback: (videoId: string) => Promise; onState: (state: PlayerState) => void; titleDurationMs?: number; - attach?: typeof attachProtectedHls; + attach?: typeof attachMp4; }): ScreeningController { let index = 0; let active: 0 | 1 = 0; let phase: PlayerPhase = 'idle'; let destroyed = false; + let operation = 0; let titleTimer: ReturnType | null = null; const attachments: [MediaAttachment | null, MediaAttachment | null] = [null, null]; const attachedVideoIds: [string | null, string | null] = [null, null]; + const slotOperations: [number, number] = [0, 0]; const notify = (error: string | null = null) => onState({phase, index, error}); - const onEnded = () => { - if (phase === 'playing') void advance(); - }; - elements.forEach((element) => element.addEventListener('ended', onEnded)); + const endedHandlers = elements.map((_element, slot) => () => { + if (phase === 'playing' && slot === active) void advance(); + }); + elements.forEach((element, slot) => + element.addEventListener('ended', endedHandlers[slot]), + ); async function prepare(clipIndex: number, slot: 0 | 1) { const clip = playlist[clipIndex]; if (!clip || attachedVideoIds[slot] === clip.videoId) return; + const slotOperation = ++slotOperations[slot]; const playback = await getPlayback(clip.videoId); - if (!playback.manifestUrl) { - throw new Error( - playback.mode === 'fake' - ? 'local fake Stream does not provide playable HLS' - : 'protected playback is unavailable', - ); + if (destroyed || slotOperation !== slotOperations[slot]) return; + if (playback.source.kind !== 'mp4') { + throw new Error('protected MP4 playback is unavailable'); } + attachments[slot]?.destroy(); - elements[slot].pause(); - elements[slot].removeAttribute('src'); - attach(elements[slot], playback.manifestUrl, undefined, fail); + attachments[slot] = null; + attachedVideoIds[slot] = null; + const attachment = attach( + elements[slot], + playback.source.url, + undefined, + (message) => { + if (destroyed || slotOperation !== slotOperations[slot]) return; + attachments[slot]?.destroy(); + attachments[slot] = null; + attachedVideoIds[slot] = null; + if (slot === active && clipIndex === index) fail(message); + }, + ); + if (destroyed || slotOperation !== slotOperations[slot]) { + attachment.destroy(); + return; + } + attachments[slot] = attachment; attachedVideoIds[slot] = clip.videoId; audio.setGain(slot, clip.gainDb); } async function playCurrent() { if (destroyed) return; + const currentOperation = ++operation; phase = 'title'; notify(); await prepare(index, active); - void prepare(index + 1, active === 0 ? 1 : 0).catch(() => undefined); + if (destroyed || currentOperation !== operation || phase !== 'title') return; + + const nextSlot = active === 0 ? 1 : 0; + void prepare(index + 1, nextSlot).catch(() => undefined); titleTimer = setTimeout(() => { - if (destroyed || phase !== 'title') return; + titleTimer = null; + if (destroyed || currentOperation !== operation || phase !== 'title') return; phase = 'playing'; notify(); void elements[active] @@ -86,6 +110,11 @@ export function createScreeningController({ } async function advance() { + operation += 1; + if (titleTimer) { + clearTimeout(titleTimer); + titleTimer = null; + } elements[active].pause(); if (index >= playlist.length - 1) { phase = 'complete'; @@ -94,17 +123,25 @@ export function createScreeningController({ } index += 1; active = active === 0 ? 1 : 0; - await playCurrent(); + await playCurrent().catch((error: unknown) => + fail(error instanceof Error ? error.message : 'playback could not start'), + ); } function fail(message: string) { + operation += 1; + if (titleTimer) { + clearTimeout(titleTimer); + titleTimer = null; + } + elements[active].pause(); phase = 'error'; notify(message); } return { async start() { - if (!playlist.length) return; + if (!playlist.length || phase !== 'idle') return; await audio.resume(); await playCurrent().catch((error: unknown) => fail(error instanceof Error ? error.message : 'playback could not start'), @@ -116,23 +153,29 @@ export function createScreeningController({ phase = 'paused'; notify(); } else if (phase === 'paused') { - await audio.resume(); - await elements[active].play(); - phase = 'playing'; - notify(); + try { + await audio.resume(); + await elements[active].play(); + phase = 'playing'; + notify(); + } catch (error) { + fail(error instanceof Error ? error.message : 'playback could not resume'); + } } }, async skip() { if (phase === 'idle' || phase === 'complete') return; - if (titleTimer) clearTimeout(titleTimer); await advance(); }, destroy() { destroyed = true; + operation += 1; + slotOperations[0] += 1; + slotOperations[1] += 1; if (titleTimer) clearTimeout(titleTimer); - elements.forEach((element) => { + elements.forEach((element, slot) => { element.pause(); - element.removeEventListener('ended', onEnded); + element.removeEventListener('ended', endedHandlers[slot]); }); attachments.forEach((attachment) => attachment?.destroy()); void audio.close(); diff --git a/src/app/player/media.ts b/src/app/player/media.ts index f6ddb1e..ec19306 100644 --- a/src/app/player/media.ts +++ b/src/app/player/media.ts @@ -1,41 +1,29 @@ -import Hls from 'hls.js'; - export interface MediaAttachment { destroy(): void; } -export function attachProtectedHls( +export function attachMp4( element: HTMLVideoElement, - manifestUrl: string, + url: string, onReady?: () => void, onError?: (message: string) => void, ): MediaAttachment { - element.crossOrigin = 'anonymous'; - element.preload = 'auto'; + const ready = () => onReady?.(); + const error = () => onError?.('private video could not be loaded'); - if (Hls.isSupported()) { - const hls = new Hls({enableWorker: true}); - hls.on(Hls.Events.MANIFEST_PARSED, () => onReady?.()); - hls.on(Hls.Events.ERROR, (_event, data) => { - if (data.fatal) onError?.('protected video could not be loaded'); - }); - hls.loadSource(manifestUrl); - hls.attachMedia(element); - return {destroy: () => hls.destroy()}; - } - - if (element.canPlayType('application/vnd.apple.mpegurl')) { - element.src = manifestUrl; - element.load(); - onReady?.(); - return { - destroy() { - element.removeAttribute('src'); - element.load(); - }, - }; - } + element.preload = 'auto'; + element.addEventListener('loadeddata', ready, {once: true}); + element.addEventListener('error', error); + element.src = url; + element.load(); - onError?.('this browser cannot play protected HLS with normalized audio'); - return {destroy() {}}; + return { + destroy() { + element.removeEventListener('loadeddata', ready); + element.removeEventListener('error', error); + element.pause(); + element.removeAttribute('src'); + element.load(); + }, + }; } diff --git a/src/app/queries/videos.ts b/src/app/queries/videos.ts index f3fbbe6..cc4a34e 100644 --- a/src/app/queries/videos.ts +++ b/src/app/queries/videos.ts @@ -64,13 +64,9 @@ export function useDeleteVideo(projectId: string) { jsonRequest('DELETE', {confirmed: true}), ), onSuccess: () => - cache.setQueryData( - ['project-video', projectId], - (current) => ({ - video: null, - streamMode: current?.streamMode ?? 'fake', - }), - ), + cache.setQueryData(['project-video', projectId], () => ({ + video: null, + })), }); } diff --git a/src/app/routes/ProjectDetailsPage.tsx b/src/app/routes/ProjectDetailsPage.tsx index 7f0d47a..cc0e70d 100644 --- a/src/app/routes/ProjectDetailsPage.tsx +++ b/src/app/routes/ProjectDetailsPage.tsx @@ -1,9 +1,10 @@ import {useState, type ChangeEvent} from 'react'; +import {useQuery} from '@tanstack/react-query'; import {Link, useLocation, useParams} from 'wouter'; import {QueryState} from '../components/AppLayout'; import {Markdown} from '../components/Markdown'; -import {useProjectVideo} from '../queries/videos'; +import {getPlayback, useProjectVideo} from '../queries/videos'; import {ProjectVideoPanel} from '../video/ProjectVideoPanel'; import { useDeleteMedia, @@ -23,6 +24,11 @@ export function ProjectDetailsPage() { const upload = useUploadMedia(projectId); const removeMedia = useDeleteMedia(projectId); const video = useProjectVideo(projectId); + const playback = useQuery({ + queryKey: ['video-playback', video.data?.video?.id], + queryFn: () => getPlayback(video.data!.video!.id), + enabled: video.data?.video?.status === 'ready', + }); const [actionError, setActionError] = useState(null); function addMedia(event: ChangeEvent) { @@ -145,7 +151,8 @@ export function ProjectDetailsPage() { video={video.data?.video ?? null} loading={video.isLoading} canManage={project.data.project.permissions.canManageMedia} - streamMode={video.data?.streamMode} + playback={playback.data} + playbackError={playback.error?.message} /> )} {project.data.project.kind === 'project' && ( diff --git a/src/app/routes/ProjectsPage.tsx b/src/app/routes/ProjectsPage.tsx index eaa1047..cc9e9da 100644 --- a/src/app/routes/ProjectsPage.tsx +++ b/src/app/routes/ProjectsPage.tsx @@ -76,11 +76,9 @@ export function ProjectsPage({isAdmin = false}: {isAdmin?: boolean}) {

- {year.data.streamMode === 'real' && ( - - watch reel - - )} + + watch reel + {year.data.year.votingEnabled && ( vote diff --git a/src/app/routes/WatchPage.tsx b/src/app/routes/WatchPage.tsx index d5e4bd6..f5defa9 100644 --- a/src/app/routes/WatchPage.tsx +++ b/src/app/routes/WatchPage.tsx @@ -21,17 +21,10 @@ export function WatchPage() {

Hackweek {yearId} / screening

play the reel

-

two-player protected HLS with measured, clamped audio gain.

+

private progressive MP4 playback in the curated screening order.

- {playlist.data.streamMode === 'disabled' ? ( - - ) : ( - - )} - {playlist.data.streamMode !== 'disabled' && playlist.data.videos.length > 0 && ( + + {playlist.data.videos.length > 0 && (

on demand

watch one project

@@ -66,12 +59,7 @@ export function ProjectVideoWatchPage() { return ( - {video.data?.streamMode === 'disabled' ? ( - - ) : video.data?.video?.status !== 'ready' ? ( + {video.data?.video?.status !== 'ready' ? ( )} + {video?.status === 'ready' && playback && ( + + )} + {playbackError && ( +

+ {playbackError} +

+ )} + {upload && (
diff --git a/src/shared/projects.ts b/src/shared/projects.ts index b509a11..dd66d79 100644 --- a/src/shared/projects.ts +++ b/src/shared/projects.ts @@ -1,6 +1,5 @@ import type {AwardSummary} from './administration'; import type {SessionUser} from './api'; -import type {StreamMode} from './videos'; export type ProjectKind = 'project' | 'idea'; @@ -68,7 +67,6 @@ export interface YearResponse { year: YearSummary; groups: GroupSummary[]; awards: AwardSummary[]; - streamMode: StreamMode; } export interface ProjectsResponse { diff --git a/src/shared/videos.ts b/src/shared/videos.ts index 6c8510c..2917c22 100644 --- a/src/shared/videos.ts +++ b/src/shared/videos.ts @@ -62,15 +62,15 @@ export interface CompleteVideoUploadRequest { } export interface PlaybackResponse { - mode: 'stream' | 'fake'; - manifestUrl: string | null; - expiresAt: string; + source: {kind: 'mp4'; url: string}; + expiresAt: null; } export interface PlaylistItem { videoId: string; projectId: string; projectName: string; + teamMembers: string[]; durationSeconds: number; gainDb: number; position: number; @@ -78,10 +78,8 @@ export interface PlaylistItem { export interface ProjectVideoResponse { video: ProjectVideo | null; - streamMode: StreamMode; } export interface PlaylistResponse { videos: PlaylistItem[]; - streamMode: StreamMode; } diff --git a/src/worker/routes/videos.ts b/src/worker/routes/videos.ts index 520d3fb..ba08d14 100644 --- a/src/worker/routes/videos.ts +++ b/src/worker/routes/videos.ts @@ -4,9 +4,10 @@ import type { CompleteVideoUploadRequest, DirectUploadRequest, DirectUploadResponse, + PlaybackResponse, + PlaylistResponse, ProjectVideoResponse, } from '../../shared/videos'; -import {streamMode} from '../integrations/stream'; import type {WorkerEnv} from '../index'; import {errorResponse, ServiceError} from '../services/errors'; import { @@ -14,21 +15,75 @@ import { completeVideoUpload, createMultipartVideoUpload, getProjectVideo, + getVideoContent, getVideoUpload, + issuePlayback, + listPlaylist, MAX_VIDEO_BYTES, retireProjectVideo, retryProjectVideo, uploadVideoPart, + VideoRangeError, } from '../services/videos'; export const videosRoutes = new Hono(); export const projectVideoRoutes = new Hono(); +videosRoutes.get('/playlist', async (c) => { + try { + const year = c.req.query('year'); + if (!year) invalid('Year is required'); + const response: PlaylistResponse = {videos: await listPlaylist(c.env.DB, year)}; + return c.json(response, 200, {'Cache-Control': 'private, no-store'}); + } catch (error) { + return respondError(c, error); + } +}); + +videosRoutes.get('/:videoId/playback', async (c) => { + try { + const response: PlaybackResponse = await issuePlayback( + c.env.DB, + c.req.param('videoId'), + ); + return c.json(response, 200, {'Cache-Control': 'private, no-store'}); + } catch (error) { + return respondError(c, error); + } +}); + +videosRoutes.get('/:videoId/content', async (c) => { + try { + const content = await getVideoContent( + c.env.DB, + c.env.VIDEOS, + c.req.param('videoId'), + c.req.header('Range'), + ); + const headers = videoContentHeaders(content); + return new Response(content.object.body, { + status: content.range ? 206 : 200, + headers, + }); + } catch (error) { + if (error instanceof VideoRangeError) { + return new Response(null, { + status: 416, + headers: { + 'Accept-Ranges': 'bytes', + 'Content-Range': `bytes */${error.size}`, + 'Cache-Control': 'private, max-age=300', + }, + }); + } + return respondError(c, error); + } +}); + projectVideoRoutes.get('/:projectId/video', async (c) => { try { const response: ProjectVideoResponse = { video: await getProjectVideo(c.env.DB, c.req.param('projectId')), - streamMode: streamMode(c.env), }; return c.json(response); } catch (error) { @@ -239,6 +294,31 @@ function parseRetirement(value: unknown) { return (value as Record).confirmed === true; } +function videoContentHeaders(content: { + object: R2ObjectBody; + range: {start: number; end: number; length: number} | null; + size: number; + etag: string; +}) { + const headers = new Headers(); + headers.set('Accept-Ranges', 'bytes'); + headers.set('Cache-Control', 'private, max-age=300'); + headers.set('Content-Disposition', 'inline'); + headers.set('Content-Type', 'video/mp4'); + headers.set('ETag', content.etag); + headers.set('X-Content-Type-Options', 'nosniff'); + if (content.range) { + headers.set( + 'Content-Range', + `bytes ${content.range.start}-${content.range.end}/${content.size}`, + ); + headers.set('Content-Length', String(content.range.length)); + } else { + headers.set('Content-Length', String(content.size)); + } + return headers; +} + function parsePartNumber(value: string) { if (!/^\d+$/.test(value)) invalid('Part number is invalid'); const number = Number(value); diff --git a/src/worker/routes/years.ts b/src/worker/routes/years.ts index 40a4531..b49d02b 100644 --- a/src/worker/routes/years.ts +++ b/src/worker/routes/years.ts @@ -2,7 +2,6 @@ import {Hono} from 'hono'; import type {GroupResponse, YearResponse, YearsResponse} from '../../shared/projects'; import type {WorkerEnv} from '../index'; -import {streamMode} from '../integrations/stream'; import {requireRole} from '../middleware/user'; import {createGroup} from '../repositories/groups'; import { @@ -57,7 +56,6 @@ yearsRoutes.get('/:yearId', async (c) => { categoryName: award.category_name, name: award.name, })), - streamMode: streamMode(c.env), }; return c.json(response); } catch (error) { diff --git a/src/worker/services/videos.ts b/src/worker/services/videos.ts index feaa38c..18b55e4 100644 --- a/src/worker/services/videos.ts +++ b/src/worker/services/videos.ts @@ -1,5 +1,7 @@ import type {SessionUser} from '../../shared/api'; import type { + PlaybackResponse, + PlaylistItem, ProjectVideo, VideoUploadPart, VideoUploadSession, @@ -21,6 +23,7 @@ interface VideoRow { size_bytes: number; status: string; processing_attempt: number; + processed_r2_key: string | null; duration_seconds: number | null; loudness_lufs: number | null; gain_db: number | null; @@ -74,6 +77,131 @@ export async function getProjectVideo(db: D1Database, projectId: string) { return row ? mapVideo(row) : null; } +export async function listPlaylist( + db: D1Database, + yearId: string, +): Promise { + const {results} = await db + .prepare( + `SELECT pv.id video_id, p.id project_id, p.name project_name, + pv.duration_seconds, pv.gain_db, so.position + FROM screening_order so + JOIN projects p ON p.id = so.project_id AND p.status = 'active' + JOIN project_videos pv ON pv.project_id = p.id + WHERE so.year_id = ? AND pv.status = 'ready' AND pv.retired_at IS NULL + AND pv.processed_r2_key IS NOT NULL + AND pv.duration_seconds IS NOT NULL AND pv.gain_db IS NOT NULL + ORDER BY so.position, p.id`, + ) + .bind(yearId) + .all<{ + video_id: string; + project_id: string; + project_name: string; + duration_seconds: number; + gain_db: number; + position: number; + }>(); + if (!results.length) return []; + + const membersByProject = new Map(); + const projectIds = results.map((row) => row.project_id); + const members = await db + .prepare( + `SELECT pm.project_id, u.display_name + FROM project_members pm JOIN users u ON u.id = pm.user_id + WHERE pm.project_id IN (${projectIds.map(() => '?').join(', ')}) + ORDER BY pm.project_id, u.display_name COLLATE NOCASE, u.id`, + ) + .bind(...projectIds) + .all<{project_id: string; display_name: string}>(); + for (const member of members.results) { + const names = membersByProject.get(member.project_id) ?? []; + names.push(member.display_name); + membersByProject.set(member.project_id, names); + } + + return results.map((row) => ({ + videoId: row.video_id, + projectId: row.project_id, + projectName: row.project_name, + teamMembers: membersByProject.get(row.project_id) ?? [], + durationSeconds: row.duration_seconds, + gainDb: row.gain_db, + position: row.position, + })); +} + +export async function issuePlayback( + db: D1Database, + videoId: string, +): Promise { + await requireReadyVideo(db, videoId); + return { + source: { + kind: 'mp4', + url: `/api/videos/${encodeURIComponent(videoId)}/content`, + }, + expiresAt: null, + }; +} + +export async function getVideoContent( + db: D1Database, + bucket: R2Bucket, + videoId: string, + rangeHeader?: string, +) { + const video = await requireReadyVideo(db, videoId); + const key = video.processed_r2_key!; + const head = await bucket.head(key); + if (!head) throw new ServiceError('NOT_FOUND', 'Video content is missing', 404); + + const range = + rangeHeader === undefined ? null : parseVideoRange(rangeHeader, head.size); + const object = await bucket.get( + key, + range ? {range: {offset: range.start, length: range.length}} : undefined, + ); + if (!object) throw new ServiceError('NOT_FOUND', 'Video content is missing', 404); + return {object, range, size: head.size, etag: head.httpEtag}; +} + +export class VideoRangeError extends Error { + constructor(readonly size: number) { + super('Requested video range is not satisfiable'); + } +} + +export function parseVideoRange(value: string, size: number) { + if (!Number.isSafeInteger(size) || size <= 0) throw new VideoRangeError(size); + const match = /^bytes=(\d*)-(\d*)$/.exec(value.trim()); + if (!match || (!match[1] && !match[2])) throw new VideoRangeError(size); + + let start: number; + let end: number; + if (!match[1]) { + const suffix = Number(match[2]); + if (!Number.isSafeInteger(suffix) || suffix <= 0) throw new VideoRangeError(size); + start = Math.max(0, size - suffix); + end = size - 1; + } else { + start = Number(match[1]); + end = match[2] ? Number(match[2]) : size - 1; + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start < 0 || + start >= size || + end < start + ) { + throw new VideoRangeError(size); + } + end = Math.min(end, size - 1); + } + return {start, end, length: end - start + 1}; +} + export async function createMultipartVideoUpload( db: D1Database, bucket: R2Bucket, @@ -715,8 +843,22 @@ function mapVideo(row: VideoRow): ProjectVideo { function videoSelect() { return `SELECT id, project_id, original_name, content_type, size_bytes, status, - processing_attempt, duration_seconds, loudness_lufs, gain_db, error_message, - created_at FROM project_videos`; + processing_attempt, processed_r2_key, duration_seconds, loudness_lufs, gain_db, + error_message, created_at FROM project_videos`; +} + +async function requireReadyVideo(db: D1Database, videoId: string) { + const video = await db + .prepare( + `${videoSelect()} WHERE id = ? AND status = 'ready' AND retired_at IS NULL + AND processed_r2_key IS NOT NULL`, + ) + .bind(videoId) + .first(); + if (!video) { + throw new ServiceError('CONFLICT', 'Video is not ready for playback', 409); + } + return video; } async function listStoredParts(db: D1Database, uploadId: string) { diff --git a/test/app/routes.test.tsx b/test/app/routes.test.tsx index 967535b..ac62dce 100644 --- a/test/app/routes.test.tsx +++ b/test/app/routes.test.tsx @@ -164,47 +164,35 @@ describe('clickable project routes', () => { expect(within(archives).queryByText('2025')).toBeNull(); }); - it.each([ - {streamMode: 'fake', expectedHref: null}, - {streamMode: 'disabled', expectedHref: null}, - {streamMode: 'real', expectedHref: '/years/2026/watch'}, - ] as const)( - 'handles the watch reel link in $streamMode stream mode', - async ({streamMode, expectedHref}) => { - fetchMock.mockImplementation(async (input) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL - ? input.href - : input.url; - if (url.includes('/api/years/2026')) { - return json({ - year: { - id: '2026', - votingEnabled: false, - submissionsClosed: false, - projectCount: 0, - ideaCount: 0, - groupCount: 0, - participantCount: 0, - }, - groups: [], - awards: [], - streamMode, - }); - } - return json({projects: [], nextCursor: null}); - }); + it('links every year to its private R2 screening reel', async () => { + fetchMock.mockImplementation(async (input) => { + const url = + typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + if (url.includes('/api/years/2026')) { + return json({ + year: { + id: '2026', + votingEnabled: false, + submissionsClosed: false, + projectCount: 0, + ideaCount: 0, + groupCount: 0, + participantCount: 0, + }, + groups: [], + awards: [], + }); + } + return json({projects: [], nextCursor: null}); + }); - renderRoute(, '/years/2026/projects', '/years/:yearId/projects'); + renderRoute(, '/years/2026/projects', '/years/:yearId/projects'); - expect(await screen.findByRole('heading', {name: 'projects & ideas'})).toBeTruthy(); - expect( - screen.queryByRole('link', {name: 'watch reel'})?.getAttribute('href') ?? null, - ).toBe(expectedHref); - }, - ); + expect(await screen.findByRole('heading', {name: 'projects & ideas'})).toBeTruthy(); + expect(screen.getByRole('link', {name: 'watch reel'}).getAttribute('href')).toBe( + '/years/2026/watch', + ); + }); it('defaults to the grid view when storage is unavailable', async () => { fetchMock.mockImplementation(async (input) => { diff --git a/test/player/controller.test.tsx b/test/player/controller.test.tsx index 73eddbf..3a2415b 100644 --- a/test/player/controller.test.tsx +++ b/test/player/controller.test.tsx @@ -21,9 +21,8 @@ describe('dual screening controller', () => { elements: videos, audio, getPlayback: async (videoId) => ({ - mode: 'stream', - manifestUrl: `https://stream/${videoId}.m3u8`, - expiresAt: 'later', + source: {kind: 'mp4', url: `/api/videos/${videoId}/content`}, + expiresAt: null, }), attach: (_element, url) => { attached.push(url); @@ -37,8 +36,8 @@ describe('dual screening controller', () => { await Promise.resolve(); expect(states.at(-1)?.phase).toBe('title'); expect(attached).toEqual([ - 'https://stream/video-1.m3u8', - 'https://stream/video-2.m3u8', + '/api/videos/video-1/content', + '/api/videos/video-2/content', ]); expect(vi.mocked(audio.setGain).mock.calls).toEqual([ [0, 6], @@ -51,6 +50,10 @@ describe('dual screening controller', () => { expect(vi.mocked(videos[0].play).mock.calls).toHaveLength(1); expect(states.at(-1)?.index).toBe(0); + videos[1].dispatchEvent(new Event('ended')); + await Promise.resolve(); + expect(states.at(-1)?.index).toBe(0); + videos[0].dispatchEvent(new Event('ended')); await Promise.resolve(); expect(states.at(-1)?.phase).toBe('title'); @@ -63,7 +66,7 @@ describe('dual screening controller', () => { expect(states.at(-1)?.phase).toBe('complete'); }); - it('supports pause, resume, explicit skip, and fake-manifest errors', async () => { + it('supports pause, resume, explicit skip, and recoverable source errors', async () => { const videos: [HTMLVideoElement, HTMLVideoElement] = [fakeVideo(), fakeVideo()]; const states: PlayerState[] = []; const controller = createScreeningController({ @@ -71,9 +74,8 @@ describe('dual screening controller', () => { elements: videos, audio: fakeAudio(), getPlayback: async () => ({ - mode: 'stream', - manifestUrl: 'manifest', - expiresAt: 'later', + source: {kind: 'mp4', url: '/api/videos/video/content'}, + expiresAt: null, }), attach: () => ({destroy: vi.fn()}), onState: (state) => states.push(state), @@ -89,19 +91,23 @@ describe('dual screening controller', () => { expect(states.at(-1)?.index).toBe(1); const errors: PlayerState[] = []; - const fakeController = createScreeningController({ + const errorController = createScreeningController({ playlist: [playlist[0]], elements: [fakeVideo(), fakeVideo()], audio: fakeAudio(), - getPlayback: async () => ({mode: 'fake', manifestUrl: null, expiresAt: 'later'}), + getPlayback: async () => { + throw new Error('private source unavailable'); + }, onState: (state) => errors.push(state), titleDurationMs: 1, }); - await fakeController.start(); + await errorController.start(); expect(errors.at(-1)).toMatchObject({ phase: 'error', - error: expect.stringContaining('fake Stream'), + error: 'private source unavailable', }); + await errorController.skip(); + expect(errors.at(-1)?.phase).toBe('complete'); }); }); @@ -128,6 +134,7 @@ const playlist: PlaylistItem[] = [ videoId: 'video-1', projectId: 'project-1', projectName: 'First', + teamMembers: ['Ada', 'Grace'], durationSeconds: 10, gainDb: 6, position: 0, @@ -136,6 +143,7 @@ const playlist: PlaylistItem[] = [ videoId: 'video-2', projectId: 'project-2', projectName: 'Second', + teamMembers: ['Linus'], durationSeconds: 20, gainDb: -3, position: 1, diff --git a/test/video-ui/video-ui.test.tsx b/test/video-ui/video-ui.test.tsx index 7c502a3..e1d6e32 100644 --- a/test/video-ui/video-ui.test.tsx +++ b/test/video-ui/video-ui.test.tsx @@ -5,6 +5,7 @@ import {Route, Router} from 'wouter'; import {memoryLocation} from 'wouter/memory-location'; import {afterEach, describe, expect, it, vi} from 'vitest'; +import {IndividualPlayer} from '../../src/app/player/IndividualPlayer'; import { handleScreeningShortcut, ScreeningPlayer, @@ -114,40 +115,23 @@ describe('video user experience', () => { expect(readResumeRecord('project', file)).toBeNull(); }); - it.each([ - {streamMode: 'fake'}, - {streamMode: 'disabled'}, - {streamMode: 'real'}, - ] as const)( - 'keeps R2 lifecycle state independent of $streamMode Stream configuration', - ({streamMode}) => { - renderQuery( - , - ); - - expect(screen.getByRole('link', {name: 'watch video'}).getAttribute('href')).toBe( - '/years/2026/projects/project/video', - ); - }, - ); - - it('keeps uploads available without depending on Stream configuration', () => { - renderQuery( + it('keeps ready playback and uploads independent of Stream configuration', () => { + const ready = renderQuery( , ); + expect(screen.getByRole('link', {name: 'watch video'}).getAttribute('href')).toBe( + '/years/2026/projects/project/video', + ); + ready.unmount(); + renderQuery( + , + ); expect(screen.getByLabelText('select project video')).toBeTruthy(); expect(screen.getByText(/private R2 storage/i)).toBeTruthy(); }); @@ -171,11 +155,44 @@ describe('video user experience', () => { expect(screen.getByRole('button', {name: 'retire video'})).toBeTruthy(); }); + it('attaches authenticated progressive MP4 directly to an HTML video element', async () => { + const load = vi + .spyOn(HTMLMediaElement.prototype, 'load') + .mockImplementation(() => undefined); + const pause = vi + .spyOn(HTMLMediaElement.prototype, 'pause') + .mockImplementation(() => undefined); + const view = renderQuery( + , + ); + const video = screen.getByLabelText('First project video') as HTMLVideoElement; + expect(video.getAttribute('src')).toBe('/api/videos/video-1/content'); + expect(video.preload).toBe('auto'); + + video.dispatchEvent(new Event('error')); + expect((await screen.findByRole('alert')).textContent).toContain( + 'private video could not be loaded', + ); + view.unmount(); + expect(video.hasAttribute('src')).toBe(false); + expect(load).toHaveBeenCalled(); + expect(pause).toHaveBeenCalled(); + load.mockRestore(); + pause.mockRestore(); + }); + it('renders accessible empty reel and individual ready-video permalinks', async () => { - fetchMock.mockResolvedValue(json({videos: playlist, streamMode: 'fake'})); + fetchMock.mockResolvedValue(json({videos: playlist})); renderRoute(, '/years/2026/watch', '/years/:yearId/watch'); expect(await screen.findByRole('heading', {name: 'play the reel'})).toBeTruthy(); expect(screen.getByRole('button', {name: 'play all'})).toBeTruthy(); + expect(screen.getByText('Ada Lovelace · Grace Hopper')).toBeTruthy(); expect(screen.getByRole('link', {name: /First project/}).getAttribute('href')).toBe( '/years/2026/watch/video-1', ); @@ -184,16 +201,6 @@ describe('video user experience', () => { expect(screen.getByRole('heading', {name: 'no videos are ready'})).toBeTruthy(); }); - it('renders a clear disabled screening state without implying playback works', async () => { - fetchMock.mockResolvedValue(json({videos: [], streamMode: 'disabled'})); - renderRoute(, '/years/2026/watch', '/years/:yearId/watch'); - - expect( - await screen.findByRole('heading', {name: 'video screening unavailable'}), - ).toBeTruthy(); - expect(screen.queryByRole('button', {name: 'play all'})).toBeNull(); - }); - it('exposes visible pause, skip, fullscreen controls and keyboard shortcuts', async () => { const actions = {togglePause: vi.fn(), skip: vi.fn(), fullscreen: vi.fn()}; for (const [code, key] of [ @@ -285,6 +292,7 @@ const playlist: PlaylistItem[] = [ videoId: 'video-1', projectId: 'project', projectName: 'First project', + teamMembers: ['Ada Lovelace', 'Grace Hopper'], durationSeconds: 30, gainDb: 0, position: 0, diff --git a/test/video/video.test.ts b/test/video/video.test.ts index 6e803f0..0e75bdf 100644 --- a/test/video/video.test.ts +++ b/test/video/video.test.ts @@ -373,6 +373,129 @@ describe('R2 multipart video lifecycle', () => { expect(await env.VIDEOS.head(second.outputKey)).not.toBeNull(); }); + it('serves authenticated canonical MP4 bytes with exact single-range semantics', async () => { + const bytes = '0123456789'; + const ready = await publishReadyVideo(projectId, ownerToken, bytes); + + const unauthorizedDescriptor = await SELF.fetch( + `${base}/videos/${ready.video.id}/playback`, + ); + const unauthorizedContent = await SELF.fetch( + `${base}/videos/${ready.video.id}/content`, + ); + expect(unauthorizedDescriptor.status).toBe(401); + expect(unauthorizedContent.status).toBe(401); + + const descriptor = await api(`/videos/${ready.video.id}/playback`, ownerToken); + expect(descriptor.status).toBe(200); + expect(descriptor.body).toEqual({ + source: {kind: 'mp4', url: `/api/videos/${ready.video.id}/content`}, + expiresAt: null, + }); + expect(descriptor.headers.get('cache-control')).toBe('private, no-store'); + + const full = await fetchVideoContent(ready.video.id, ownerToken); + expect(full.status).toBe(200); + expect(await responseText(full)).toBe(bytes); + expect(full.headers.get('accept-ranges')).toBe('bytes'); + expect(full.headers.get('content-length')).toBe('10'); + expect(full.headers.get('content-type')).toBe('video/mp4'); + expect(full.headers.get('content-disposition')).toBe('inline'); + expect(full.headers.get('cache-control')).toBe('private, max-age=300'); + expect(full.headers.get('etag')).toBeTruthy(); + + for (const [range, body, contentRange] of [ + ['bytes=2-5', '2345', 'bytes 2-5/10'], + ['bytes=7-', '789', 'bytes 7-9/10'], + ['bytes=-3', '789', 'bytes 7-9/10'], + ['bytes=0-99', bytes, 'bytes 0-9/10'], + ]) { + const partial = await fetchVideoContent(ready.video.id, ownerToken, range); + expect(partial.status, range).toBe(206); + expect(await responseText(partial), range).toBe(body); + expect(partial.headers.get('content-range'), range).toBe(contentRange); + expect(partial.headers.get('content-length'), range).toBe(String(body.length)); + expect(partial.headers.get('accept-ranges'), range).toBe('bytes'); + } + + for (const range of [ + 'bytes=10-', + 'bytes=5-2', + 'bytes=-0', + 'bytes=', + 'items=0-1', + 'bytes=0-1,3-4', + ]) { + const rejected = await fetchVideoContent(ready.video.id, ownerToken, range); + expect(rejected.status, range).toBe(416); + expect(rejected.headers.get('content-range'), range).toBe('bytes */10'); + expect(rejected.headers.get('accept-ranges'), range).toBe('bytes'); + expect(await responseText(rejected), range).toBe(''); + } + }); + + it('returns only ready active videos in curated order with team display names', async () => { + const firstProject = await createProject('Curated first'); + const secondProject = await createProject('Curated second'); + const unreadyProject = await createProject('Curated unready'); + const first = await publishReadyVideo(firstProject, ownerToken, 'first canonical'); + const second = await publishReadyVideo(secondProject, ownerToken, 'second canonical'); + const unready = await completeSmallUpload(unreadyProject, ownerToken, 'not ready'); + await env.DB.batch([ + env.DB.prepare( + 'INSERT INTO project_members (project_id, user_id) VALUES (?, ?)', + ).bind(firstProject, memberId), + env.DB.prepare( + 'INSERT INTO screening_order (year_id, project_id, position) VALUES (?, ?, 0)', + ).bind(yearId, secondProject), + env.DB.prepare( + 'INSERT INTO screening_order (year_id, project_id, position) VALUES (?, ?, 1)', + ).bind(yearId, unreadyProject), + env.DB.prepare( + 'INSERT INTO screening_order (year_id, project_id, position) VALUES (?, ?, 2)', + ).bind(yearId, firstProject), + ]); + + const playlist = await api(`/videos/playlist?year=${yearId}`, ownerToken); + expect(playlist.status).toBe(200); + expect(playlist.body.videos.map((item: {videoId: string}) => item.videoId)).toEqual([ + second.video.id, + first.video.id, + ]); + expect(playlist.body.videos[0]).toMatchObject({ + projectName: 'Curated second', + position: 0, + teamMembers: ['Hackweek Member'], + }); + expect(playlist.body.videos[1]).toMatchObject({ + projectName: 'Curated first', + position: 2, + teamMembers: ['Hackweek Member', 'Hackweek Member'], + }); + expect( + playlist.body.videos.some( + (item: {videoId: string}) => item.videoId === unready.video.id, + ), + ).toBe(false); + expect((await api(`/videos/${unready.video.id}/playback`, ownerToken)).status).toBe( + 409, + ); + + const retired = await api(`/projects/${secondProject}/video`, ownerToken, { + method: 'DELETE', + body: {confirmed: true}, + }); + expect(retired.status).toBe(204); + expect(await env.VIDEOS.head(second.outputKey)).not.toBeNull(); + expect((await api(`/videos/${second.video.id}/playback`, ownerToken)).status).toBe( + 409, + ); + const afterRetirement = await api(`/videos/playlist?year=${yearId}`, ownerToken); + expect( + afterRetirement.body.videos.map((item: {videoId: string}) => item.videoId), + ).toEqual([first.video.id]); + }); + it('limits local processing to one while independent projects remain queued', async () => { const leftProject = await createProject('Processor left'); const rightProject = await createProject('Processor right'); @@ -414,6 +537,32 @@ const canonicalResult: VideoProcessorResult = { sha256: 'a'.repeat(64), }; +async function publishReadyVideo(project: string, token: string, bytes: string) { + const completed = await completeSmallUpload(project, token, bytes); + const claim = await claimVideoProcessingAttempt(env.DB, completed.video.id, 1, 1); + if (claim.status !== 'claimed') throw new Error('attempt was not claimed'); + await env.VIDEOS.put(claim.outputKey, bytes, { + httpMetadata: {contentType: 'video/mp4'}, + }); + expect( + await publishVideoProcessingAttempt(env.DB, completed.video.id, 1, claim.outputKey, { + ...canonicalResult, + durationSeconds: bytes.length, + }), + ).toBe(true); + return {...completed, outputKey: claim.outputKey}; +} + +function fetchVideoContent(videoId: string, token: string, range?: string) { + return SELF.fetch(`${base}/videos/${videoId}/content`, { + headers: {Cookie: token, ...(range ? {Range: range} : {})}, + }); +} + +async function responseText(response: Response) { + return new TextDecoder().decode(await response.arrayBuffer()); +} + async function completeSmallUpload(project: string, token: string, bytes: string) { const created = await createUpload(project, token, bytes.length); expect(created.status).toBe(201); From b3eeffa7227ea3197953975ac43e19af9d3743f6 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 14:48:50 +0200 Subject: [PATCH 04/18] test(video): prove local E2E and prepare rollout Replace fake readiness and direct SQL promotion with generated media uploaded through local multipart R2, processed by the real Workflow and pinned FFmpeg Container, then verified through probe, authenticated ranges, playlist inclusion, retirement, and retained bytes.\n\nAdd reproducible processor benchmarks, isolated production resource declarations capped at two, structured processing events, and an operations runbook covering approval, provisioning, smoke, observability, rollback, and retained storage. Remove superseded Stream measurement and archive surfaces, and include processor plus real-byte readiness in the complete verification gate. --- .dev.vars.example | 4 +- .github/workflows/deploy.yml | 2 +- .github/workflows/video-archive.yml | 34 -- .github/workflows/video-measure.yml | 26 - README.md | 62 +- VIDEO_ROLLOUT.md | 153 +++++ package.json | 10 +- scripts/archive-to-drive.ts | 72 --- scripts/benchmark-video-processor.ts | 135 +++++ scripts/local-readiness.ts | 724 +++++++++++++++++------ scripts/local-video-workflow.ts | 469 --------------- src/shared/videos.ts | 3 - src/worker/index.ts | 7 - src/worker/integrations/stream/fake.ts | 86 --- src/worker/integrations/stream/index.ts | 40 -- src/worker/integrations/stream/real.ts | 162 ----- src/worker/integrations/stream/types.ts | 38 -- src/worker/middleware/service-auth.ts | 28 - src/worker/workflows/video-processing.ts | 33 +- test/e2e/video-rollout.test.ts | 75 +++ vitest.config.ts | 5 - worker-configuration.d.ts | 3 +- wrangler.production.json | 9 +- 23 files changed, 1000 insertions(+), 1180 deletions(-) delete mode 100644 .github/workflows/video-archive.yml delete mode 100644 .github/workflows/video-measure.yml create mode 100644 VIDEO_ROLLOUT.md delete mode 100644 scripts/archive-to-drive.ts create mode 100644 scripts/benchmark-video-processor.ts delete mode 100644 scripts/local-video-workflow.ts delete mode 100644 src/worker/integrations/stream/fake.ts delete mode 100644 src/worker/integrations/stream/index.ts delete mode 100644 src/worker/integrations/stream/real.ts delete mode 100644 src/worker/integrations/stream/types.ts delete mode 100644 src/worker/middleware/service-auth.ts create mode 100644 test/e2e/video-rollout.test.ts diff --git a/.dev.vars.example b/.dev.vars.example index cad93a0..1726ab0 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -6,6 +6,6 @@ GOOGLE_CLIENT_SECRET= GOOGLE_REDIRECT_URI=http://localhost:5173/api/auth/callback ALLOWED_EMAIL_DOMAIN="sentry.io" -# R2 uploads are processed by the local Workflow and pinned FFmpeg Container. +# R2 uploads use the local Workflow and pinned FFmpeg Container declared in +# wrangler.jsonc. No Cloudflare video credentials or remote resources are needed. # Ready derivatives play through the authenticated same-origin MP4 endpoint. -VIDEO_SERVICE_TOKEN="replace-with-a-local-job-token" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6afc227..819ab2a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -39,7 +39,7 @@ jobs: run: | test -n "$CLOUDFLARE_API_TOKEN" || { echo 'Missing CLOUDFLARE_API_TOKEN'; exit 1; } test "$CLOUDFLARE_ACCOUNT_ID" = '773afa1f62ff86c80db4f24f7ff1e9c8' || { echo 'Unexpected Cloudflare account'; exit 1; } - node -e "const c=require('./wrangler.production.json'); if (c.account_id !== '773afa1f62ff86c80db4f24f7ff1e9c8' || c.vars.STREAM_MODE !== 'disabled') process.exit(1); for (const value of [c.d1_databases[0].database_id,c.r2_buckets[0].bucket_name,c.vars.APP_ORIGIN,c.vars.GOOGLE_REDIRECT_URI,c.vars.GOOGLE_CLIENT_ID]) if (!value || /replace.me/i.test(value) || value === '00000000-0000-0000-0000-000000000000') process.exit(1)" + node -e "const c=require('./wrangler.production.json'); const videos=c.r2_buckets.find(x=>x.binding==='VIDEOS'); const workflow=c.workflows.find(x=>x.binding==='VIDEO_PROCESSING_WORKFLOW'); const container=c.containers.find(x=>x.class_name==='VideoProcessorContainer'); if (c.account_id !== '773afa1f62ff86c80db4f24f7ff1e9c8' || videos?.bucket_name !== 'hackweek-video-media-production' || workflow?.name !== 'hackweek-video-processing-production' || container?.name !== 'hackweek-video-processor-production' || container?.max_instances !== 2 || c.vars.VIDEO_PROCESSOR_CONCURRENCY !== '2') process.exit(1); for (const value of [c.d1_databases[0].database_id,c.r2_buckets[0].bucket_name,c.vars.APP_ORIGIN,c.vars.GOOGLE_REDIRECT_URI,c.vars.GOOGLE_CLIENT_ID]) if (!value || /replace.me/i.test(value) || value === '00000000-0000-0000-0000-000000000000') process.exit(1)" - run: npm run build - name: Apply reviewed D1 migrations run: npx wrangler d1 migrations apply hackweek-db --remote --config wrangler.production.json --yes diff --git a/.github/workflows/video-archive.yml b/.github/workflows/video-archive.yml deleted file mode 100644 index 7571d4c..0000000 --- a/.github/workflows/video-archive.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Archive videos to Drive - -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: video-archive - cancel-in-progress: false - -jobs: - archive: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24.19.0 - cache: npm - - run: npm ci - - name: Install rclone - run: curl https://rclone.org/install.sh | sudo bash - - name: Configure Drive and archive ready videos - run: | - mkdir -p ~/.config/rclone - printf '%s' "$RCLONE_CONFIG" > ~/.config/rclone/rclone.conf - npx tsx scripts/archive-to-drive.ts - env: - VIDEO_API_URL: ${{ secrets.VIDEO_API_URL }} - VIDEO_SERVICE_TOKEN: ${{ secrets.VIDEO_SERVICE_TOKEN }} - RCLONE_CONFIG: ${{ secrets.RCLONE_CONFIG }} - RCLONE_DRIVE_DESTINATION: ${{ vars.RCLONE_DRIVE_DESTINATION }} diff --git a/.github/workflows/video-measure.yml b/.github/workflows/video-measure.yml deleted file mode 100644 index 9c7cd43..0000000 --- a/.github/workflows/video-measure.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Measure video loudness - -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: video-measurement - cancel-in-progress: false - -jobs: - measure: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24.19.0 - cache: npm - - run: npm ci - - run: npx tsx scripts/measure-loudness.ts - env: - VIDEO_API_URL: ${{ secrets.VIDEO_API_URL }} - VIDEO_SERVICE_TOKEN: ${{ secrets.VIDEO_SERVICE_TOKEN }} diff --git a/README.md b/README.md index bce12a0..475764e 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,85 @@ # Sentry Hackweek -Hackweek is an internal React + TypeScript application served by one Hono Cloudflare Worker. Application-owned Google OAuth authenticates users, D1 owns sessions/data/roles, and private R2 stores attachments. The core production rollout serves the SPA with Cloudflare Static Assets at `https://hackweek.getsentry.workers.dev` and keeps `STREAM_MODE=disabled`; Stream, video screening, and archive operations are dormant until a separately approved rollout. The UI preserves the Sentry `#HACKWEEK` identity and archive hierarchy. +Hackweek is an internal React + TypeScript application served by one Hono Cloudflare Worker. Application-owned Google OAuth authenticates users, D1 owns sessions/data/roles, and private R2 stores attachments plus immutable video originals and canonical MP4 derivatives. Project videos are processed by a Cloudflare Workflow using the pinned FFmpeg Container in `Dockerfile.video-processor`; ready media is served only through authenticated same-origin range endpoints. ## Requirements - Node.js 24.11 or newer (Volta and CI pin 24.19) - npm 11 or newer +- Docker with a running Linux engine (Docker Desktop or OrbStack) +- `ffmpeg` and `ffprobe` 8.x on the host for generated local fixtures -## Deterministic local start +No Cloudflare video resource or credential is required for local development. + +## Local video environment + +Complete the one-time setup without replacing an existing `.dev.vars`: ```bash npm ci -cp .dev.vars.example .dev.vars -rm -rf .wrangler/state +[ -f .dev.vars ] || cp .dev.vars.example .dev.vars npm run db:migrate:local npm run migrate:local -- \ --database test/fixtures/firebase/database.json \ --storage-manifest test/fixtures/firebase/storage-manifest.json \ --storage-root test/fixtures/firebase/storage -npm run dev ``` -Before starting the app, configure a Google OAuth Web application to allow the JavaScript origin `http://localhost:5173` and redirect URI `http://localhost:5173/api/auth/callback`. Replace the placeholders in `.dev.vars` with its client ID and the client secret from the shared vault; never commit `.dev.vars`. +Configure the Google OAuth Web application in `.dev.vars` for JavaScript origin `http://localhost:5173` and redirect URI `http://localhost:5173/api/auth/callback`. Use the shared-vault client secret; never commit `.dev.vars`. -Open `http://localhost:5173` and sign in with Google. D1 remains the sole role authority. To promote your local user after signing in once, replace the email below and run: +Then one command starts the application, local D1/R2, local Workflow, and the real pinned FFmpeg Container: + +```bash +npm run dev:video +``` + +Open `http://localhost:5173`, sign in, and use a current project’s video panel. Uploading a video performs real multipart local-R2 upload and Workflow/Container processing. When the status becomes ready, verify project playback, then save the project in the admin screening order and open the year reel. Originals and derivatives remain private and are retained after video retirement. + +To promote a local user after signing in once, replace the email below and run: ```bash npx wrangler d1 execute hackweek-db --local --command \ "UPDATE users SET is_admin = 1, updated_at = CURRENT_TIMESTAMP WHERE google_subject IS NOT NULL AND email = 'you@sentry.io'" ``` -Resetting `.wrangler/state` removes the promotion. Never run that command with `--remote`. +Never run that command with `--remote`. -## Authentication +### Troubleshooting + +- **Container does not start:** run `docker version` and `npm run video:processor:build`. Both client and server must be available. +- **Upload remains queued:** keep `npm run dev:video` running and inspect its Workflow step output. Local processing concurrency is intentionally one. +- **OAuth callback fails:** ensure `APP_ORIGIN`, the Google allowed origin, and `GOOGLE_REDIRECT_URI` all use `http://localhost:5173` exactly. +- **Stale local data:** stop the app and remove only `.wrangler/state`, then repeat the local migrations. This never touches remote resources. +- **Playback fails:** confirm the video is ready and signed-in playback returns `200` or `206`; unready, retired, and anonymous reads are intentionally rejected. -Google OAuth is the only browser authentication path in every environment, including local development. It uses the Authorization Code flow with PKCE, state, nonce, confidential server exchange, Google JWKS validation, exact verified `@sentry.io` enforcement, hashed opaque D1 sessions, and HttpOnly cookies. +## Automated real-byte readiness -All core browser APIs except health require a D1-backed user. Authenticated mutations and logout require an exact same-origin `Origin` header. Logout revokes the current D1 session. Login rotates existing sessions. Google/client claims never grant admin access. Dormant Stream webhook and video-job endpoints use separate machine-auth boundaries if real Stream is approved later. +```bash +npm run test:video-processor +npm run test:readiness +``` + +`test:video-processor` builds the pinned image and covers loud, silent, rotated, low-resolution, malformed, over-duration, and forced-failure fixtures generated at runtime. `test:readiness` creates isolated temporary D1/R2 state, preserves the developer’s `.dev.vars`, uploads generated MP4 bytes through multipart R2, resumes the upload, runs the local Workflow and Container, probes the canonical output, verifies authenticated full/range playback and curated playlist inclusion, retires the submission, proves retained objects, then removes its processes and state. + +For a manual browser companion pass, run `npm run dev:video` and check: + +1. pause/reload during upload and resume from the recorded part; +2. queued → processing → ready status; +3. project playback seek (range delivery); +4. curated reel overlay, pause, skip, fullscreen, and advance; +5. retirement removes playback/reel visibility without deleting stored bytes. + +## Authentication + +Google OAuth is the only browser authentication path. It uses Authorization Code with PKCE, state and nonce validation, Google JWKS verification, exact verified `@sentry.io` enforcement, hashed opaque D1 sessions, and HttpOnly cookies. D1 is the sole role authority. Authenticated mutations require the exact same-origin `Origin` header. ## Quality gates ```bash npm run verify +npm audit --omit=dev --audit-level=high ``` -This generates binding types, typechecks, checks formatting/lint, runs Worker/frontend/migration/player tests, builds, performs a credential-free deployment dry run, and runs an isolated seeded D1/R2 journey with fake-Stream contract coverage. Local tests do not prove real Google OAuth, deployed bindings, or imported data. Real Stream is outside that gate. +The deterministic gate generates binding types, typechecks, checks formatting/lint, runs Worker/frontend/migration/player tests, builds, performs a credential-free production dry run, builds and exercises the real pinned processor, and runs the isolated real-byte local E2E. It does not deploy, provision, access remote resources, or prove real Google OAuth. + +Production resource names, benchmark evidence, observability, rollout, smoke, rollback, and retained-storage policy are documented in [`VIDEO_ROLLOUT.md`](VIDEO_ROLLOUT.md). Every production mutation remains behind explicit later human approval. diff --git a/VIDEO_ROLLOUT.md b/VIDEO_ROLLOUT.md new file mode 100644 index 0000000..c7231f3 --- /dev/null +++ b/VIDEO_ROLLOUT.md @@ -0,0 +1,153 @@ +# Video processing production rollout + +This runbook prepares the R2 + Workflow + Container path. It does not authorize or perform any Cloudflare production mutation. + +## Prepared resource contract + +The production declaration in `wrangler.production.json` uses these isolated future video resources: + +| Binding | Proposed resource | Purpose | +| --------------------------- | -------------------------------------------------------- | -------------------------------------------------------------- | +| `VIDEOS` | R2 bucket `hackweek-video-media-production` | Private immutable originals and canonical derivatives | +| `VIDEO_PROCESSING_WORKFLOW` | Workflow `hackweek-video-processing-production` | One durable instance per video attempt | +| `VIDEO_PROCESSOR` | Container Durable Object class `VideoProcessorContainer` | Attempt-isolated FFmpeg invocation | +| Container application | `hackweek-video-processor-production` | Digest-pinned `Dockerfile.video-processor` image | +| `DB` | Existing `hackweek-db` binding | Upload, attempt, fencing, state, and retained-object inventory | + +Production declares both `max_instances: 2` and `VIDEO_PROCESSOR_CONCURRENCY=2`. Keep the two values equal. Local development uses one. The Container has no Internet access and receives no R2 account credential; its outbound handler scopes source/output access to the current D1 attempt. + +Required non-secret variables are `APP_ORIGIN`, `GOOGLE_REDIRECT_URI`, `GOOGLE_CLIENT_ID`, `ALLOWED_EMAIL_DOMAIN`, `VIDEO_PROCESSOR_CONCURRENCY`, and `VIDEO_PROCESSING_AUTOSTART`. `GOOGLE_CLIENT_SECRET` remains the required Worker secret. There is no Stream, HLS, public-R2, service-token, Queue, or general R2 credential binding. + +## Explicit approval boundary + +A human production owner must approve the exact commit, account, names, expected storage growth, benchmark/tuning decision, D1 backup window, and smoke/rollback operators before any command below that uses `--remote`, `r2 bucket create`, `secret put`, `deploy`, or `rollback` is run. + +Until that approval, only these non-mutating local checks are allowed: + +```bash +npm ci +npm run verify +npm audit --omit=dev --audit-level=high +npm run video:benchmark +npm run build +npm run deploy:dry-run +``` + +`deploy:dry-run` compiles the Worker, validates bindings, and builds the Container locally. It does not upload, create, list, or mutate a Cloudflare resource. + +## Local benchmark evidence + +Command: `npm run video:benchmark` + +Environment recorded 2026-08-11: OrbStack Linux ARM64 Docker engine 29.4.0 on an ARM64 development host; digest-pinned FFmpeg 8.0.1 image. CPU is the delta of cgroup `usage_usec` for the running Container; wall time wraps the `process-file` invocation. Fixtures are generated at runtime with synthetic motion plus audible 48 kHz audio. They are deliberately short and bounded, so these numbers validate profiles and provide a tuning baseline—not a production cost, throughput, or ten-minute latency promise. + +| Profile | Source | Input | Output | Wall | Container CPU | Output loudness | +| ---------------- | ---------------------- | ----------: | ----------: | ------: | ------------: | --------------: | +| correctness-360p | 640×360, 2 s, 24 fps | 435,740 B | 263,556 B | 0.805 s | 1.052 s | -15.95 LUFS | +| bounded-720p | 1280×720, 4 s, 24 fps | 3,252,808 B | 1,829,962 B | 0.657 s | 2.181 s | -15.96 LUFS | +| bounded-1080p | 1920×1080, 4 s, 24 fps | 7,263,137 B | 3,620,283 B | 1.008 s | 4.464 s | -15.96 LUFS | + +Before production approval, repeat the benchmark on the release commit and run a separately approved bounded staging sample representative of expected durations. Start with concurrency two only if p95 processing wall time, Container CPU/memory, scratch disk, Workflow retries, and queued wait remain within the agreed event window. Reduce both concurrency declarations together if account/container pressure appears; increasing beyond two requires a new review and benchmark. + +## Later provisioning order (mutating; approval required) + +The following is an operator checklist, not deployment automation. Stop if the configured Cloudflare account is not `773afa1f62ff86c80db4f24f7ff1e9c8` or any proposed resource name is already owned for another purpose. + +1. Record the release commit and current Worker version ID for rollback. Obtain an explicit go/no-go from the production owner. +2. Run all local checks from the approval section and archive their output with the release record. +3. Create the isolated private video bucket: + + ```bash + npx wrangler r2 bucket create hackweek-video-media-production \ + --config wrangler.production.json + ``` + +4. Set the existing Worker’s Google OAuth secret if it is not already present. The command prompts securely; never place the value in shell history: + + ```bash + npx wrangler secret put GOOGLE_CLIENT_SECRET \ + --config wrangler.production.json + ``` + +5. Apply reviewed D1 migrations during the approved backup window: + + ```bash + npx wrangler d1 migrations apply hackweek-db \ + --remote --config wrangler.production.json + ``` + +6. Deploy the reviewed declaration: + + ```bash + npx wrangler deploy --config wrangler.production.json \ + --containers-rollout gradual + ``` + + Wrangler materializes the declared `hackweek-video-processing-production` Workflow, `hackweek-video-processor-production` Container application/image, and `VideoProcessorContainer` Durable Object migration as part of this approved deploy. Do not create similarly named resources by hand. + +7. Perform the smoke checklist below with one small generated/approved non-sensitive clip before allowing event uploads. + +The existing manual GitHub deployment workflow remains an alternative controlled entry point only after its environment approval and typed confirmation. Do not run both paths for one release. + +## Production smoke criteria + +Use a test project and authenticated creator/member/admin accounts. The release is healthy only when all checks pass: + +1. Anonymous create, playback descriptor, and content requests return `401`; a non-member upload returns `403`. +2. Multipart create → part upload → refresh/resume → completion succeeds; duplicate completion returns the same video/attempt. +3. State advances queued → processing → ready, and Workflow instance `video--attempt-1` completes all named steps. +4. D1 records distinct original/processed keys, duration ≤600 s, and loudness within -16 ±0.7 LUFS; the downloaded derivative probes as H.264/AAC `yuv420p`. +5. Authenticated content returns `200`, a seek returns exact `206`/`Content-Range`, and an unsatisfiable range returns `416`. +6. Only the ready video appears in saved screening order with the correct project/team overlay. Pause, skip, fullscreen, ended advance, and recoverable error advance work in the event browser. +7. Confirmed retirement removes project/reel playback while both R2 objects remain present. Do not delete the smoke objects. +8. Two concurrent independent project jobs can run; a third waits/retries. A same-project second active upload conflicts. + +Useful read-only diagnostics after deployment: + +```bash +npx wrangler tail hackweek --config wrangler.production.json --format json +npx wrangler workflows instances describe \ + hackweek-video-processing-production video--attempt- \ + --config wrangler.production.json +``` + +## Observability and alerts + +`observability.enabled` is set in production. Workflow logs emit JSON with `component=video-processing`, an event name, `videoId`, attempt, and bounded failure text; they never include object keys, media bytes, cookies, OAuth values, or R2 credentials. + +Create dashboard/alert ownership before rollout for: + +- Workflow failed/terminated instance count >0 over 5 minutes; +- `processor_failed` or `claim_failed` events >0, grouped by attempt and bounded error; +- oldest queued/running D1 attempt >10 minutes; +- queued depth above 2 for 10 minutes (capacity pressure at cap two); +- Worker `/api/projects/*/video*` and `/api/videos/*/content` 5xx rate >1% over 5 minutes; +- Container CPU, memory, scratch disk, restart, and timeout pressure; +- `project_videos.status='failed'` growth and retries per video; +- R2 object count/bytes and monthly growth for `hackweek-video-media-production`. + +Never place full request headers, session cookies, source/output object keys, or media payloads in an alert. Link alerts to this runbook and name an event-time operator. + +## Rollback + +Rollback is state-preserving. Do not delete R2 objects, Workflow instances, D1 rows, or the Container application during incident response. + +1. Pause new video completion/processing by preparing `VIDEO_PROCESSING_AUTOSTART=false` on the reviewed incident commit and deploying it through the same approved path. New completed uploads remain queued rather than being published incorrectly. +2. If the Worker release itself is faulty, roll back to the recorded compatible Worker version: + + ```bash + npx wrangler rollback \ + --name hackweek --message "Rollback video rollout: " + ``` + +3. Leave queued/running/failed attempt rows and all original/derivative objects intact. Inspect current attempt fencing before any retry. A late result cannot publish over a retired or newer attempt. +4. Restore service only after `npm run verify`, the production smoke subset, and incident-owner approval pass on the corrective release. Re-enable autostart and keep concurrency at or below two. +5. Reconcile status and inventory; do not manually mark a video ready and do not copy an unprobed object into a canonical key. + +D1 migration 0007 is forward-only. Rollback does not reverse it or restore the old Stream lifecycle. + +## Retained-storage policy + +No automatic deletion is permitted for completed originals or derivatives, including retired submissions and stale completed derivatives. Only incomplete expired multipart uploads may be aborted. The retained bytes support recovery and later delivery changes, but storage growth is an accepted operational risk. + +Track per-video original/derivative keys and sizes in the monthly inventory, alert on growth, and review retention with the data owner after the event. Any future deletion policy requires separate human approval, an inventory/export plan, and a new implementation; it is not part of this rollout. diff --git a/package.json b/package.json index 7d3c698..e6fe791 100644 --- a/package.json +++ b/package.json @@ -14,22 +14,22 @@ "test:app": "vp test run --config vitest.app.config.ts", "test:migration": "vp test run --config vitest.migration.config.ts", "test:readiness": "tsx scripts/local-readiness.ts", - "verify": "npm run cf-typegen && npm run typecheck && npm run format:check && npm run lint && npm test && npm run build && npm run deploy:dry-run && npm run test:readiness", + "verify": "npm run cf-typegen && npm run typecheck && npm run format:check && npm run lint && npm test && npm run build && npm run deploy:dry-run && npm run test:video-processor && npm run test:readiness", "deploy:dry-run": "wrangler deploy --dry-run --config wrangler.production.json --outdir .wrangler/deploy-dry-run", "build": "vp build", "preview": "vp preview", "cf-typegen": "wrangler types worker-configuration.d.ts --config wrangler.production.json", "db:migrate:local": "wrangler d1 migrations apply hackweek-db --local", - "video:dev": "npm run db:migrate:local && vp dev", + "dev:video": "npm run db:migrate:local && vp dev", "video:processor:build": "docker build --file Dockerfile.video-processor --tag hackweek-video-processor:local .", + "video:benchmark": "tsx scripts/benchmark-video-processor.ts", "test:video-processor": "tsx scripts/test-video-processor.ts", - "test:video-workflow": "tsx scripts/local-video-workflow.ts", + "test:video-workflow": "npm run test:readiness", "migrate:validate": "tsx scripts/migrate/cli.ts validate", "migrate:dry-run": "tsx scripts/migrate/cli.ts dry-run", "migrate:local": "tsx scripts/migrate/cli.ts import --target local", "migrate:cloudflare": "tsx scripts/migrate/cli.ts import --target cloudflare", - "migrate:reconcile": "tsx scripts/migrate/cli.ts reconcile", - "video:archive": "tsx scripts/archive-to-drive.ts" + "migrate:reconcile": "tsx scripts/migrate/cli.ts reconcile" }, "dependencies": { "@cloudflare/containers": "0.3.7", diff --git a/scripts/archive-to-drive.ts b/scripts/archive-to-drive.ts deleted file mode 100644 index 0363d1f..0000000 --- a/scripts/archive-to-drive.ts +++ /dev/null @@ -1,72 +0,0 @@ -import {spawn} from 'node:child_process'; -import process from 'node:process'; - -interface QueueItem { - videoId: string; - fileName: string; - downloadUrl: string; -} - -const apiUrl = required('VIDEO_API_URL').replace(/\/$/, ''); -const serviceToken = required('VIDEO_SERVICE_TOKEN'); -const driveDestination = required('RCLONE_DRIVE_DESTINATION').replace(/\/$/, ''); -const queue = await request<{videos: QueueItem[]}>('/api/video-jobs/archives'); - -for (const video of queue.videos) { - try { - await run('rclone', [ - 'copyurl', - '--no-clobber', - video.downloadUrl, - `${driveDestination}/${video.fileName}`, - ]); - await report(video.videoId, 'archived', null); - console.log(`Archived ${video.videoId} as ${video.fileName}`); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - await report(video.videoId, 'failed', message); - console.error(`Archive failed for ${video.videoId}: ${message}`); - } -} - -function run(command: string, args: string[]) { - return new Promise((resolve, reject) => { - const child = spawn(command, args, {stdio: ['ignore', 'inherit', 'pipe']}); - let stderr = ''; - child.stderr.setEncoding('utf8'); - child.stderr.on('data', (chunk: string) => (stderr += chunk)); - child.once('error', reject); - child.once('close', (code) => - code === 0 - ? resolve() - : reject(new Error(`rclone exited ${code}: ${stderr.slice(-500)}`)), - ); - }); -} - -function report(videoId: string, status: 'archived' | 'failed', error: string | null) { - return request(`/api/video-jobs/archives/${encodeURIComponent(videoId)}`, { - method: 'POST', - body: JSON.stringify({status, error}), - }); -} - -async function request(path: string, init: RequestInit = {}) { - const response = await fetch(`${apiUrl}${path}`, { - ...init, - headers: new Headers({ - Authorization: `Bearer ${serviceToken}`, - ...(init.body ? {'Content-Type': 'application/json'} : {}), - ...Object.fromEntries(new Headers(init.headers).entries()), - }), - }); - if (!response.ok) - throw new Error(`Video API returned ${response.status}: ${await response.text()}`); - return (response.status === 204 ? undefined : await response.json()) as T; -} - -function required(name: string) { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -} diff --git a/scripts/benchmark-video-processor.ts b/scripts/benchmark-video-processor.ts new file mode 100644 index 0000000..387ccd6 --- /dev/null +++ b/scripts/benchmark-video-processor.ts @@ -0,0 +1,135 @@ +#!/usr/bin/env node +import {execFileSync, spawnSync} from 'node:child_process'; +import {chmod, mkdtemp, rm, stat} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; + +const root = process.cwd(); +const image = 'hackweek-video-processor:local'; +const work = await mkdtemp(path.join(tmpdir(), 'hackweek-video-benchmark-')); +const container = `hackweek-video-benchmark-${process.pid}`; +const profiles = [ + {name: 'correctness-360p', width: 640, height: 360, duration: 2}, + {name: 'bounded-720p', width: 1280, height: 720, duration: 4}, + {name: 'bounded-1080p', width: 1920, height: 1080, duration: 4}, +]; +let containerStarted = false; + +try { + run('docker', ['build', '--file', 'Dockerfile.video-processor', '--tag', image, '.']); + await chmod(work, 0o777); + for (const profile of profiles) { + ffmpeg([ + '-f', + 'lavfi', + '-i', + `testsrc2=size=${profile.width}x${profile.height}:rate=24`, + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:sample_rate=48000', + '-t', + String(profile.duration), + '-af', + 'volume=0.05', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a', + 'aac', + '-shortest', + path.join(work, `${profile.name}-input.mp4`), + ]); + } + await Promise.all( + profiles.map((profile) => chmod(path.join(work, `${profile.name}-input.mp4`), 0o644)), + ); + + run('docker', [ + 'run', + '--detach', + '--rm', + '--name', + container, + '--volume', + `${work}:/work`, + image, + ]); + containerStarted = true; + const results = []; + for (const profile of profiles) { + const input = `/work/${profile.name}-input.mp4`; + const output = `/work/${profile.name}-output.mp4`; + const cpuBefore = containerCpuUsec(); + const wallStarted = performance.now(); + const metadata = JSON.parse( + outputOf('docker', [ + 'exec', + container, + 'node', + '/app/video-processor.mjs', + 'process-file', + input, + output, + ]) + .trim() + .split('\n') + .at(-1)!, + ) as {width: number; height: number; loudnessLufs: number | null}; + const wallSeconds = (performance.now() - wallStarted) / 1000; + const cpuSeconds = (containerCpuUsec() - cpuBefore) / 1_000_000; + const inputBytes = (await stat(path.join(work, `${profile.name}-input.mp4`))).size; + const outputBytes = (await stat(path.join(work, `${profile.name}-output.mp4`))).size; + results.push({ + profile: profile.name, + source: `${profile.width}x${profile.height}, ${profile.duration}s, 24fps`, + inputBytes, + outputBytes, + wallSeconds: Number(wallSeconds.toFixed(3)), + cpuSeconds: Number(cpuSeconds.toFixed(3)), + outputResolution: `${metadata.width}x${metadata.height}`, + loudnessLufs: metadata.loudnessLufs, + }); + } + + console.table(results); + console.log(JSON.stringify({image, profiles: results}, null, 2)); +} finally { + if (containerStarted) { + const removed = spawnSync('docker', ['rm', '--force', container], { + cwd: root, + stdio: 'ignore', + }); + if (removed.status !== 0) { + console.error(`Could not remove benchmark container ${container}`); + process.exitCode = 1; + } + } + await rm(work, {recursive: true, force: true}); +} + +function containerCpuUsec() { + const stats = outputOf('docker', [ + 'exec', + container, + 'sh', + '-c', + "awk '/^usage_usec / {print $2}' /sys/fs/cgroup/cpu.stat", + ]).trim(); + const value = Number(stats); + if (!Number.isFinite(value)) throw new Error(`Invalid container CPU usage: ${stats}`); + return value; +} + +function ffmpeg(args: string[]) { + run('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-nostdin', '-y', ...args]); +} + +function run(command: string, args: string[]) { + execFileSync(command, args, {cwd: root, stdio: 'inherit'}); +} + +function outputOf(command: string, args: string[]) { + return execFileSync(command, args, {cwd: root, encoding: 'utf8'}); +} diff --git a/scripts/local-readiness.ts b/scripts/local-readiness.ts index d1f2574..835aa3f 100644 --- a/scripts/local-readiness.ts +++ b/scripts/local-readiness.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node -import {execFileSync, spawn, type ChildProcess} from 'node:child_process'; +import {execFileSync, spawn, spawnSync, type ChildProcess} from 'node:child_process'; import {createHash} from 'node:crypto'; -import {mkdtemp, rename, rm, stat, writeFile} from 'node:fs/promises'; +import {mkdtemp, readFile, rm, stat, writeFile} from 'node:fs/promises'; import {tmpdir} from 'node:os'; import path from 'node:path'; @@ -10,20 +10,26 @@ const fixture = path.join(root, 'test/fixtures/firebase'); const state = await mkdtemp(path.join(tmpdir(), 'hackweek-readiness-')); const port = Number(process.env.READINESS_PORT ?? 5199); const origin = `http://127.0.0.1:${port}`; +const config = path.join(state, 'wrangler.readiness.json'); +const source = path.join(state, 'readiness-source.mp4'); +const original = path.join(state, 'readiness-original.mp4'); +const derivative = path.join(state, 'readiness-derivative.mp4'); const googleClientId = 'local-readiness.apps.googleusercontent.com'; -const googleClientSecret = 'local-readiness-client-secret'; +const googleClientSecret = 'synthetic-readiness-value'; const sessionToken = createHash('sha256') .update('hackweek-local-readiness') .digest('base64url'); const sessionTokenHash = createHash('sha256').update(sessionToken).digest('hex'); -const devVars = `APP_ORIGIN="${origin}"\nGOOGLE_CLIENT_ID="${googleClientId}"\nGOOGLE_CLIENT_SECRET="${googleClientSecret}"\nGOOGLE_REDIRECT_URI="${origin}/api/auth/callback"\nALLOWED_EMAIL_DOMAIN="sentry.io"\nSTREAM_MODE="fake"\nSTREAM_ALLOWED_ORIGIN="localhost"\nSTREAM_DELIVERY_HOST="customer-fake.cloudflarestream.com"\nSTREAM_WEBHOOK_SECRET="local-readiness-webhook-secret"\nVIDEO_SERVICE_TOKEN="local-readiness-video-service-token"\n`; +const rootDevVars = path.join(root, '.dev.vars'); +const devVarsBefore = await optionalFile(rootDevVars); +const dockerContainersBefore = new Set(dockerContainerNames()); let server: ChildProcess | undefined; const serverLog: string[] = []; -const rootDevVars = path.join(root, '.dev.vars'); -const savedDevVars = path.join(state, '.dev.vars.saved'); -const hadDevVars = await exists(rootDevVars); try { + await writeFile(config, JSON.stringify(localConfig()), {mode: 0o600}); + await writeFile(path.join(state, '.dev.vars'), localDevVars(), {mode: 0o600}); + run('npx', [ 'wrangler', 'd1', @@ -33,6 +39,8 @@ try { '--local', '--persist-to', state, + '--config', + config, ]); run('npm', [ 'run', @@ -44,6 +52,10 @@ try { path.join(fixture, 'storage-manifest.json'), '--storage-root', path.join(fixture, 'storage'), + '--bucket-name', + 'hackweek-attachments-readiness', + '--config', + config, '--persist-to', state, ]); @@ -59,62 +71,58 @@ try { path.join(fixture, 'storage'), '--target', 'local', + '--bucket-name', + 'hackweek-attachments-readiness', + '--config', + config, '--persist-to', state, ]); - const now = Math.floor(Date.now() / 1000); - sql( - `INSERT INTO users - (id, source_uid, google_subject, email, display_name, avatar_url) - VALUES - ('local-readiness-user', 'local-readiness-user', 'google-readiness-user', - 'developer@sentry.io', 'Local Developer', NULL); - INSERT INTO user_sessions - (token_hash, user_id, expires_at, created_at, last_used_at) - VALUES - ('${sessionTokenHash}', 'local-readiness-user', ${now + 28_800}, ${now}, ${now});`, - ); - const isolatedConfig = path.join(state, 'wrangler.readiness.json'); - await writeFile(path.join(state, '.dev.vars'), devVars, {mode: 0o600}); - await writeFile( - isolatedConfig, - JSON.stringify({ - name: 'hackweek-readiness', - main: path.join(root, 'src/worker/index.ts'), - compatibility_date: '2026-08-03', - assets: { - directory: path.join(root, 'public'), - not_found_handling: 'single-page-application', - run_worker_first: ['/api/*'], - }, - d1_databases: [ - { - binding: 'DB', - database_name: 'hackweek-db', - database_id: 'local', - migrations_dir: path.join(root, 'migrations'), - }, - ], - r2_buckets: [{binding: 'ATTACHMENTS', bucket_name: 'hackweek-attachments-local'}], - vars: { - APP_ORIGIN: origin, - GOOGLE_CLIENT_ID: googleClientId, - GOOGLE_CLIENT_SECRET: googleClientSecret, - GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, - ALLOWED_EMAIL_DOMAIN: 'sentry.io', - STREAM_MODE: 'fake', - STREAM_ALLOWED_ORIGIN: 'localhost', - STREAM_DELIVERY_HOST: 'customer-fake.cloudflarestream.com', - STREAM_WEBHOOK_SECRET: 'local-readiness-webhook-secret', - VIDEO_SERVICE_TOKEN: 'local-readiness-video-service-token', - }, - }), - {mode: 0o600}, - ); + const now = Math.floor(Date.now() / 1000); + sql(` + INSERT INTO users + (id, source_uid, google_subject, email, display_name, avatar_url, is_admin) + VALUES + ('readiness-user', 'readiness-user', 'google-readiness-user', + 'developer@sentry.io', 'Local Developer', NULL, 1); + INSERT INTO user_sessions + (token_hash, user_id, expires_at, created_at, last_used_at) + VALUES ('${sessionTokenHash}', 'readiness-user', ${now + 28_800}, ${now}, ${now}); + INSERT INTO years (id) VALUES ('9999'); + INSERT INTO groups (id, source_id, year_id, name, creator_id) + VALUES ('readiness-group', 'readiness-group', '9999', 'Readiness Team', 'readiness-user'); + INSERT INTO projects + (id, source_id, year_id, creator_id, group_id, name, summary, kind) + VALUES + ('readiness-project', 'readiness-project', '9999', 'readiness-user', + 'readiness-group', 'Readiness Video', 'Real local video E2E', 'project'); + INSERT INTO project_members (project_id, user_id) + VALUES ('readiness-project', 'readiness-user'); + `); - if (hadDevVars) await rename(rootDevVars, savedDevVars); - await writeFile(rootDevVars, devVars, {mode: 0o600}); + ffmpeg([ + '-f', + 'lavfi', + '-i', + 'testsrc2=size=640x360:rate=24', + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:sample_rate=48000', + '-t', + '2', + '-af', + 'volume=0.05', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a', + 'aac', + '-shortest', + source, + ]); server = spawn( process.execPath, @@ -130,134 +138,410 @@ try { { cwd: root, detached: true, - env: readinessEnv(), + env: localEnvironment(), stdio: ['ignore', 'pipe', 'pipe'], }, ); server.stdout?.on('data', (chunk) => serverLog.push(String(chunk))); server.stderr?.on('data', (chunk) => serverLog.push(String(chunk))); - await waitForServer(serverLog); + await waitForServer(); const login = await request('/api/auth/login', {redirect: 'manual'}); const authorization = new URL(login.headers.get('Location')!); assert( authorization.searchParams.get('client_id') === googleClientId, - 'Google OAuth client is configured', - ); - assert( - authorization.searchParams.get('redirect_uri') === `${origin}/api/auth/callback`, - 'Google OAuth loopback redirect is configured', + 'loopback Google OAuth configuration is active', ); - const session = await get('/api/session'); - assert(session.user.role === 'member', 'seeded Google session starts as a member'); - assert(session.user.email === 'developer@sentry.io', 'seeded Google user is explicit'); - - const years = await get('/api/years'); assert( - years.years.some((year: {id: string}) => year.id === '2024'), - 'archive is seeded', + session.user.role === 'admin' && session.user.email === 'developer@sentry.io', + 'seeded D1 session authenticates the readiness administrator', ); + const unauthorized = await fetch(`${origin}/api/projects/readiness-project/video`); + assert(unauthorized.status === 401, 'video APIs reject unauthenticated requests'); + const projects = await get('/api/projects?year=2024&limit=50'); assert( projects.projects.some( (project: {name: string}) => project.name === 'Historical Telescope', ), - 'migrated project is browseable', + 'isolated D1 contains the migrated archive fixture', ); + const historical = await get('/api/projects/project-history'); + const media = await request(`/api/media/${historical.project.media[0].id}/content`); assert( - projects.projects.some((project: {name: string}) => project.name === 'Idea Compass'), - 'project-free idea remains browseable', + media.status === 200 && (await media.text()).includes('Synthetic'), + 'isolated attachment R2 contains reconciled fixture bytes', + ); + + await sendJson('PUT', '/api/admin/years/9999/screening-order', { + projectIds: ['readiness-project'], + }); + const bytes = await readFile(source); + const created = await sendJson( + 'POST', + '/api/projects/readiness-project/video/upload', + { + fileName: 'readiness-source.mp4', + fileSize: bytes.byteLength, + contentType: 'video/mp4', + }, + 201, ); - const project = await get('/api/projects/project-history'); - assert(project.project.members.length === 2, 'migrated team is preserved'); assert( - project.project.media[0]?.originalName === 'poster.txt', - 'migrated media is linked', + created.upload.status === 'uploading' && created.upload.completedParts.length === 0, + 'real local R2 multipart upload is created', ); - const media = await request(`/api/media/${project.project.media[0].id}/content`); - assert(media.status === 200, 'private R2 attachment downloads through the Worker'); - assert((await media.text()).includes('Synthetic'), 'seeded R2 bytes reconcile'); - - const voting = await get('/api/votes?year=2024'); - assert(voting.year.votingEnabled === true, 'seeded voting state is enabled'); - assert(voting.categories[0]?.name === 'Impact', 'seeded ballot is available'); - const memberAdmin = await request('/api/admin/years/2024'); - assert(memberAdmin.status === 403, 'member cannot use admin APIs'); - - sql( - "UPDATE users SET is_admin = 1, updated_at = CURRENT_TIMESTAMP WHERE google_subject = 'google-readiness-user' AND email = 'developer@sentry.io'", + const uploadId = created.upload.uploadId as string; + const interrupted = await get( + `/api/projects/readiness-project/video/upload/${uploadId}`, ); - const admin = await get('/api/session'); - assert(admin.user.role === 'admin', 'D1 promotion enables the admin role'); - const adminYear = await get('/api/admin/years/2024'); - assert(adminYear.awards[0]?.name === 'Impact winner', 'awards/admin data is available'); - const analytics = await get('/api/admin/analytics?year=2024'); assert( - analytics.years[0]?.voteCount === 1, - 'admin analytics reconcile the fixture vote', + interrupted.upload.completedParts.length === 0, + 'an interrupted upload resumes from durable server state', ); - await send('PUT', '/api/admin/years/2024/screening-order', { - projectIds: ['project-history'], + const partResponse = await request( + `/api/projects/readiness-project/video/upload/${uploadId}/parts/1`, + { + method: 'PUT', + headers: { + Origin: origin, + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(bytes.byteLength), + }, + body: bytes, + }, + ); + const partBody = await responseJson(partResponse); + assert(partResponse.status === 200, 'generated media bytes stream into multipart R2'); + const resumed = await get(`/api/projects/readiness-project/video/upload/${uploadId}`); + assert( + resumed.upload.completedParts[0]?.etag === partBody.part.etag, + 'uploaded part ETag survives a resume lookup', + ); + + const completionPath = `/api/projects/readiness-project/video/upload/${uploadId}/complete`; + const completionInput = { + parts: [{partNumber: 1, etag: partBody.part.etag}], + }; + const completed = await sendJson('POST', completionPath, completionInput); + assert(completed.video.status === 'queued', 'multipart completion queues processing'); + const videoId = completed.video.id as string; + const duplicate = await sendJson('POST', completionPath, completionInput); + assert( + duplicate.video.id === videoId && duplicate.video.processingAttempt === 1, + 'duplicate completion reuses the fenced Workflow attempt', + ); + + const ready = await waitForReady(); + assert(ready.status === 'ready', 'local Workflow conditionally publishes ready state'); + assert( + Math.abs(ready.loudnessLufs + 16) <= 0.7, + 'ready metadata records normalized loudness within ±0.7 LU', + ); + + const workflowEvidence = output('npx', [ + 'wrangler', + 'workflows', + 'instances', + 'describe', + 'hackweek-video-processing-readiness', + `video-${videoId}-attempt-1`, + '--local', + '--port', + String(port), + '--config', + config, + ]); + assert( + workflowEvidence.includes('run pinned ffmpeg processor') && + workflowEvidence.toLowerCase().includes('complete'), + 'local Workflow records a completed pinned FFmpeg Container step', + ); + + const descriptor = await get(`/api/videos/${videoId}/playback`); + assert( + descriptor.source.kind === 'mp4' && + descriptor.source.url === `/api/videos/${videoId}/content`, + 'playback returns a storage-neutral authenticated MP4 descriptor', + ); + const unauthorizedContent = await fetch(`${origin}/api/videos/${videoId}/content`); + assert( + unauthorizedContent.status === 401, + 'private derivative rejects anonymous reads', + ); + const full = await request(`/api/videos/${videoId}/content`); + const fullBytes = Buffer.from(await full.arrayBuffer()); + assert( + full.status === 200 && + full.headers.get('accept-ranges') === 'bytes' && + fullBytes.byteLength > 0, + 'authenticated full playback returns real derivative bytes', + ); + const rangeEnd = Math.min(1023, fullBytes.byteLength - 1); + const partial = await request(`/api/videos/${videoId}/content`, { + headers: {Range: `bytes=0-${rangeEnd}`}, }); - const fakeUpload = await send('POST', '/api/projects/project-history/video/upload', { - fileName: 'local-demo.mp4', - fileSize: 300_000_000, + const partialBytes = Buffer.from(await partial.arrayBuffer()); + assert( + partial.status === 206 && + partial.headers.get('content-range') === + `bytes 0-${rangeEnd}/${fullBytes.byteLength}` && + partialBytes.equals(fullBytes.subarray(0, rangeEnd + 1)), + 'authenticated range playback returns the exact derivative slice', + ); + const unsatisfiable = await request(`/api/videos/${videoId}/content`, { + headers: {Range: `bytes=${fullBytes.byteLength}-`}, }); - assert(fakeUpload.upload.protocol === 'tus', 'fake Stream exposes the tus contract'); assert( - fakeUpload.upload.url.startsWith('https://upload.videodelivery.net/fake/'), - 'fake upload stays visibly non-real', + unsatisfiable.status === 416 && + unsatisfiable.headers.get('content-range') === `bytes */${fullBytes.byteLength}`, + 'unsatisfiable playback range returns deterministic 416 metadata', + ); + + const playlist = await get('/api/videos/playlist?year=9999'); + assert( + playlist.videos.length === 1 && + playlist.videos[0].videoId === videoId && + playlist.videos[0].projectName === 'Readiness Video' && + playlist.videos[0].teamMembers.includes('Local Developer'), + 'ready derivative appears in curated reel order with team overlay data', + ); + + await sendJson('DELETE', '/api/projects/readiness-project/video', {confirmed: true}); + const afterRetirement = await get('/api/videos/playlist?year=9999'); + assert(afterRetirement.videos.length === 0, 'retired video leaves the curated reel'); + const retiredPlayback = await request(`/api/videos/${videoId}/content`); + assert(retiredPlayback.status === 409, 'retired derivative is no longer playable'); + + await stopServer(); + const row = query<{ + original_r2_key: string; + processed_r2_key: string; + status: string; + }>( + `SELECT original_r2_key, processed_r2_key, status FROM project_videos WHERE id = '${escapeSql(videoId)}'`, + ); + assert( + row.status === 'retired' && row.original_r2_key !== row.processed_r2_key, + 'D1 retains distinct immutable original and derivative keys after retirement', + ); + getR2Object(row.original_r2_key, original); + getR2Object(row.processed_r2_key, derivative); + assert( + createHash('sha256') + .update(await readFile(original)) + .digest('hex') === createHash('sha256').update(bytes).digest('hex'), + 'retained R2 original matches the generated upload bytes', + ); + assert( + (await stat(derivative)).size === fullBytes.byteLength, + 'retained R2 derivative matches playback bytes', ); - const emptyPlaylist = await get('/api/videos/playlist?year=2024'); - assert(emptyPlaylist.videos.length === 0, 'unready uploads stay out of screening'); - sql( - `UPDATE project_videos SET status = 'ready', duration_seconds = 42, loudness_lufs = -18, gain_db = 2 WHERE id = '${escapeSql(fakeUpload.video.id)}'`, + const probe = JSON.parse( + output('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration:stream=codec_type,codec_name,width,height,pix_fmt', + '-of', + 'json', + derivative, + ]), + ) as Probe; + const video = probe.streams.find((stream) => stream.codec_type === 'video'); + const audio = probe.streams.find((stream) => stream.codec_type === 'audio'); + assert( + video?.codec_name === 'h264' && + video.pix_fmt === 'yuv420p' && + (video.width ?? 0) <= 1920 && + (video.height ?? 0) <= 1080, + 'ffprobe confirms H.264 yuv420p output at or below 1080p', ); - const playlist = await get('/api/videos/playlist?year=2024'); - assert(playlist.videos.length === 1, 'ready videos follow the saved screening order'); - const playback = await get(`/api/videos/${fakeUpload.video.id}/playback`); + assert(audio?.codec_name === 'aac', 'ffprobe confirms canonical AAC audio'); + assert(Number(probe.format.duration) <= 600, 'ffprobe confirms bounded duration'); + assert(await hasFastStart(derivative), 'canonical MP4 places moov before mdat'); + const measuredLoudness = measureLoudness(derivative); assert( - playback.mode === 'fake' && playback.manifestUrl === null, - 'local playback refuses to impersonate real Stream HLS', + Math.abs(measuredLoudness + 16) <= 0.7, + `ffmpeg measures canonical output at ${measuredLoudness} LUFS`, ); - console.log('Local cutover readiness: 22 checks passed'); + console.log('Local video readiness: 30 checks passed'); } finally { await stopServer(); - await rm(rootDevVars, {force: true}); - if (hadDevVars) await rename(savedDevVars, rootDevVars); + cleanupReadinessContainers(); + const devVarsAfter = await optionalFile(rootDevVars); + const devVarsPreserved = sameOptionalBytes(devVarsBefore, devVarsAfter); await rm(state, {recursive: true, force: true}); + if (!devVarsPreserved) { + console.error('Developer .dev.vars changed during isolated readiness'); + process.exitCode = 1; + } +} + +interface Probe { + streams: Array<{ + codec_type: string; + codec_name?: string; + width?: number; + height?: number; + pix_fmt?: string; + }>; + format: {duration: string}; } -function readinessEnv() { +function localConfig() { + return { + name: 'hackweek-video-readiness', + main: path.join(root, 'src/worker/index.ts'), + compatibility_date: '2026-08-03', + assets: { + directory: path.join(root, 'public'), + not_found_handling: 'single-page-application', + binding: 'ASSETS', + run_worker_first: true, + }, + d1_databases: [ + { + binding: 'DB', + database_name: 'hackweek-db', + database_id: 'local', + migrations_dir: path.join(root, 'migrations'), + }, + ], + r2_buckets: [ + {binding: 'ATTACHMENTS', bucket_name: 'hackweek-attachments-readiness'}, + {binding: 'VIDEOS', bucket_name: 'hackweek-videos-readiness'}, + ], + workflows: [ + { + binding: 'VIDEO_PROCESSING_WORKFLOW', + name: 'hackweek-video-processing-readiness', + class_name: 'VideoProcessingWorkflow', + }, + ], + containers: [ + { + name: 'hackweek-video-processor-readiness', + class_name: 'VideoProcessorContainer', + image: path.join(root, 'Dockerfile.video-processor'), + image_build_context: root, + max_instances: 1, + instance_type: 'standard-2', + }, + ], + durable_objects: { + bindings: [{name: 'VIDEO_PROCESSOR', class_name: 'VideoProcessorContainer'}], + }, + migrations: [ + {tag: 'video-processor-v1', new_sqlite_classes: ['VideoProcessorContainer']}, + ], + vars: { + APP_ORIGIN: origin, + GOOGLE_CLIENT_ID: googleClientId, + GOOGLE_CLIENT_SECRET: googleClientSecret, + GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, + ALLOWED_EMAIL_DOMAIN: 'sentry.io', + VIDEO_PROCESSOR_CONCURRENCY: '1', + VIDEO_PROCESSING_AUTOSTART: 'true', + }, + observability: {enabled: true}, + }; +} + +function localDevVars() { + return `APP_ORIGIN="${origin}"\nGOOGLE_CLIENT_ID="${googleClientId}"\nGOOGLE_CLIENT_SECRET="${googleClientSecret}"\nGOOGLE_REDIRECT_URI="${origin}/api/auth/callback"\nALLOWED_EMAIL_DOMAIN="sentry.io"\nVIDEO_PROCESSOR_CONCURRENCY="1"\nVIDEO_PROCESSING_AUTOSTART="true"\n`; +} + +function localEnvironment() { return { ...process.env, - CLOUDFLARE_VITE_DEV_VARS_PATH: '/dev/null', HACKWEEK_LOCAL_STATE_PATH: state, - HACKWEEK_WRANGLER_CONFIG: path.join(state, 'wrangler.readiness.json'), + HACKWEEK_WRANGLER_CONFIG: config, APP_ORIGIN: origin, GOOGLE_CLIENT_ID: googleClientId, GOOGLE_CLIENT_SECRET: googleClientSecret, GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, ALLOWED_EMAIL_DOMAIN: 'sentry.io', - STREAM_MODE: 'fake', - STREAM_ALLOWED_ORIGIN: 'localhost', - STREAM_DELIVERY_HOST: 'customer-fake.cloudflarestream.com', - STREAM_WEBHOOK_SECRET: 'local-readiness-webhook-secret', - VIDEO_SERVICE_TOKEN: 'local-readiness-video-service-token', + VIDEO_PROCESSOR_CONCURRENCY: '1', + VIDEO_PROCESSING_AUTOSTART: 'true', }; } -function run(command: string, args: string[]) { - // Detach stdin so wrangler never sees a TTY and auto-confirms its prompts. - execFileSync(command, args, { - cwd: root, - env: readinessEnv(), - stdio: ['ignore', 'inherit', 'inherit'], +async function waitForServer() { + for (let attempt = 0; attempt < 1_200; attempt += 1) { + if (server?.exitCode !== null) { + throw new Error(`Local server exited early:\n${serverLog.join('')}`); + } + try { + if ((await fetch(`${origin}/api/health`)).ok) return; + } catch { + // Worker and Container image are still starting. + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`Timed out waiting for local server:\n${serverLog.join('')}`); +} + +async function waitForReady() { + for (let attempt = 0; attempt < 600; attempt += 1) { + const response = await get('/api/projects/readiness-project/video'); + const video = response.video as { + status: string; + loudnessLufs: number; + errorMessage: string | null; + }; + if (video.status === 'ready') return video; + if (video.status === 'failed') { + throw new Error( + `Local video processing failed: ${video.errorMessage}\n${serverLog.join('')}`, + ); + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`Timed out waiting for Workflow:\n${serverLog.join('')}`); +} + +async function get(pathname: string) { + const response = await request(pathname); + return responseJson(response); +} + +async function sendJson( + method: 'POST' | 'PUT' | 'DELETE', + pathname: string, + body: unknown, + expectedStatus = method === 'DELETE' ? 204 : 200, +) { + const response = await request(pathname, { + method, + headers: {'Content-Type': 'application/json', Origin: origin}, + body: JSON.stringify(body), }); + if (response.status !== expectedStatus) { + throw new Error( + `${pathname} returned ${response.status}, expected ${expectedStatus}: ${await response.text()}\n${serverLog.join('')}`, + ); + } + return response.status === 204 ? null : ((await response.json()) as any); +} + +async function responseJson(response: Response) { + if (!response.ok) { + throw new Error( + `${new URL(response.url).pathname} returned ${response.status}: ${await response.text()}\n${serverLog.join('')}`, + ); + } + return response.json() as Promise; +} + +function request(pathname: string, init: RequestInit = {}) { + const headers = new Headers(init.headers); + headers.set('Cookie', `sentry-hackweek-session=${sessionToken}`); + return fetch(`${origin}${pathname}`, {...init, headers}); } function sql(command: string) { @@ -269,11 +553,91 @@ function sql(command: string) { '--local', '--persist-to', state, + '--config', + config, '--command', command, ]); } +function query(command: string) { + const parsed = JSON.parse( + output('npx', [ + 'wrangler', + 'd1', + 'execute', + 'hackweek-db', + '--local', + '--persist-to', + state, + '--config', + config, + '--command', + command, + '--json', + ]), + ) as Array<{results: T[]}>; + const row = parsed[0]?.results[0]; + if (!row) throw new Error(`D1 query returned no rows: ${command}`); + return row; +} + +function getR2Object(key: string, destination: string) { + run('npx', [ + 'wrangler', + 'r2', + 'object', + 'get', + `hackweek-videos-readiness/${key}`, + '--file', + destination, + '--local', + '--persist-to', + state, + '--config', + config, + ]); +} + +function ffmpeg(args: string[]) { + run('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-nostdin', '-y', ...args]); +} + +function measureLoudness(file: string) { + const result = spawnSync( + 'ffmpeg', + [ + '-hide_banner', + '-nostdin', + '-i', + file, + '-map', + '0:a:0', + '-af', + 'loudnorm=I=-16:LRA=11:TP=-1.5:print_format=json', + '-f', + 'null', + '-', + ], + {cwd: root, encoding: 'utf8'}, + ); + if (result.status !== 0) { + throw new Error(`Loudness probe failed:\n${result.stdout}\n${result.stderr}`); + } + const blocks = [...result.stderr.matchAll(/\{[\s\S]*?"input_i"[\s\S]*?\}/g)]; + const input = blocks.at(-1)?.[0]; + const loudness = input ? Number(JSON.parse(input).input_i) : Number.NaN; + if (!Number.isFinite(loudness)) throw new Error('Loudness probe returned no value'); + return loudness; +} + +async function hasFastStart(file: string) { + const bytes = await readFile(file); + const moov = bytes.indexOf(Buffer.from('moov')); + const mdat = bytes.indexOf(Buffer.from('mdat')); + return moov >= 0 && mdat >= 0 && moov < mdat; +} + async function stopServer() { if (!server?.pid || server.exitCode !== null) return; try { @@ -283,64 +647,36 @@ async function stopServer() { } const exited = await Promise.race([ new Promise((resolve) => server!.once('exit', () => resolve(true))), - new Promise((resolve) => setTimeout(() => resolve(false), 2_000)), + new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), ]); if (!exited) { try { process.kill(-server.pid, 'SIGKILL'); } catch { - // The process group exited between checks. - } - } -} - -async function waitForServer(log: string[]) { - for (let attempt = 0; attempt < 100; attempt += 1) { - if (server?.exitCode !== null) { - throw new Error(`Local server exited early:\n${log.join('')}`); - } - try { - const response = await fetch(`${origin}/api/health`); - if (response.ok) { - await new Promise((resolve) => setTimeout(resolve, 500)); - if (server?.exitCode === null) return; - } - } catch { - // The development server is still starting. + // The detached process group exited between checks. } - await new Promise((resolve) => setTimeout(resolve, 100)); } - throw new Error(`Timed out waiting for local server:\n${log.join('')}`); } -async function get(pathname: string) { - const response = await request(pathname); - if (!response.ok) { - throw new Error( - `${pathname} returned ${response.status}: ${await response.text()}\n${serverLog.join('')}`, - ); - } - return response.json() as Promise; -} - -async function send(method: 'POST' | 'PUT', pathname: string, body: unknown) { - const response = await request(pathname, { - method, - headers: {'Content-Type': 'application/json', Origin: origin}, - body: JSON.stringify(body), +function run(command: string, args: string[]) { + execFileSync(command, args, { + cwd: root, + env: localEnvironment(), + stdio: ['ignore', 'inherit', 'inherit'], }); - if (!response.ok) throw new Error(`${pathname} returned ${response.status}`); - return response.json() as Promise; } -function request(pathname: string, init: RequestInit = {}) { - const headers = new Headers(init.headers); - headers.set('Cookie', `sentry-hackweek-session=${sessionToken}`); - return fetch(`${origin}${pathname}`, {...init, headers}); +function output(command: string, args: string[]) { + return execFileSync(command, args, { + cwd: root, + env: localEnvironment(), + encoding: 'utf8', + }); } function assert(value: unknown, message: string): asserts value { - if (!value) throw new Error(`Readiness check failed: ${message}`); + if (!value) + throw new Error(`Readiness check failed: ${message}\n${serverLog.join('')}`); console.log(`✓ ${message}`); } @@ -348,9 +684,39 @@ function escapeSql(value: string) { return value.replaceAll("'", "''"); } -function exists(filename: string) { - return stat(filename).then( - () => true, - () => false, +function optionalFile(file: string) { + return readFile(file).then( + (contents) => contents, + (error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return null; + throw error; + }, ); } + +function sameOptionalBytes(left: Buffer | null, right: Buffer | null) { + return left === null ? right === null : right !== null && left.equals(right); +} + +function dockerContainerNames() { + const result = spawnSync('docker', ['ps', '--all', '--format', '{{.Names}}'], { + cwd: root, + encoding: 'utf8', + }); + if (result.status !== 0) return []; + return result.stdout + .split('\n') + .map((name) => name.trim()) + .filter(Boolean); +} + +function cleanupReadinessContainers() { + for (const name of dockerContainerNames()) { + if ( + !dockerContainersBefore.has(name) && + name.startsWith('workerd-hackweek-video-readiness-') + ) { + spawnSync('docker', ['rm', '--force', name], {cwd: root, stdio: 'ignore'}); + } + } +} diff --git a/scripts/local-video-workflow.ts b/scripts/local-video-workflow.ts deleted file mode 100644 index b8acf36..0000000 --- a/scripts/local-video-workflow.ts +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env node -import {execFileSync, spawn, type ChildProcess} from 'node:child_process'; -import {createHash} from 'node:crypto'; -import {mkdtemp, readFile, rename, rm, stat, writeFile} from 'node:fs/promises'; -import {tmpdir} from 'node:os'; -import path from 'node:path'; - -const root = process.cwd(); -const state = await mkdtemp(path.join(tmpdir(), 'hackweek-workflow-smoke-')); -const port = Number(process.env.VIDEO_WORKFLOW_PORT ?? 5201); -const origin = `http://127.0.0.1:${port}`; -const config = path.join(state, 'wrangler.video-workflow.json'); -const source = path.join(state, 'workflow-source.mp4'); -const derivative = path.join(state, 'workflow-derivative.mp4'); -const token = createHash('sha256').update('local-video-workflow').digest('base64url'); -const tokenHash = createHash('sha256').update(token).digest('hex'); -const rootDevVars = path.join(root, '.dev.vars'); -const savedDevVars = path.join(state, '.dev.vars.saved'); -const hadDevVars = await exists(rootDevVars); -let server: ChildProcess | undefined; -const logs: string[] = []; - -try { - await writeFile(config, JSON.stringify(localConfig()), {mode: 0o600}); - const devVars = `APP_ORIGIN="${origin}"\nGOOGLE_CLIENT_ID="local.apps.googleusercontent.com"\nGOOGLE_CLIENT_SECRET="local-secret"\nGOOGLE_REDIRECT_URI="${origin}/api/auth/callback"\nALLOWED_EMAIL_DOMAIN="sentry.io"\nSTREAM_MODE="fake"\nVIDEO_PROCESSOR_CONCURRENCY="1"\nVIDEO_PROCESSING_AUTOSTART="true"\n`; - await writeFile(path.join(state, '.dev.vars'), devVars, {mode: 0o600}); - if (hadDevVars) await rename(rootDevVars, savedDevVars); - await writeFile(rootDevVars, devVars, {mode: 0o600}); - - run('npx', [ - 'wrangler', - 'd1', - 'migrations', - 'apply', - 'hackweek-db', - '--local', - '--persist-to', - state, - '--config', - config, - ]); - const now = Math.floor(Date.now() / 1000); - sql(` - INSERT INTO users - (id, source_uid, google_subject, email, display_name, is_admin) - VALUES - ('workflow-user', 'workflow-user', 'workflow-google-user', - 'workflow@sentry.io', 'Workflow User', 1); - INSERT INTO user_sessions - (token_hash, user_id, expires_at, created_at, last_used_at) - VALUES ('${tokenHash}', 'workflow-user', ${now + 3600}, ${now}, ${now}); - INSERT INTO years (id) VALUES ('9999'); - INSERT INTO groups (id, source_id, year_id, name, creator_id) - VALUES ('workflow-group', 'workflow-group', '9999', 'Workflow Group', 'workflow-user'); - INSERT INTO projects - (id, source_id, year_id, creator_id, group_id, name, summary, kind) - VALUES - ('workflow-project', 'workflow-project', '9999', 'workflow-user', - 'workflow-group', 'Workflow Project', 'Local real Workflow smoke', 'project'); - `); - run('ffmpeg', [ - '-hide_banner', - '-loglevel', - 'error', - '-nostdin', - '-y', - '-f', - 'lavfi', - '-i', - 'testsrc2=size=640x360:rate=24', - '-f', - 'lavfi', - '-i', - 'sine=frequency=440:sample_rate=48000', - '-t', - '2', - '-af', - 'volume=0.05', - '-c:v', - 'libx264', - '-preset', - 'ultrafast', - '-c:a', - 'aac', - '-shortest', - source, - ]); - - server = spawn( - process.execPath, - [ - path.join(root, 'node_modules/vite-plus/bin/vp'), - 'dev', - '--host', - '127.0.0.1', - '--port', - String(port), - '--strictPort', - ], - { - cwd: root, - detached: true, - env: localEnvironment(), - stdio: ['ignore', 'pipe', 'pipe'], - }, - ); - server.stdout?.on('data', (chunk) => logs.push(String(chunk))); - server.stderr?.on('data', (chunk) => logs.push(String(chunk))); - await waitForServer(); - - const bytes = await readFile(source); - const created = await api('/api/projects/workflow-project/video/upload', { - method: 'POST', - body: JSON.stringify({ - fileName: 'workflow-source.mp4', - fileSize: bytes.byteLength, - contentType: 'video/mp4', - }), - }); - assert( - created.response.status === 201, - 'multipart upload is created through the Worker', - ); - const uploadId = created.body.upload.uploadId as string; - const partResponse = await fetch( - `${origin}/api/projects/workflow-project/video/upload/${uploadId}/parts/1`, - { - method: 'PUT', - headers: authenticatedHeaders({ - Origin: origin, - 'Content-Type': 'application/octet-stream', - 'Content-Length': String(bytes.byteLength), - }), - body: bytes, - }, - ); - const part = (await partResponse.json()) as {part: {partNumber: number; etag: string}}; - assert(partResponse.status === 200, 'real source bytes stream into local R2'); - const completed = await api( - `/api/projects/workflow-project/video/upload/${uploadId}/complete`, - { - method: 'POST', - body: JSON.stringify({parts: [part.part]}), - }, - ); - assert(completed.response.status === 200, 'multipart completion starts processing'); - const videoId = completed.body.video.id as string; - assert( - completed.body.video.status === 'queued', - 'completed upload is initially queued', - ); - const duplicateCompletion = await api( - `/api/projects/workflow-project/video/upload/${uploadId}/complete`, - { - method: 'POST', - body: JSON.stringify({parts: [part.part]}), - }, - ); - assert( - duplicateCompletion.body.video.id === videoId, - 'duplicate completion reuses the deterministic Workflow attempt', - ); - - const ready = await waitForReady(); - assert(ready.status === 'ready', 'real local Workflow conditionally marks video ready'); - assert( - Math.abs(ready.loudnessLufs + 16) <= 0.7, - 'Workflow records normalized loudness', - ); - - await new Promise((resolve) => setTimeout(resolve, 500)); - const workflowEvidence = output('npx', [ - 'wrangler', - 'workflows', - 'instances', - 'describe', - 'hackweek-video-processing-local-smoke', - `video-${videoId}-attempt-1`, - '--local', - '--port', - String(port), - '--config', - config, - ]); - assert( - workflowEvidence.includes('run pinned ffmpeg processor'), - 'local Workflow records FFmpeg step', - ); - assert( - workflowEvidence.toLowerCase().includes('complete'), - 'local Workflow instance completes', - ); - - await stopServer(); - const row = query<{original_r2_key: string; processed_r2_key: string}>( - `SELECT original_r2_key, processed_r2_key FROM project_videos WHERE id = '${videoId}'`, - ); - assert( - row.original_r2_key !== row.processed_r2_key, - 'original and derivative R2 keys differ', - ); - run('npx', [ - 'wrangler', - 'r2', - 'object', - 'get', - `hackweek-videos-local-smoke/${row.processed_r2_key}`, - '--file', - derivative, - '--local', - '--persist-to', - state, - '--config', - config, - ]); - const probe = JSON.parse( - output('ffprobe', [ - '-v', - 'error', - '-show_entries', - 'format=duration:stream=codec_type,codec_name,width,height,pix_fmt', - '-of', - 'json', - derivative, - ]), - ) as { - streams: Array<{ - codec_type: string; - codec_name: string; - width?: number; - height?: number; - pix_fmt?: string; - }>; - format: {duration: string}; - }; - const video = probe.streams.find((stream) => stream.codec_type === 'video'); - const audio = probe.streams.find((stream) => stream.codec_type === 'audio'); - assert( - video?.codec_name === 'h264' && video.pix_fmt === 'yuv420p', - 'R2 derivative is H.264 yuv420p', - ); - assert(audio?.codec_name === 'aac', 'R2 derivative contains AAC audio'); - assert( - (video?.width ?? 0) <= 1920 && (video?.height ?? 0) <= 1080, - 'R2 derivative is <=1080p', - ); - assert( - Number(probe.format.duration) <= 600, - 'R2 derivative is within the duration limit', - ); - - console.log('Local Workflow + Container: 14 checks passed'); -} finally { - await stopServer(); - await rm(rootDevVars, {force: true}); - if (hadDevVars) await rename(savedDevVars, rootDevVars); - await rm(state, {recursive: true, force: true}); -} - -function localConfig() { - return { - name: 'hackweek-video-workflow-smoke', - main: path.join(root, 'src/worker/index.ts'), - compatibility_date: '2026-08-03', - assets: { - directory: path.join(root, 'public'), - not_found_handling: 'single-page-application', - binding: 'ASSETS', - run_worker_first: true, - }, - d1_databases: [ - { - binding: 'DB', - database_name: 'hackweek-db', - database_id: 'local', - migrations_dir: path.join(root, 'migrations'), - }, - ], - r2_buckets: [ - {binding: 'ATTACHMENTS', bucket_name: 'hackweek-attachments-local-smoke'}, - {binding: 'VIDEOS', bucket_name: 'hackweek-videos-local-smoke'}, - ], - workflows: [ - { - binding: 'VIDEO_PROCESSING_WORKFLOW', - name: 'hackweek-video-processing-local-smoke', - class_name: 'VideoProcessingWorkflow', - }, - ], - containers: [ - { - name: 'hackweek-video-processor-local-smoke', - class_name: 'VideoProcessorContainer', - image: path.join(root, 'Dockerfile.video-processor'), - image_build_context: root, - max_instances: 1, - instance_type: 'standard-2', - }, - ], - durable_objects: { - bindings: [{name: 'VIDEO_PROCESSOR', class_name: 'VideoProcessorContainer'}], - }, - migrations: [ - {tag: 'video-processor-v1', new_sqlite_classes: ['VideoProcessorContainer']}, - ], - vars: { - APP_ORIGIN: origin, - GOOGLE_CLIENT_ID: 'local.apps.googleusercontent.com', - GOOGLE_CLIENT_SECRET: 'local-secret', - GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, - ALLOWED_EMAIL_DOMAIN: 'sentry.io', - STREAM_MODE: 'fake', - VIDEO_PROCESSOR_CONCURRENCY: '1', - VIDEO_PROCESSING_AUTOSTART: 'true', - }, - }; -} - -function localEnvironment() { - return { - ...process.env, - CLOUDFLARE_VITE_DEV_VARS_PATH: '/dev/null', - HACKWEEK_LOCAL_STATE_PATH: state, - HACKWEEK_WRANGLER_CONFIG: config, - APP_ORIGIN: origin, - GOOGLE_CLIENT_ID: 'local.apps.googleusercontent.com', - GOOGLE_CLIENT_SECRET: 'local-secret', - GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, - ALLOWED_EMAIL_DOMAIN: 'sentry.io', - STREAM_MODE: 'fake', - VIDEO_PROCESSOR_CONCURRENCY: '1', - VIDEO_PROCESSING_AUTOSTART: 'true', - }; -} - -async function waitForServer() { - for (let index = 0; index < 600; index += 1) { - if (server?.exitCode !== null) - throw new Error(`Local server exited:\n${logs.join('')}`); - try { - if ((await fetch(`${origin}/api/health`)).ok) return; - } catch { - // Worker and Container image are still starting. - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - throw new Error(`Timed out waiting for local Worker:\n${logs.join('')}`); -} - -async function waitForReady() { - for (let index = 0; index < 300; index += 1) { - const result = await api('/api/projects/workflow-project/video'); - const video = result.body.video as { - status: string; - loudnessLufs: number; - errorMessage: string | null; - }; - if (video.status === 'ready') return video; - if (video.status === 'failed') { - throw new Error( - `Local video processing failed: ${video.errorMessage}\n${logs.join('')}`, - ); - } - await new Promise((resolve) => setTimeout(resolve, 200)); - } - throw new Error(`Timed out waiting for Workflow:\n${logs.join('')}`); -} - -async function api(pathname: string, init: RequestInit = {}) { - const headers = authenticatedHeaders(init.headers); - if (init.body) headers.set('Content-Type', 'application/json'); - if (init.method && init.method !== 'GET') headers.set('Origin', origin); - const response = await fetch(`${origin}${pathname}`, {...init, headers}); - const body = (await response.json()) as any; - if (!response.ok) { - throw new Error( - `${pathname} returned ${response.status}: ${JSON.stringify(body)}\n${logs.join('')}`, - ); - } - return {response, body}; -} - -function authenticatedHeaders(init?: ConstructorParameters[0]) { - const headers = new Headers(init); - headers.set('Cookie', `sentry-hackweek-session=${token}`); - return headers; -} - -function sql(command: string) { - run('npx', [ - 'wrangler', - 'd1', - 'execute', - 'hackweek-db', - '--local', - '--persist-to', - state, - '--config', - config, - '--command', - command, - ]); -} - -function query(command: string) { - const json = output('npx', [ - 'wrangler', - 'd1', - 'execute', - 'hackweek-db', - '--local', - '--persist-to', - state, - '--config', - config, - '--command', - command, - '--json', - ]); - const parsed = JSON.parse(json) as Array<{results: T[]}>; - const row = parsed[0]?.results[0]; - if (!row) throw new Error(`D1 query returned no rows: ${command}`); - return row; -} - -function run(command: string, args: string[]) { - execFileSync(command, args, {cwd: root, env: localEnvironment(), stdio: 'inherit'}); -} - -function output(command: string, args: string[]) { - return execFileSync(command, args, { - cwd: root, - env: localEnvironment(), - encoding: 'utf8', - }); -} - -async function stopServer() { - if (!server?.pid || server.exitCode !== null) return; - try { - process.kill(-server.pid, 'SIGTERM'); - } catch { - return; - } - const exited = await Promise.race([ - new Promise((resolve) => server!.once('exit', () => resolve(true))), - new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), - ]); - if (!exited) { - try { - process.kill(-server.pid, 'SIGKILL'); - } catch { - // The detached process group exited between checks. - } - } -} - -function assert(value: unknown, message: string): asserts value { - if (!value) - throw new Error(`Local Workflow check failed: ${message}\n${logs.join('')}`); - console.log(`✓ ${message}`); -} - -function exists(file: string) { - return stat(file).then( - () => true, - () => false, - ); -} diff --git a/src/shared/videos.ts b/src/shared/videos.ts index 2917c22..91710c2 100644 --- a/src/shared/videos.ts +++ b/src/shared/videos.ts @@ -1,8 +1,5 @@ -export type StreamMode = 'disabled' | 'fake' | 'real'; - export type VideoStatus = 'queued' | 'processing' | 'ready' | 'failed'; export type VideoFailureStage = 'processing'; -export type ArchiveStatus = 'pending' | 'archiving' | 'archived' | 'failed'; export type VideoUploadStatus = | 'creating' | 'uploading' diff --git a/src/worker/index.ts b/src/worker/index.ts index 3920f86..9d8fc88 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -24,13 +24,6 @@ export {VideoProcessingWorkflow} from './workflows/video-processing'; export interface VideoBindings { VIDEOS: R2Bucket; - STREAM_MODE?: string; - STREAM_ACCOUNT_ID?: string; - STREAM_API_TOKEN?: string; - STREAM_WEBHOOK_SECRET?: string; - STREAM_ALLOWED_ORIGIN?: string; - STREAM_DELIVERY_HOST?: string; - VIDEO_SERVICE_TOKEN?: string; VIDEO_PROCESSING_WORKFLOW: Workflow; VIDEO_PROCESSOR: DurableObjectNamespace; VIDEO_PROCESSOR_CONCURRENCY: string; diff --git a/src/worker/integrations/stream/fake.ts b/src/worker/integrations/stream/fake.ts deleted file mode 100644 index 787e57d..0000000 --- a/src/worker/integrations/stream/fake.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { - DirectUpload, - DirectUploadInput, - DownloadAsset, - HistoricalPromotionInput, - StreamGateway, -} from './types'; -import {StreamGatewayError} from './types'; - -export interface FakeStreamRecord { - uid: string; - source: 'direct' | 'historical'; - deleted: boolean; - downloadStatus: DownloadAsset['status']; -} - -export class FakeStreamGateway implements StreamGateway { - readonly records = new Map(); - - async createDirectUpload(input: DirectUploadInput): Promise { - const uid = fakeUid(); - this.records.set(uid, { - uid, - source: 'direct', - deleted: false, - downloadStatus: 'ready', - }); - return { - uid, - uploadUrl: `https://upload.videodelivery.net/fake/${uid}`, - expiresAt: input.expiresAt, - protocol: 'tus', - }; - } - - async promoteHistoricalVideo(_input: HistoricalPromotionInput): Promise { - const uid = fakeUid(); - this.records.set(uid, { - uid, - source: 'historical', - deleted: false, - downloadStatus: 'ready', - }); - return uid; - } - - async createPlaybackToken(uid: string, expiresAt: Date): Promise { - this.assertVideo(uid); - return fakeToken('playback', uid, expiresAt); - } - - async createDownloadToken(uid: string, expiresAt: Date): Promise { - this.assertVideo(uid); - return fakeToken('download', uid, expiresAt); - } - - async ensureDownload(uid: string): Promise { - const record = this.assertVideo(uid); - return { - status: record.downloadStatus, - url: - record.downloadStatus === 'error' - ? null - : `https://customer-fake.cloudflarestream.com/${uid}/downloads/default.mp4`, - }; - } - - async deleteVideo(uid: string): Promise { - this.assertVideo(uid).deleted = true; - } - - private assertVideo(uid: string) { - const record = this.records.get(uid); - if (!record || record.deleted) - throw new StreamGatewayError('Fake Stream video not found'); - return record; - } -} - -function fakeUid() { - return `fake-${crypto.randomUUID()}`; -} - -function fakeToken(kind: string, uid: string, expiresAt: Date) { - return `fake.${kind}.${uid}.${Math.floor(expiresAt.getTime() / 1000)}`; -} diff --git a/src/worker/integrations/stream/index.ts b/src/worker/integrations/stream/index.ts deleted file mode 100644 index 5160d67..0000000 --- a/src/worker/integrations/stream/index.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type {WorkerEnv} from '../../index'; -import {FakeStreamGateway} from './fake'; -import {RealStreamGateway} from './real'; -import type {StreamMode} from '../../../shared/videos'; -import {ServiceError} from '../../services/errors'; -import type {StreamGateway} from './types'; - -const fakeGateway = new FakeStreamGateway(); - -export function streamMode(env: WorkerEnv['Bindings']): StreamMode { - const mode = env.STREAM_MODE; - if (mode === 'disabled' || mode === 'fake' || mode === 'real') return mode; - throw new ServiceError( - 'AUTH_CONFIG_INVALID', - 'STREAM_MODE must be explicitly configured as disabled, fake, or real', - 500, - ); -} - -export function streamGateway(env: WorkerEnv['Bindings']): StreamGateway { - const mode = streamMode(env); - if (mode === 'disabled') { - throw new ServiceError( - 'SERVICE_UNAVAILABLE', - 'Video processing is temporarily unavailable', - 503, - ); - } - if (mode === 'fake') return fakeGateway; - if (!env.STREAM_ACCOUNT_ID?.trim() || !env.STREAM_API_TOKEN?.trim()) { - throw new ServiceError( - 'AUTH_CONFIG_INVALID', - 'Real Stream integration is not configured', - 500, - ); - } - return new RealStreamGateway(env.STREAM_ACCOUNT_ID, env.STREAM_API_TOKEN); -} - -export type {StreamGateway} from './types'; diff --git a/src/worker/integrations/stream/real.ts b/src/worker/integrations/stream/real.ts deleted file mode 100644 index 395aa13..0000000 --- a/src/worker/integrations/stream/real.ts +++ /dev/null @@ -1,162 +0,0 @@ -import type { - DirectUpload, - DirectUploadInput, - DownloadAsset, - HistoricalPromotionInput, - StreamGateway, -} from './types'; -import {StreamGatewayError} from './types'; - -interface CloudflareEnvelope { - success?: boolean; - result?: T; - errors?: Array<{message?: string}>; -} - -interface TokenResult { - token?: string; -} - -interface DownloadsResult { - default?: {status?: string; url?: string}; -} - -export class RealStreamGateway implements StreamGateway { - constructor( - private readonly accountId: string, - private readonly apiToken: string, - ) {} - - async createDirectUpload(input: DirectUploadInput): Promise { - const response = await fetch(`${this.baseUrl}?direct_user=true`, { - method: 'POST', - headers: { - Authorization: `Bearer ${this.apiToken}`, - 'Tus-Resumable': '1.0.0', - 'Upload-Length': String(input.fileSize), - 'Upload-Creator': input.creator, - 'Upload-Metadata': uploadMetadata({ - name: input.fileName, - maxdurationseconds: String(input.maxDurationSeconds), - requiresignedurls: null, - allowedorigins: input.allowedOrigin, - expiry: input.expiresAt.toISOString(), - }), - }, - }); - const uploadUrl = response.headers.get('Location'); - const uid = response.headers.get('stream-media-id'); - if (response.status !== 201 || !uploadUrl || !uid) { - throw await responseError( - response, - 'Cloudflare Stream did not create a tus upload', - ); - } - return {uid, uploadUrl, expiresAt: input.expiresAt, protocol: 'tus'}; - } - - async promoteHistoricalVideo(input: HistoricalPromotionInput): Promise { - const result = await this.request<{uid?: string}>('/copy', { - method: 'POST', - body: JSON.stringify({ - url: input.sourceUrl, - creator: input.creator, - allowedOrigins: [input.allowedOrigin], - requireSignedURLs: true, - meta: {name: input.fileName}, - }), - }); - if (!result.uid) throw new StreamGatewayError('Stream copy response omitted uid'); - return result.uid; - } - - async createPlaybackToken(uid: string, expiresAt: Date): Promise { - return this.createToken(uid, expiresAt, false); - } - - async createDownloadToken(uid: string, expiresAt: Date): Promise { - return this.createToken(uid, expiresAt, true); - } - - async ensureDownload(uid: string): Promise { - const result = await this.request( - `/${encodeURIComponent(uid)}/downloads`, - {method: 'POST'}, - ); - const download = result.default; - if (!download || !['inprogress', 'ready', 'error'].includes(download.status ?? '')) { - throw new StreamGatewayError('Stream download response was invalid'); - } - return { - status: download.status as DownloadAsset['status'], - url: download.url ?? null, - }; - } - - async deleteVideo(uid: string): Promise { - await this.request(`/${encodeURIComponent(uid)}`, {method: 'DELETE'}); - } - - private async createToken(uid: string, expiresAt: Date, downloadable: boolean) { - const result = await this.request(`/${encodeURIComponent(uid)}/token`, { - method: 'POST', - body: JSON.stringify({ - exp: Math.floor(expiresAt.getTime() / 1000), - downloadable, - }), - }); - if (!result.token) - throw new StreamGatewayError('Stream token response omitted token'); - return result.token; - } - - private async request(path: string, init: RequestInit): Promise { - const response = await fetch(`${this.baseUrl}${path}`, { - ...init, - headers: new Headers({ - Authorization: `Bearer ${this.apiToken}`, - 'Content-Type': 'application/json', - ...Object.fromEntries(new Headers(init.headers).entries()), - }), - }); - const envelope = (await response.json().catch(() => ({}))) as CloudflareEnvelope; - if (!response.ok || envelope.success !== true || envelope.result === undefined) { - const message = - envelope.errors - ?.map(({message}) => message) - .filter(Boolean) - .join('; ') || - `Cloudflare Stream request failed with status ${response.status}`; - throw new StreamGatewayError(message); - } - return envelope.result; - } - - private get baseUrl() { - return `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(this.accountId)}/stream`; - } -} - -function uploadMetadata(values: Record) { - return Object.entries(values) - .map(([key, value]) => (value === null ? key : `${key} ${base64(value)}`)) - .join(','); -} - -function base64(value: string) { - const bytes = new TextEncoder().encode(value); - let binary = ''; - for (const byte of bytes) binary += String.fromCharCode(byte); - return btoa(binary); -} - -async function responseError(response: Response, fallback: string) { - const envelope = (await response - .json() - .catch(() => ({}))) as CloudflareEnvelope; - const detail = envelope.errors - ?.map(({message}) => message) - .filter(Boolean) - .join('; '); - return new StreamGatewayError(detail || `${fallback} (${response.status})`); -} diff --git a/src/worker/integrations/stream/types.ts b/src/worker/integrations/stream/types.ts deleted file mode 100644 index a92324d..0000000 --- a/src/worker/integrations/stream/types.ts +++ /dev/null @@ -1,38 +0,0 @@ -export interface DirectUploadInput { - creator: string; - fileName: string; - fileSize: number; - maxDurationSeconds: number; - allowedOrigin: string; - expiresAt: Date; -} - -export interface DirectUpload { - uid: string; - uploadUrl: string; - expiresAt: Date; - protocol: 'tus'; -} - -export interface HistoricalPromotionInput { - creator: string; - sourceUrl: string; - fileName: string; - allowedOrigin: string; -} - -export interface DownloadAsset { - status: 'inprogress' | 'ready' | 'error'; - url: string | null; -} - -export interface StreamGateway { - createDirectUpload(input: DirectUploadInput): Promise; - promoteHistoricalVideo(input: HistoricalPromotionInput): Promise; - createPlaybackToken(uid: string, expiresAt: Date): Promise; - createDownloadToken(uid: string, expiresAt: Date): Promise; - ensureDownload(uid: string): Promise; - deleteVideo(uid: string): Promise; -} - -export class StreamGatewayError extends Error {} diff --git a/src/worker/middleware/service-auth.ts b/src/worker/middleware/service-auth.ts deleted file mode 100644 index 1ce8e7c..0000000 --- a/src/worker/middleware/service-auth.ts +++ /dev/null @@ -1,28 +0,0 @@ -import {createMiddleware} from 'hono/factory'; - -import type {WorkerEnv} from '../index'; - -export const requireVideoService = createMiddleware(async (c, next) => { - const expected = c.env.VIDEO_SERVICE_TOKEN?.trim(); - const authorization = c.req.header('Authorization'); - const actual = authorization?.startsWith('Bearer ') ? authorization.slice(7) : ''; - if (!expected || !actual || !timingSafeEqual(expected, actual)) { - return c.json( - {error: {code: 'AUTH_REQUIRED', message: 'Video service token is required'}}, - 401, - ); - } - await next(); -}); - -function timingSafeEqual(left: string, right: string) { - const encoder = new TextEncoder(); - const leftBytes = encoder.encode(left); - const rightBytes = encoder.encode(right); - let difference = leftBytes.length ^ rightBytes.length; - const length = Math.max(leftBytes.length, rightBytes.length); - for (let index = 0; index < length; index += 1) { - difference |= (leftBytes[index] ?? 0) ^ (rightBytes[index] ?? 0); - } - return difference === 0; -} diff --git a/src/worker/workflows/video-processing.ts b/src/worker/workflows/video-processing.ts index b7cd392..f96ef6f 100644 --- a/src/worker/workflows/video-processing.ts +++ b/src/worker/workflows/video-processing.ts @@ -49,12 +49,18 @@ export class VideoProcessingWorkflow extends WorkflowEntrypoint< ), ); } catch (error) { + const message = errorMessage(error); + logVideoProcessing('error', 'claim_failed', {videoId, attempt, message}); await step.do('record claim failure', () => - failVideoProcessingAttempt(this.env.DB, videoId, attempt, errorMessage(error)), + failVideoProcessingAttempt(this.env.DB, videoId, attempt, message), ); return {status: 'failed', stage: 'claim'}; } - if (claim.status === 'stale') return {status: 'stale'}; + if (claim.status === 'stale') { + logVideoProcessing('info', 'stale_before_processing', {videoId, attempt}); + return {status: 'stale'}; + } + logVideoProcessing('info', 'processing_started', {videoId, attempt}); let result: VideoProcessorResult; try { @@ -83,8 +89,10 @@ export class VideoProcessingWorkflow extends WorkflowEntrypoint< }, ); } catch (error) { + const message = errorMessage(error); + logVideoProcessing('error', 'processor_failed', {videoId, attempt, message}); await step.do('record processor failure', () => - failVideoProcessingAttempt(this.env.DB, videoId, attempt, errorMessage(error)), + failVideoProcessingAttempt(this.env.DB, videoId, attempt, message), ); return {status: 'failed', stage: 'processor'}; } @@ -98,6 +106,15 @@ export class VideoProcessingWorkflow extends WorkflowEntrypoint< result, ), ); + logVideoProcessing( + 'info', + published ? 'processing_ready' : 'stale_after_processing', + { + videoId, + attempt, + durationSeconds: result.durationSeconds, + }, + ); return {status: published ? 'ready' : 'stale', result}; } } @@ -161,5 +178,13 @@ function invalidProcessorResult() { } function errorMessage(error: unknown) { - return error instanceof Error ? error.message : String(error); + return (error instanceof Error ? error.message : String(error)).slice(0, 500); +} + +function logVideoProcessing( + level: 'info' | 'error', + event: string, + fields: Record, +) { + console[level](JSON.stringify({component: 'video-processing', event, ...fields})); } diff --git a/test/e2e/video-rollout.test.ts b/test/e2e/video-rollout.test.ts new file mode 100644 index 0000000..41fa84f --- /dev/null +++ b/test/e2e/video-rollout.test.ts @@ -0,0 +1,75 @@ +import {readFile} from 'node:fs/promises'; + +import {describe, expect, it} from 'vitest'; + +interface WranglerConfig { + r2_buckets: Array<{binding: string; bucket_name: string}>; + workflows: Array<{binding: string; name: string; class_name: string}>; + containers: Array<{ + name: string; + class_name: string; + image: string; + max_instances: number; + }>; + vars: Record; + observability: {enabled: boolean}; +} + +describe('video rollout preparation', () => { + it('declares isolated production video resources with concurrency capped at two', async () => { + const config = JSON.parse( + await readFile('wrangler.production.json', 'utf8'), + ) as WranglerConfig; + + expect(config.r2_buckets.find(({binding}) => binding === 'VIDEOS')).toEqual({ + binding: 'VIDEOS', + bucket_name: 'hackweek-video-media-production', + }); + expect( + config.workflows.find(({binding}) => binding === 'VIDEO_PROCESSING_WORKFLOW'), + ).toMatchObject({ + name: 'hackweek-video-processing-production', + class_name: 'VideoProcessingWorkflow', + }); + expect( + config.containers.find(({class_name}) => class_name === 'VideoProcessorContainer'), + ).toMatchObject({ + name: 'hackweek-video-processor-production', + image: './Dockerfile.video-processor', + max_instances: 2, + }); + expect(config.vars).toMatchObject({ + VIDEO_PROCESSOR_CONCURRENCY: '2', + VIDEO_PROCESSING_AUTOSTART: 'true', + }); + expect(config.vars).not.toHaveProperty('STREAM_MODE'); + expect(config.observability.enabled).toBe(true); + }); + + it('pins both processor image stages by digest', async () => { + const dockerfile = await readFile('Dockerfile.video-processor', 'utf8'); + const stages = dockerfile.match(/^FROM .+@sha256:[a-f0-9]{64}.*$/gm) ?? []; + expect(stages).toHaveLength(2); + expect(dockerfile).toContain('mwader/static-ffmpeg:8.0.1@sha256:'); + expect(dockerfile).toContain('node:24.11.0-bookworm-slim@sha256:'); + }); + + it('keeps readiness on real bytes and lifecycle APIs rather than fake readiness', async () => { + const readiness = await readFile('scripts/local-readiness.ts', 'utf8'); + expect(readiness).toContain("'wrangler',\n 'workflows'"); + expect(readiness).toContain('/parts/1'); + expect(readiness).toContain('headers: {Range:'); + expect(readiness).toContain('/api/videos/playlist?year=9999'); + expect(readiness).toContain("output('ffprobe'"); + expect(readiness).not.toContain('STREAM_MODE'); + expect(readiness).not.toMatch(/UPDATE project_videos SET status\s*=\s*'ready'/); + }); + + it('documents retained storage and the explicit production approval boundary', async () => { + const runbook = await readFile('VIDEO_ROLLOUT.md', 'utf8'); + expect(runbook).toContain('Explicit approval boundary'); + expect(runbook).toContain('max_instances: 2'); + expect(runbook).toContain('No automatic deletion'); + expect(runbook).toContain('hackweek-video-media-production'); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 4af6ade..1f2cba4 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -24,11 +24,6 @@ export default defineConfig({ GOOGLE_REDIRECT_URI: 'https://hackweek.test/api/auth/callback', GOOGLE_TOKEN_ENDPOINT: 'https://tokens.hackweek.test/token', ALLOWED_EMAIL_DOMAIN: 'sentry.io', - STREAM_MODE: 'fake', - STREAM_ALLOWED_ORIGIN: 'hackweek.test', - STREAM_DELIVERY_HOST: 'customer-fake.cloudflarestream.com', - STREAM_WEBHOOK_SECRET: 'test-webhook-secret', - VIDEO_SERVICE_TOKEN: 'test-video-service-token', VIDEO_PROCESSING_AUTOSTART: 'false', GOOGLE_JWKS_JSON: '{"keys":[{"kty":"RSA","n":"3SSum9jtxKTheDwctdDnp80Mv5_hAQzcKJJcxpw3wShOU0LyEpt23riO3ncaOC4iVm5xseM9PJmFjYMQJcplKi6I3nDC7tToFWrFqrn7LSjdvJS3WqUjn20CUiUxYZ3QLZcYyERU6M39M8nE1zFHQ3tHz7YkjoNQTPMUXMRydeL8yuBizdsrGQosgpGJceTAFIHJkKtdCipbSBZA3qrrE-HDJa9nZSYloywLVsaxzKJG2SiJzvVBydZbCQ2ZQeR44qdpCIibU2IMyVelKqiCqHwoBwzYybGx4Tcx4N_1UrNZQnECbcN7jSzxRp1agrK6p2w-svyYYXmt7ymqa3kjdQ","e":"AQAB","kid":"google-test","alg":"RS256","use":"sig"}]}', diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 1f76b38..ae81371 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --config=wrangler.production.json` (hash: 1e7f342b27ec67431d48ecd80cae37c9) +// Generated by Wrangler by running `wrangler types --config=wrangler.production.json` (hash: bc092216b5e73ef63da8313b6c4a1921) // Runtime types generated with workerd@1.20260730.1 2026-08-03 interface __BaseEnv_Env { ATTACHMENTS: R2Bucket; @@ -12,7 +12,6 @@ interface __BaseEnv_Env { GOOGLE_REDIRECT_URI: "https://hackweek.sentry.new/api/auth/callback"; GOOGLE_CLIENT_ID: "694837489680-25m2umkr51lofdads5uvocgtcdqcs6c4.apps.googleusercontent.com"; ALLOWED_EMAIL_DOMAIN: "sentry.io"; - STREAM_MODE: "disabled"; VIDEO_PROCESSOR: DurableObjectNamespace; VIDEO_PROCESSING_WORKFLOW: Workflow[0]['payload']>; } diff --git a/wrangler.production.json b/wrangler.production.json index 12f78be..ae3c8cc 100644 --- a/wrangler.production.json +++ b/wrangler.production.json @@ -25,19 +25,19 @@ }, { "binding": "VIDEOS", - "bucket_name": "hackweek-videos" + "bucket_name": "hackweek-video-media-production" } ], "workflows": [ { "binding": "VIDEO_PROCESSING_WORKFLOW", - "name": "hackweek-video-processing", + "name": "hackweek-video-processing-production", "class_name": "VideoProcessingWorkflow" } ], "containers": [ { - "name": "hackweek-video-processor", + "name": "hackweek-video-processor-production", "class_name": "VideoProcessorContainer", "image": "./Dockerfile.video-processor", "image_build_context": ".", @@ -65,8 +65,7 @@ "APP_ORIGIN": "https://hackweek.sentry.new", "GOOGLE_REDIRECT_URI": "https://hackweek.sentry.new/api/auth/callback", "GOOGLE_CLIENT_ID": "694837489680-25m2umkr51lofdads5uvocgtcdqcs6c4.apps.googleusercontent.com", - "ALLOWED_EMAIL_DOMAIN": "sentry.io", - "STREAM_MODE": "disabled" + "ALLOWED_EMAIL_DOMAIN": "sentry.io" }, "observability": {"enabled": true} } From e47071cb2208c5d02b46d510a66dcee9eca0e4df Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 16:11:54 +0200 Subject: [PATCH 05/18] fix(video): make R2 migration expand-compatible Keep the legacy project_videos and stream_events schema intact during the rollout and move the new R2 lifecycle onto video_submissions. This lets both the deployed pre-release Worker and the R2 Worker operate after migration 0007, preserving a valid Worker rollback target without restoring active Stream behavior in the new release.\n\nAdd an executable populated-schema migration proof for legacy reads and writes, R2 writes and active constraints, retained data, and foreign keys. Update rollout automation and documentation to identify the safe expand/deploy/rollback sequence and require a separately approved future contraction. --- .github/workflows/deploy.yml | 2 +- VIDEO_ROLLOUT.md | 32 +++-- migrations/0007_r2_video_lifecycle.sql | 40 ++---- scripts/local-readiness.ts | 2 +- src/worker/containers/video-processor.ts | 2 +- src/worker/db/schema.ts | 2 + src/worker/repositories/administration.ts | 2 +- src/worker/services/videos.ts | 26 ++-- test/e2e/video-rollout.test.ts | 20 ++- test/migration/migration.test.ts | 160 ++++++++++++++++++++-- test/video/video.test.ts | 12 +- 11 files changed, 227 insertions(+), 73 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 819ab2a..6551507 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -41,7 +41,7 @@ jobs: test "$CLOUDFLARE_ACCOUNT_ID" = '773afa1f62ff86c80db4f24f7ff1e9c8' || { echo 'Unexpected Cloudflare account'; exit 1; } node -e "const c=require('./wrangler.production.json'); const videos=c.r2_buckets.find(x=>x.binding==='VIDEOS'); const workflow=c.workflows.find(x=>x.binding==='VIDEO_PROCESSING_WORKFLOW'); const container=c.containers.find(x=>x.class_name==='VideoProcessorContainer'); if (c.account_id !== '773afa1f62ff86c80db4f24f7ff1e9c8' || videos?.bucket_name !== 'hackweek-video-media-production' || workflow?.name !== 'hackweek-video-processing-production' || container?.name !== 'hackweek-video-processor-production' || container?.max_instances !== 2 || c.vars.VIDEO_PROCESSOR_CONCURRENCY !== '2') process.exit(1); for (const value of [c.d1_databases[0].database_id,c.r2_buckets[0].bucket_name,c.vars.APP_ORIGIN,c.vars.GOOGLE_REDIRECT_URI,c.vars.GOOGLE_CLIENT_ID]) if (!value || /replace.me/i.test(value) || value === '00000000-0000-0000-0000-000000000000') process.exit(1)" - run: npm run build - - name: Apply reviewed D1 migrations + - name: Apply expand-compatible D1 migrations run: npx wrangler d1 migrations apply hackweek-db --remote --config wrangler.production.json --yes env: &cloudflare CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} diff --git a/VIDEO_ROLLOUT.md b/VIDEO_ROLLOUT.md index c7231f3..a2de46d 100644 --- a/VIDEO_ROLLOUT.md +++ b/VIDEO_ROLLOUT.md @@ -14,13 +14,15 @@ The production declaration in `wrangler.production.json` uses these isolated fut | Container application | `hackweek-video-processor-production` | Digest-pinned `Dockerfile.video-processor` image | | `DB` | Existing `hackweek-db` binding | Upload, attempt, fencing, state, and retained-object inventory | +Migration 0007 is an expand-only transition. The R2 Worker uses `video_submissions`, `video_uploads`, `video_upload_parts`, and `video_processing_attempts`; the legacy `project_videos` columns and `stream_events` table remain unchanged solely so the recorded pre-release Worker can run before deployment or after rollback. The R2 Worker has no active Stream path. + Production declares both `max_instances: 2` and `VIDEO_PROCESSOR_CONCURRENCY=2`. Keep the two values equal. Local development uses one. The Container has no Internet access and receives no R2 account credential; its outbound handler scopes source/output access to the current D1 attempt. Required non-secret variables are `APP_ORIGIN`, `GOOGLE_REDIRECT_URI`, `GOOGLE_CLIENT_ID`, `ALLOWED_EMAIL_DOMAIN`, `VIDEO_PROCESSOR_CONCURRENCY`, and `VIDEO_PROCESSING_AUTOSTART`. `GOOGLE_CLIENT_SECRET` remains the required Worker secret. There is no Stream, HLS, public-R2, service-token, Queue, or general R2 credential binding. ## Explicit approval boundary -A human production owner must approve the exact commit, account, names, expected storage growth, benchmark/tuning decision, D1 backup window, and smoke/rollback operators before any command below that uses `--remote`, `r2 bucket create`, `secret put`, `deploy`, or `rollback` is run. +A human production owner must approve the exact commit, account, names, expected storage growth, benchmark/tuning decision, D1 backup window, smoke/rollback operators, and recorded pre-release Worker version before any command below that uses `--remote`, `r2 bucket create`, `secret put`, `deploy`, or `rollback` is run. The release record must include the passing populated-legacy expand/old-query/new-query/FK migration test for that exact commit. Until that approval, only these non-mutating local checks are allowed: @@ -53,8 +55,8 @@ Before production approval, repeat the benchmark on the release commit and run a The following is an operator checklist, not deployment automation. Stop if the configured Cloudflare account is not `773afa1f62ff86c80db4f24f7ff1e9c8` or any proposed resource name is already owned for another purpose. -1. Record the release commit and current Worker version ID for rollback. Obtain an explicit go/no-go from the production owner. -2. Run all local checks from the approval section and archive their output with the release record. +1. Record the release commit and current pre-release Worker version ID for rollback. Confirm it is the version covered by the legacy `project_videos`/`stream_events` SQL contract in the migration test. Obtain an explicit go/no-go from the production owner. +2. Run all local checks from the approval section and archive their output with the release record, including `npm run test:migration -- test/migration/migration.test.ts test/e2e/video-rollout.test.ts`. 3. Create the isolated private video bucket: ```bash @@ -76,6 +78,8 @@ The following is an operator checklist, not deployment automation. Stop if the c --remote --config wrangler.production.json ``` + Migration 0007 only expands the schema. The still-deployed pre-release Worker continues to read and write its unchanged `project_videos` columns and `stream_events`; the new R2 tables can be queried independently. If migration application or the following deploy fails, leave the pre-release Worker serving and investigate—do not attempt a destructive schema reversal. + 6. Deploy the reviewed declaration: ```bash @@ -123,7 +127,7 @@ Create dashboard/alert ownership before rollout for: - queued depth above 2 for 10 minutes (capacity pressure at cap two); - Worker `/api/projects/*/video*` and `/api/videos/*/content` 5xx rate >1% over 5 minutes; - Container CPU, memory, scratch disk, restart, and timeout pressure; -- `project_videos.status='failed'` growth and retries per video; +- `video_submissions.status='failed'` growth and retries per video; - R2 object count/bytes and monthly growth for `hackweek-video-media-production`. Never place full request headers, session cookies, source/output object keys, or media payloads in an alert. Link alerts to this runbook and name an event-time operator. @@ -132,19 +136,27 @@ Never place full request headers, session cookies, source/output object keys, or Rollback is state-preserving. Do not delete R2 objects, Workflow instances, D1 rows, or the Container application during incident response. -1. Pause new video completion/processing by preparing `VIDEO_PROCESSING_AUTOSTART=false` on the reviewed incident commit and deploying it through the same approved path. New completed uploads remain queued rather than being published incorrectly. -2. If the Worker release itself is faulty, roll back to the recorded compatible Worker version: +1. Pause new R2 video completion/processing by preparing `VIDEO_PROCESSING_AUTOSTART=false` on a reviewed incident commit when the Worker is healthy enough to deploy that change. New completed R2 uploads remain queued rather than being published incorrectly. +2. If the Worker release itself is faulty, roll back to the recorded pre-release Worker version that was captured and compatibility-tested before rollout: ```bash - npx wrangler rollback \ + npx wrangler rollback \ --name hackweek --message "Rollback video rollout: " ``` -3. Leave queued/running/failed attempt rows and all original/derivative objects intact. Inspect current attempt fencing before any retry. A late result cannot publish over a retired or newer attempt. -4. Restore service only after `npm run verify`, the production smoke subset, and incident-owner approval pass on the corrective release. Re-enable autostart and keep concurrency at or below two. + This target remains schema-compatible after 0007 because the migration does not alter `project_videos` or `stream_events`. Do not select an older unrecorded version. The rollback version may use its original Stream lifecycle; no Stream behavior is present in the new R2 Worker. + +3. Do not reverse migration 0007. Leave legacy rows/events, R2 queued/running/failed attempt rows, and all original/derivative objects intact. A late R2 result cannot publish over a retired or newer attempt. +4. Restore the R2 release only after `npm run verify`, the production smoke subset, reconciliation of any legacy writes made during rollback, and incident-owner approval pass on the corrective release. Re-enable autostart and keep concurrency at or below two. 5. Reconcile status and inventory; do not manually mark a video ready and do not copy an unprobed object into a canonical key. -D1 migration 0007 is forward-only. Rollback does not reverse it or restore the old Stream lifecycle. +D1 migration 0007 is a forward-only expansion. Worker rollback does not reverse it, and no database downgrade is required. + +## Future contraction (separate approval required) + +Do not drop, rename, or repurpose `project_videos`, its legacy columns/indexes, or `stream_events` in this release. They define the tested pre-release rollback contract. + +A later contraction requires a separate production-owner approval and release after this rollback target is retired. Before contraction, inventory and reconcile legacy rows/events—including writes made during any rollback—record a new R2-compatible rollback target, prove no deployed Worker queries the legacy schema, and take the approved D1 backup. Only then may a new migration remove the legacy tables. Renaming `video_submissions` is not part of 0007; if desired, it requires its own expand/deploy/contract sequence rather than an in-place destructive rename. ## Retained-storage policy diff --git a/migrations/0007_r2_video_lifecycle.sql b/migrations/0007_r2_video_lifecycle.sql index 8590dd4..1ca0369 100644 --- a/migrations/0007_r2_video_lifecycle.sql +++ b/migrations/0007_r2_video_lifecycle.sql @@ -1,8 +1,8 @@ -PRAGMA foreign_keys = OFF; - -ALTER TABLE project_videos RENAME TO legacy_project_videos; +PRAGMA foreign_keys = ON; -CREATE TABLE project_videos ( +-- Expand only: keep the legacy project_videos and stream_events contract intact +-- until a separately approved contraction after the rollback window. +CREATE TABLE video_submissions ( id TEXT PRIMARY KEY NOT NULL, project_id TEXT NOT NULL REFERENCES projects(id) ON UPDATE CASCADE ON DELETE CASCADE, original_name TEXT NOT NULL CHECK (length(trim(original_name)) BETWEEN 1 AND 255), @@ -24,24 +24,10 @@ CREATE TABLE project_videos ( CHECK (status = 'retired' OR (original_r2_key IS NOT NULL AND size_bytes IS NOT NULL)) ) STRICT; -INSERT INTO project_videos ( - id, project_id, original_name, status, processing_attempt, - duration_seconds, loudness_lufs, gain_db, error_message, - retired_at, created_at, updated_at -) -SELECT - id, project_id, 'Legacy Stream video', 'retired', 1, - duration_seconds, loudness_lufs, gain_db, error_message, - updated_at, created_at, updated_at -FROM legacy_project_videos; - -DROP TABLE legacy_project_videos; -DROP TABLE stream_events; - -CREATE UNIQUE INDEX project_videos_active_project_idx - ON project_videos(project_id) WHERE retired_at IS NULL; -CREATE INDEX project_videos_status_idx - ON project_videos(status, updated_at); +CREATE UNIQUE INDEX video_submissions_active_project_idx + ON video_submissions(project_id) WHERE retired_at IS NULL; +CREATE INDEX video_submissions_status_idx + ON video_submissions(status, updated_at); CREATE TABLE video_uploads ( id TEXT PRIMARY KEY NOT NULL, @@ -74,15 +60,15 @@ CREATE TRIGGER video_uploads_reject_active_submission BEFORE INSERT ON video_uploads WHEN NEW.status IN ('creating', 'uploading', 'completing') AND EXISTS ( - SELECT 1 FROM project_videos + SELECT 1 FROM video_submissions WHERE project_id = NEW.project_id AND retired_at IS NULL ) BEGIN SELECT RAISE(ABORT, 'active project video exists'); END; -CREATE TRIGGER project_videos_reject_active_upload -BEFORE INSERT ON project_videos +CREATE TRIGGER video_submissions_reject_active_upload +BEFORE INSERT ON video_submissions WHEN NEW.retired_at IS NULL AND EXISTS ( SELECT 1 FROM video_uploads @@ -103,7 +89,7 @@ CREATE TABLE video_upload_parts ( ) STRICT, WITHOUT ROWID; CREATE TABLE video_processing_attempts ( - video_id TEXT NOT NULL REFERENCES project_videos(id) ON UPDATE CASCADE ON DELETE CASCADE, + video_id TEXT NOT NULL REFERENCES video_submissions(id) ON UPDATE CASCADE ON DELETE CASCADE, attempt INTEGER NOT NULL CHECK (attempt >= 1), status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')), @@ -118,5 +104,3 @@ CREATE TABLE video_processing_attempts ( CREATE INDEX video_processing_attempts_status_idx ON video_processing_attempts(status, created_at); - -PRAGMA foreign_keys = ON; diff --git a/scripts/local-readiness.ts b/scripts/local-readiness.ts index 835aa3f..0efda37 100644 --- a/scripts/local-readiness.ts +++ b/scripts/local-readiness.ts @@ -321,7 +321,7 @@ try { processed_r2_key: string; status: string; }>( - `SELECT original_r2_key, processed_r2_key, status FROM project_videos WHERE id = '${escapeSql(videoId)}'`, + `SELECT original_r2_key, processed_r2_key, status FROM video_submissions WHERE id = '${escapeSql(videoId)}'`, ); assert( row.status === 'retired' && row.original_r2_key !== row.processed_r2_key, diff --git a/src/worker/containers/video-processor.ts b/src/worker/containers/video-processor.ts index 0d7dfbb..f1704be 100644 --- a/src/worker/containers/video-processor.ts +++ b/src/worker/containers/video-processor.ts @@ -78,7 +78,7 @@ async function currentStorage(db: D1Database, scope: VideoProcessingParams) { return db .prepare( `SELECT pv.original_r2_key, vpa.output_r2_key - FROM project_videos pv + FROM video_submissions pv JOIN video_processing_attempts vpa ON vpa.video_id = pv.id AND vpa.attempt = ? WHERE pv.id = ? AND pv.processing_attempt = ? diff --git a/src/worker/db/schema.ts b/src/worker/db/schema.ts index 84ac216..9c657a1 100644 --- a/src/worker/db/schema.ts +++ b/src/worker/db/schema.ts @@ -12,6 +12,8 @@ export const tableNames = [ 'awards', 'media', 'project_videos', + 'stream_events', + 'video_submissions', 'video_uploads', 'video_upload_parts', 'video_processing_attempts', diff --git a/src/worker/repositories/administration.ts b/src/worker/repositories/administration.ts index 52ea4f1..95d0b6b 100644 --- a/src/worker/repositories/administration.ts +++ b/src/worker/repositories/administration.ts @@ -235,7 +235,7 @@ export async function getAdminYear( db .prepare( `SELECT p.id, p.name, pv.status video_status FROM projects p - LEFT JOIN project_videos pv ON pv.project_id = p.id AND pv.retired_at IS NULL + LEFT JOIN video_submissions pv ON pv.project_id = p.id AND pv.retired_at IS NULL WHERE p.year_id = ? AND p.kind = 'project' AND p.status = 'active' ORDER BY p.name COLLATE NOCASE, p.id`, ) diff --git a/src/worker/services/videos.ts b/src/worker/services/videos.ts index 18b55e4..538d014 100644 --- a/src/worker/services/videos.ts +++ b/src/worker/services/videos.ts @@ -87,7 +87,7 @@ export async function listPlaylist( pv.duration_seconds, pv.gain_db, so.position FROM screening_order so JOIN projects p ON p.id = so.project_id AND p.status = 'active' - JOIN project_videos pv ON pv.project_id = p.id + JOIN video_submissions pv ON pv.project_id = p.id WHERE so.year_id = ? AND pv.status = 'ready' AND pv.retired_at IS NULL AND pv.processed_r2_key IS NOT NULL AND pv.duration_seconds IS NOT NULL AND pv.gain_db IS NOT NULL @@ -438,7 +438,7 @@ export async function completeVideoUpload( .bind(upload.id), db .prepare( - `INSERT INTO project_videos ( + `INSERT INTO video_submissions ( id, project_id, original_name, content_type, size_bytes, original_r2_key, status, processing_attempt ) VALUES (?, ?, ?, ?, ?, ?, 'queued', 1)`, @@ -531,7 +531,7 @@ export async function retryProjectVideo( const results = await db.batch([ db .prepare( - `UPDATE project_videos SET status = 'queued', processing_attempt = ?, + `UPDATE video_submissions SET status = 'queued', processing_attempt = ?, duration_seconds = NULL, loudness_lufs = NULL, gain_db = NULL, error_message = NULL, processed_r2_key = NULL, updated_at = CURRENT_TIMESTAMP @@ -543,7 +543,7 @@ export async function retryProjectVideo( .prepare( `INSERT INTO video_processing_attempts (video_id, attempt, status) SELECT ?, ?, 'queued' WHERE EXISTS ( - SELECT 1 FROM project_videos WHERE id = ? + SELECT 1 FROM video_submissions WHERE id = ? AND processing_attempt = ? AND status = 'queued' AND retired_at IS NULL )`, ) @@ -589,14 +589,14 @@ export async function claimVideoProcessingAttempt( WHERE video_id = ? AND attempt = ? AND status = 'queued' AND (SELECT COUNT(*) FROM video_processing_attempts WHERE status = 'running') < ? AND EXISTS ( - SELECT 1 FROM project_videos WHERE id = ? AND processing_attempt = ? + SELECT 1 FROM video_submissions WHERE id = ? AND processing_attempt = ? AND status = 'queued' AND retired_at IS NULL )`, ) .bind(outputKey, videoId, attempt, concurrency, videoId, attempt), db .prepare( - `UPDATE project_videos SET status = 'processing', error_message = NULL, + `UPDATE video_submissions SET status = 'processing', error_message = NULL, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND processing_attempt = ? AND status = 'queued' AND retired_at IS NULL AND EXISTS ( @@ -627,7 +627,7 @@ export async function publishVideoProcessingAttempt( const updates = await db.batch([ db .prepare( - `UPDATE project_videos SET status = 'ready', processed_r2_key = ?, + `UPDATE video_submissions SET status = 'ready', processed_r2_key = ?, duration_seconds = ?, loudness_lufs = ?, gain_db = 0, error_message = NULL, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND processing_attempt = ? AND status = 'processing' @@ -654,7 +654,7 @@ export async function publishVideoProcessingAttempt( finished_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE video_id = ? AND attempt = ? AND status = 'running' AND output_r2_key = ? AND EXISTS ( - SELECT 1 FROM project_videos WHERE id = ? AND processing_attempt = ? + SELECT 1 FROM video_submissions WHERE id = ? AND processing_attempt = ? AND status = 'ready' AND retired_at IS NULL AND processed_r2_key = ? )`, ) @@ -673,7 +673,7 @@ export async function failVideoProcessingAttempt( const updates = await db.batch([ db .prepare( - `UPDATE project_videos SET status = 'failed', error_message = ?, + `UPDATE video_submissions SET status = 'failed', error_message = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND processing_attempt = ? AND retired_at IS NULL AND status IN ('queued', 'processing') @@ -689,7 +689,7 @@ export async function failVideoProcessingAttempt( finished_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE video_id = ? AND attempt = ? AND status IN ('queued', 'running') AND EXISTS ( - SELECT 1 FROM project_videos WHERE id = ? AND processing_attempt = ? + SELECT 1 FROM video_submissions WHERE id = ? AND processing_attempt = ? AND status = 'failed' AND retired_at IS NULL )`, ) @@ -720,7 +720,7 @@ export async function retireProjectVideo( await db.batch([ db .prepare( - `UPDATE project_videos SET status = 'retired', retired_at = CURRENT_TIMESTAMP, + `UPDATE video_submissions SET status = 'retired', retired_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND retired_at IS NULL`, ) .bind(video.id), @@ -844,7 +844,7 @@ function mapVideo(row: VideoRow): ProjectVideo { function videoSelect() { return `SELECT id, project_id, original_name, content_type, size_bytes, status, processing_attempt, processed_r2_key, duration_seconds, loudness_lufs, gain_db, - error_message, created_at FROM project_videos`; + error_message, created_at FROM video_submissions`; } async function requireReadyVideo(db: D1Database, videoId: string) { @@ -942,7 +942,7 @@ async function processingAttempt(db: D1Database, videoId: string, attempt: numbe .prepare( `SELECT pv.id video_id, pv.project_id, pv.original_r2_key, pv.processing_attempt, pv.status video_status, vpa.status attempt_status - FROM project_videos pv + FROM video_submissions pv JOIN video_processing_attempts vpa ON vpa.video_id = pv.id AND vpa.attempt = ? WHERE pv.id = ?`, diff --git a/test/e2e/video-rollout.test.ts b/test/e2e/video-rollout.test.ts index 41fa84f..aa8ffc1 100644 --- a/test/e2e/video-rollout.test.ts +++ b/test/e2e/video-rollout.test.ts @@ -62,7 +62,7 @@ describe('video rollout preparation', () => { expect(readiness).toContain('/api/videos/playlist?year=9999'); expect(readiness).toContain("output('ffprobe'"); expect(readiness).not.toContain('STREAM_MODE'); - expect(readiness).not.toMatch(/UPDATE project_videos SET status\s*=\s*'ready'/); + expect(readiness).not.toMatch(/UPDATE video_submissions SET status\s*=\s*'ready'/); }); it('documents retained storage and the explicit production approval boundary', async () => { @@ -72,4 +72,22 @@ describe('video rollout preparation', () => { expect(runbook).toContain('No automatic deletion'); expect(runbook).toContain('hackweek-video-media-production'); }); + + it('keeps migration, deployment, rollback, and future contraction compatible', async () => { + const [migration, workflow, runbook] = await Promise.all([ + readFile('migrations/0007_r2_video_lifecycle.sql', 'utf8'), + readFile('.github/workflows/deploy.yml', 'utf8'), + readFile('VIDEO_ROLLOUT.md', 'utf8'), + ]); + + expect(migration).toContain('CREATE TABLE video_submissions'); + expect(migration).not.toMatch(/ALTER TABLE project_videos|DROP TABLE stream_events/); + expect(workflow).toContain('Apply expand-compatible D1 migrations'); + expect(workflow.indexOf('Apply expand-compatible D1 migrations')).toBeLessThan( + workflow.indexOf('Deploy Worker and static assets'), + ); + expect(runbook).toContain('recorded pre-release Worker version'); + expect(runbook).toContain('does not alter `project_videos` or `stream_events`'); + expect(runbook).toContain('Future contraction (separate approval required)'); + }); }); diff --git a/test/migration/migration.test.ts b/test/migration/migration.test.ts index 22f5928..a9d6db9 100644 --- a/test/migration/migration.test.ts +++ b/test/migration/migration.test.ts @@ -1,5 +1,6 @@ import {readFile} from 'node:fs/promises'; import path from 'node:path'; +import {DatabaseSync} from 'node:sqlite'; import {describe, expect, it} from 'vitest'; import {migrationSql} from '../../scripts/migrate/import'; @@ -23,18 +24,155 @@ async function fixture(name: string) { } describe('Firebase migration transformation', () => { - it('adds the forward R2 video history and multipart constraints', async () => { - const sql = await readFile( - path.resolve('migrations/0007_r2_video_lifecycle.sql'), - 'utf8', - ); + it('keeps populated legacy SQL rollback-compatible while adding the R2 lifecycle', async () => { + const database = new DatabaseSync(':memory:'); + try { + for (let version = 1; version <= 6; version += 1) { + const name = String(version).padStart(4, '0'); + const migration = await readFile( + path.resolve( + 'migrations', + `${name}_${ + [ + 'initial', + 'access_identity', + 'voting_administration', + 'stream_video_lifecycle', + 'google_oauth_sessions', + 'session_view_mode', + ][version - 1] + }.sql`, + ), + 'utf8', + ); + database.exec(migration); + } + database.exec(` + INSERT INTO users (id, source_uid, email, display_name) + VALUES ('legacy-user', 'legacy-source', 'legacy@example.com', 'Legacy User'); + INSERT INTO years (id) VALUES ('legacy-year'); + INSERT INTO projects (id, source_id, year_id, creator_id, name) + VALUES + ('legacy-project', 'legacy-project', 'legacy-year', 'legacy-user', 'Legacy Project'), + ('r2-project', 'r2-project', 'legacy-year', 'legacy-user', 'R2 Project'); + INSERT INTO media ( + id, source_id, project_id, original_name, r2_key, media_type, status + ) VALUES ( + 'legacy-media', 'legacy-media', 'legacy-project', 'legacy.mp4', + 'legacy/media.mp4', 'video/mp4', 'available' + ); + INSERT INTO project_videos ( + id, project_id, stream_uid, source_media_id, status, duration_seconds, + loudness_lufs, gain_db, sort_order, upload_expires_at, failure_stage, + measurement_attempts, archive_status, archive_attempts + ) VALUES ( + 'legacy-video', 'legacy-project', 'stream-before', 'legacy-media', 'ready', + 42, -18, 2, 1, NULL, NULL, 1, 'pending', 0 + ); + INSERT INTO stream_events (event_id, stream_uid, event_type) + VALUES ('legacy-event', 'stream-before', 'video.ready'); + `); + + const expand = await readFile( + path.resolve('migrations/0007_r2_video_lifecycle.sql'), + 'utf8', + ); + expect(expand).not.toMatch(/ALTER TABLE project_videos|DROP TABLE stream_events/); + database.exec(expand); + + expect( + database + .prepare(`SELECT id, project_id, stream_uid, source_media_id, status, + duration_seconds, loudness_lufs, gain_db, error_message, failure_stage, + archive_status, archive_error FROM project_videos WHERE stream_uid = ?`) + .get('stream-before'), + ).toMatchObject({ + id: 'legacy-video', + project_id: 'legacy-project', + source_media_id: 'legacy-media', + status: 'ready', + duration_seconds: 42, + }); + expect( + database + .prepare('SELECT event_type FROM stream_events WHERE event_id = ?') + .get('legacy-event'), + ).toMatchObject({event_type: 'video.ready'}); + + database.exec(` + INSERT INTO stream_events (event_id, stream_uid, event_type) + VALUES ('rollback-event', 'stream-before', 'video.uploading'); + INSERT INTO project_videos ( + id, project_id, stream_uid, status, upload_expires_at, + error_message, failure_stage, duration_seconds, loudness_lufs, gain_db, + archive_status, archive_error + ) VALUES ( + 'ignored-on-conflict', 'legacy-project', 'stream-after', 'uploading', + '2030-01-01T00:00:00.000Z', NULL, NULL, NULL, NULL, NULL, 'pending', NULL + ) ON CONFLICT(project_id) DO UPDATE SET + stream_uid = excluded.stream_uid, source_media_id = NULL, status = 'uploading', + upload_expires_at = excluded.upload_expires_at, duration_seconds = NULL, + loudness_lufs = NULL, gain_db = NULL, error_message = NULL, + failure_stage = NULL, archive_status = 'pending', archive_error = NULL, + archived_at = NULL, updated_at = CURRENT_TIMESTAMP; + `); + expect( + database + .prepare('SELECT stream_uid, status FROM project_videos WHERE project_id = ?') + .get('legacy-project'), + ).toMatchObject({stream_uid: 'stream-after', status: 'uploading'}); + expect( + database.prepare('SELECT COUNT(*) count FROM stream_events').get(), + ).toMatchObject({count: 2}); - expect(sql).toContain('ALTER TABLE project_videos RENAME TO legacy_project_videos'); - expect(sql).toContain('CREATE TABLE video_uploads'); - expect(sql).toContain('CREATE TABLE video_upload_parts'); - expect(sql).toContain('CREATE TABLE video_processing_attempts'); - expect(sql).toContain('WHERE retired_at IS NULL'); - expect(sql).toContain("status IN ('creating', 'uploading', 'completing')"); + database.exec(` + INSERT INTO video_uploads ( + id, video_id, project_id, creator_id, r2_upload_id, original_r2_key, + original_name, content_type, expected_size_bytes, part_size_bytes, + status, expires_at, completed_at + ) VALUES ( + 'r2-upload', 'r2-video', 'r2-project', 'legacy-user', 'multipart-id', + 'r2/original.mp4', 'original.mp4', 'video/mp4', 11, 5242880, + 'completed', '2030-01-01T00:00:00.000Z', CURRENT_TIMESTAMP + ); + INSERT INTO video_submissions ( + id, project_id, original_name, content_type, size_bytes, original_r2_key, + status, processing_attempt + ) VALUES ( + 'r2-video', 'r2-project', 'original.mp4', 'video/mp4', 11, + 'r2/original.mp4', 'queued', 1 + ); + INSERT INTO video_processing_attempts (video_id, attempt, status) + VALUES ('r2-video', 1, 'queued'); + `); + expect( + database + .prepare(`SELECT vs.status, vs.original_r2_key, vpa.status attempt_status + FROM video_submissions vs + JOIN video_processing_attempts vpa ON vpa.video_id = vs.id + WHERE vs.id = 'r2-video' AND vpa.attempt = 1`) + .get(), + ).toMatchObject({ + status: 'queued', + original_r2_key: 'r2/original.mp4', + attempt_status: 'queued', + }); + expect(() => + database.exec(`INSERT INTO video_submissions ( + id, project_id, original_name, size_bytes, original_r2_key + ) VALUES ('r2-conflict', 'r2-project', 'conflict.mp4', 5, 'r2/conflict.mp4')`), + ).toThrow(/UNIQUE constraint failed/); + database.exec(` + UPDATE video_submissions SET status = 'retired', retired_at = CURRENT_TIMESTAMP + WHERE id = 'r2-video'; + INSERT INTO video_submissions ( + id, project_id, original_name, size_bytes, original_r2_key + ) VALUES ('r2-replacement', 'r2-project', 'replacement.mp4', 5, 'r2/replacement.mp4'); + `); + expect(database.prepare('PRAGMA foreign_key_check').all()).toEqual([]); + } finally { + database.close(); + } }); it('preserves deterministic IDs, relationships, and storage keys', async () => { diff --git a/test/video/video.test.ts b/test/video/video.test.ts index 0e75bdf..ac92c20 100644 --- a/test/video/video.test.ts +++ b/test/video/video.test.ts @@ -130,7 +130,7 @@ describe('R2 multipart video lifecycle', () => { expect(duplicateCompletion.body.video.id).toBe(completed.body.video.id); const stored = await env.DB.prepare( - `SELECT original_r2_key FROM project_videos WHERE id = ?`, + `SELECT original_r2_key FROM video_submissions WHERE id = ?`, ) .bind(completed.body.video.id) .first<{original_r2_key: string}>(); @@ -162,7 +162,7 @@ describe('R2 multipart video lifecycle', () => { const activeIndex = await env.DB.prepare( `SELECT sql FROM sqlite_master WHERE type = 'index' - AND name = 'project_videos_active_project_idx'`, + AND name = 'video_submissions_active_project_idx'`, ).first<{sql: string}>(); expect(activeIndex?.sql).toContain('WHERE retired_at IS NULL'); }); @@ -182,7 +182,7 @@ describe('R2 multipart video lifecycle', () => { expect(retired.status).toBe(204); expect(await env.VIDEOS.head(key)).not.toBeNull(); const retiredRow = await env.DB.prepare( - 'SELECT status, original_r2_key FROM project_videos WHERE id = ?', + 'SELECT status, original_r2_key FROM video_submissions WHERE id = ?', ) .bind(video.id) .first<{status: string; original_r2_key: string}>(); @@ -308,7 +308,7 @@ describe('R2 multipart video lifecycle', () => { ).toBe(false); const stored = await env.DB.prepare( `SELECT status, original_r2_key, processed_r2_key, duration_seconds, - loudness_lufs, processing_attempt FROM project_videos WHERE id = ?`, + loudness_lufs, processing_attempt FROM video_submissions WHERE id = ?`, ) .bind(video.id) .first(); @@ -508,7 +508,7 @@ describe('R2 multipart video lifecycle', () => { claimVideoProcessingAttempt(env.DB, right.video.id, 1, 1), ).rejects.toBeInstanceOf(ProcessingCapacityError); expect( - await env.DB.prepare('SELECT status FROM project_videos WHERE id = ?') + await env.DB.prepare('SELECT status FROM video_submissions WHERE id = ?') .bind(right.video.id) .first('status'), ).toBe('queued'); @@ -587,7 +587,7 @@ async function completeSmallUpload(project: string, token: string, bytes: string ); expect(completed.status).toBe(200); const stored = await env.DB.prepare( - 'SELECT original_r2_key FROM project_videos WHERE id = ?', + 'SELECT original_r2_key FROM video_submissions WHERE id = ?', ) .bind(completed.body.video.id) .first<{original_r2_key: string}>(); From 91d26cc5513495edf3b168ccd2e2cbf8ec236baa Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 16:26:16 +0200 Subject: [PATCH 06/18] fix(video): reap expired multipart uploads before create Fence stale upload rows in an active expiring state before aborting their exact R2 multipart session, and release the project slot only after storage cleanup is confirmed. Treat missing uploads idempotently while preserving the fence and returning a retryable error for transient abort failures. Lease in-flight completion attempts so expiration cleanup cannot preempt a current completion. Cover lost resume metadata, duplicate fresh creates, missing multipart state, transient failures, and completion races. --- migrations/0007_r2_video_lifecycle.sql | 12 +- src/shared/videos.ts | 1 + src/worker/services/videos.ts | 209 ++++++++++++++++++------ test/video/video.test.ts | 217 +++++++++++++++++++++++++ 4 files changed, 387 insertions(+), 52 deletions(-) diff --git a/migrations/0007_r2_video_lifecycle.sql b/migrations/0007_r2_video_lifecycle.sql index 1ca0369..4d1b0fc 100644 --- a/migrations/0007_r2_video_lifecycle.sql +++ b/migrations/0007_r2_video_lifecycle.sql @@ -41,24 +41,26 @@ CREATE TABLE video_uploads ( expected_size_bytes INTEGER NOT NULL CHECK (expected_size_bytes BETWEEN 1 AND 5368709120), part_size_bytes INTEGER NOT NULL CHECK (part_size_bytes >= 5242880), status TEXT NOT NULL DEFAULT 'creating' - CHECK (status IN ('creating', 'uploading', 'completing', 'completed', 'aborted', 'expired')), + CHECK (status IN ( + 'creating', 'uploading', 'completing', 'expiring', 'completed', 'aborted', 'expired' + )), expires_at TEXT NOT NULL, completed_at TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - CHECK (r2_upload_id IS NOT NULL OR status IN ('creating', 'aborted')), + CHECK (r2_upload_id IS NOT NULL OR status IN ('creating', 'expiring', 'aborted', 'expired')), CHECK ((status = 'completed') = (completed_at IS NOT NULL)) ) STRICT; CREATE UNIQUE INDEX video_uploads_active_project_idx ON video_uploads(project_id) - WHERE status IN ('creating', 'uploading', 'completing'); + WHERE status IN ('creating', 'uploading', 'completing', 'expiring'); CREATE INDEX video_uploads_expiry_idx ON video_uploads(status, expires_at); CREATE TRIGGER video_uploads_reject_active_submission BEFORE INSERT ON video_uploads -WHEN NEW.status IN ('creating', 'uploading', 'completing') +WHEN NEW.status IN ('creating', 'uploading', 'completing', 'expiring') AND EXISTS ( SELECT 1 FROM video_submissions WHERE project_id = NEW.project_id AND retired_at IS NULL @@ -73,7 +75,7 @@ WHEN NEW.retired_at IS NULL AND EXISTS ( SELECT 1 FROM video_uploads WHERE project_id = NEW.project_id - AND status IN ('creating', 'uploading', 'completing') + AND status IN ('creating', 'uploading', 'completing', 'expiring') ) BEGIN SELECT RAISE(ABORT, 'active project upload exists'); diff --git a/src/shared/videos.ts b/src/shared/videos.ts index 91710c2..b859163 100644 --- a/src/shared/videos.ts +++ b/src/shared/videos.ts @@ -4,6 +4,7 @@ export type VideoUploadStatus = | 'creating' | 'uploading' | 'completing' + | 'expiring' | 'completed' | 'aborted' | 'expired'; diff --git a/src/worker/services/videos.ts b/src/worker/services/videos.ts index 538d014..99c81c2 100644 --- a/src/worker/services/videos.ts +++ b/src/worker/services/videos.ts @@ -14,6 +14,8 @@ import {ServiceError} from './errors'; export const MAX_VIDEO_BYTES = 5 * 1024 * 1024 * 1024; export const VIDEO_PART_SIZE = 50 * 1024 * 1024; export const UPLOAD_EXPIRY_MINUTES = 24 * 60; +const UPLOAD_COMPLETION_LEASE_MINUTES = 15; +const MAX_EXPIRED_UPLOAD_SWEEP = 100; interface VideoRow { id: string; @@ -57,6 +59,12 @@ interface UploadRow { expires_at: string; } +interface ReapExpiredUploadOptions { + projectId?: string; + now?: Date; + limit?: number; +} + interface ProjectAuthorizationRow { id: string; year_id: string; @@ -211,6 +219,7 @@ export async function createMultipartVideoUpload( now = new Date(), ) { await authorizeVideoWrite(db, projectId, user); + await reapExpiredMultipartVideoUploads(db, bucket, {projectId, now, limit: 1}); const uploadId = crypto.randomUUID(); const videoId = crypto.randomUUID(); const originalKey = videoOriginalKey(projectId, videoId, input.fileName); @@ -274,6 +283,44 @@ export async function createMultipartVideoUpload( return getVideoUpload(db, bucket, projectId, uploadId, user, now); } +export async function reapExpiredMultipartVideoUploads( + db: D1Database, + bucket: R2Bucket, + options: ReapExpiredUploadOptions = {}, +) { + const now = options.now ?? new Date(); + const limit = options.limit ?? MAX_EXPIRED_UPLOAD_SWEEP; + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_EXPIRED_UPLOAD_SWEEP) { + throw new Error( + `Expired video upload sweep limit must be between 1 and ${MAX_EXPIRED_UPLOAD_SWEEP}`, + ); + } + + const projectClause = options.projectId ? 'AND project_id = ?' : ''; + const bindings: Array = [now.toISOString()]; + if (options.projectId) bindings.push(options.projectId); + bindings.push(limit); + const {results} = await db + .prepare( + `SELECT id, video_id, project_id, creator_id, r2_upload_id, original_r2_key, + original_name, content_type, expected_size_bytes, part_size_bytes, + status, expires_at + FROM video_uploads + WHERE (status = 'expiring' OR ( + status IN ('creating', 'uploading', 'completing') AND expires_at <= ? + )) ${projectClause} + ORDER BY expires_at, id LIMIT ?`, + ) + .bind(...bindings) + .all(); + + let reaped = 0; + for (const upload of results) { + if (await reapExpiredMultipartUpload(db, bucket, upload, now)) reaped += 1; + } + return reaped; +} + export async function getVideoUpload( db: D1Database, bucket: R2Bucket, @@ -283,16 +330,10 @@ export async function getVideoUpload( now = new Date(), ) { await authorizeVideoWrite(db, projectId, user); - const upload = await requireUpload(db, projectId, uploadId); - if (isExpired(upload, now)) { - if (upload.r2_upload_id) { - await bucket - .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) - .abort() - .catch(() => undefined); - } - await markExpired(db, upload.id); - upload.status = 'expired'; + let upload = await requireUpload(db, projectId, uploadId); + if (isExpired(upload, now) || upload.status === 'expiring') { + await reapExpiredMultipartUpload(db, bucket, upload, now); + upload = await requireUpload(db, projectId, uploadId); } const video = upload.status === 'completed' ? await requireVideoById(db, upload.video_id) : null; @@ -386,19 +427,24 @@ export async function completeVideoUpload( const storedParts = await listStoredParts(db, upload.id); validateCompletionParts(upload, storedParts, suppliedParts); - if (upload.status === 'uploading') { - const claimed = await db - .prepare( - `UPDATE video_uploads SET status = 'completing', updated_at = CURRENT_TIMESTAMP - WHERE id = ? AND status = 'uploading'`, - ) - .bind(upload.id) - .run(); - if (!claimed.meta.changes) { - throw new ServiceError('CONFLICT', 'Upload completion is already in progress', 409); - } - upload.status = 'completing'; + const completionLease = new Date( + now.getTime() + UPLOAD_COMPLETION_LEASE_MINUTES * 60_000, + ).toISOString(); + const claimed = await db + .prepare( + `UPDATE video_uploads SET status = 'completing', + expires_at = CASE WHEN expires_at < ? THEN ? ELSE expires_at END, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status IN ('uploading', 'completing') AND expires_at > ?`, + ) + .bind(completionLease, completionLease, upload.id, now.toISOString()) + .run(); + if (!claimed.meta.changes) { + throw new ServiceError('CONFLICT', 'Upload completion was superseded', 409); } + upload.status = 'completing'; + upload.expires_at = + upload.expires_at < completionLease ? completionLease : upload.expires_at; let object = await bucket.head(upload.original_r2_key); if (!object) { @@ -482,13 +528,9 @@ export async function abortVideoUpload( await authorizeVideoWrite(db, projectId, user); const upload = await requireUpload(db, projectId, uploadId); if (upload.status === 'aborted') return; - if (upload.status === 'expired') { - if (upload.r2_upload_id) { - await bucket - .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) - .abort() - .catch(() => undefined); - } + if (upload.status === 'expired') return; + if (upload.status === 'expiring') { + await reapExpiredMultipartUpload(db, bucket, upload, new Date()); return; } if (upload.status === 'completed') { @@ -740,14 +782,8 @@ async function assertUploadIsWritable( upload: UploadRow, now: Date, ) { - if (isExpired(upload, now)) { - if (upload.r2_upload_id) { - await bucket - .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) - .abort() - .catch(() => undefined); - } - await markExpired(db, upload.id); + if (isExpired(upload, now) || upload.status === 'expiring') { + await reapExpiredMultipartUpload(db, bucket, upload, now); throw new ServiceError('CONFLICT', 'Upload session has expired', 409); } if (upload.status === 'aborted' || upload.status === 'expired') { @@ -755,6 +791,95 @@ async function assertUploadIsWritable( } } +async function reapExpiredMultipartUpload( + db: D1Database, + bucket: R2Bucket, + upload: UploadRow, + now: Date, +) { + if (upload.status !== 'expiring') { + const fenced = await db + .prepare( + `UPDATE video_uploads SET status = 'expiring', updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND project_id = ? AND video_id = ? AND original_r2_key = ? + AND status = ? AND expires_at = ? AND expires_at <= ? + AND (r2_upload_id = ? OR (r2_upload_id IS NULL AND ? IS NULL))`, + ) + .bind( + upload.id, + upload.project_id, + upload.video_id, + upload.original_r2_key, + upload.status, + upload.expires_at, + now.toISOString(), + upload.r2_upload_id, + upload.r2_upload_id, + ) + .run(); + if (!fenced.meta.changes) return false; + upload.status = 'expiring'; + } + + if (upload.r2_upload_id) { + try { + await bucket + .resumeMultipartUpload(upload.original_r2_key, upload.r2_upload_id) + .abort(); + } catch (error) { + if (!isMissingMultipartUpload(error)) throw uploadCleanupUnavailable(); + } + + let completedObject: R2Object | null; + try { + completedObject = await bucket.head(upload.original_r2_key); + } catch { + throw uploadCleanupUnavailable(); + } + if (completedObject) { + throw new ServiceError( + 'CONFLICT', + 'Video upload completed while expiration cleanup was running', + 409, + ); + } + } + + const expired = await db + .prepare( + `UPDATE video_uploads SET status = 'expired', updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND project_id = ? AND video_id = ? AND status = 'expiring' + AND original_r2_key = ? + AND (r2_upload_id = ? OR (r2_upload_id IS NULL AND ? IS NULL))`, + ) + .bind( + upload.id, + upload.project_id, + upload.video_id, + upload.original_r2_key, + upload.r2_upload_id, + upload.r2_upload_id, + ) + .run(); + return expired.meta.changes === 1; +} + +function uploadCleanupUnavailable() { + return new ServiceError( + 'SERVICE_UNAVAILABLE', + 'Expired video upload cleanup could not confirm storage abort; retry the upload request', + 503, + ); +} + +function isMissingMultipartUpload(error: unknown) { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes('(10024)') || + message.includes('The specified multipart upload does not exist') + ); +} + async function authorizeVideoWrite(db: D1Database, projectId: string, user: SessionUser) { const project = await db .prepare( @@ -918,16 +1043,6 @@ function isExpired(upload: UploadRow, now: Date) { ); } -function markExpired(db: D1Database, uploadId: string) { - return db - .prepare( - `UPDATE video_uploads SET status = 'expired', updated_at = CURRENT_TIMESTAMP - WHERE id = ? AND status IN ('creating', 'uploading', 'completing')`, - ) - .bind(uploadId) - .run(); -} - function isVideoSlotConflict(error: unknown) { const message = error instanceof Error ? error.message : String(error); return ( diff --git a/test/video/video.test.ts b/test/video/video.test.ts index ac92c20..342c3ef 100644 --- a/test/video/video.test.ts +++ b/test/video/video.test.ts @@ -4,10 +4,13 @@ import {beforeEach, describe, expect, it} from 'vitest'; import type {ProjectWriteRequest} from '../../src/shared/projects'; import { claimVideoProcessingAttempt, + completeVideoUpload, + createMultipartVideoUpload, failVideoProcessingAttempt, MAX_VIDEO_BYTES, ProcessingCapacityError, publishVideoProcessingAttempt, + reapExpiredMultipartVideoUploads, VIDEO_PART_SIZE, } from '../../src/worker/services/videos'; import type {VideoProcessorResult} from '../../src/worker/video-processing'; @@ -167,6 +170,202 @@ describe('R2 multipart video lifecycle', () => { expect(activeIndex?.sql).toContain('WHERE retired_at IS NULL'); }); + it('reaps an expired upload with lost resume state before creating a fresh upload', async () => { + const stale = await createUpload(projectId, ownerToken, 11); + const staleUploadId = stale.body.upload.uploadId as string; + expect( + ( + await putPart( + projectId, + staleUploadId, + 1, + new TextEncoder().encode('stale video'), + ownerToken, + ) + ).status, + ).toBe(200); + const staleStorage = await uploadStorage(staleUploadId); + await expireUpload(staleUploadId); + + const fresh = await createUpload(projectId, ownerToken, 12); + expect(fresh.status).toBe(201); + expect(fresh.body.upload.uploadId).not.toBe(staleUploadId); + expect( + await env.DB.prepare('SELECT status FROM video_uploads WHERE id = ?') + .bind(staleUploadId) + .first('status'), + ).toBe('expired'); + expect(await env.VIDEOS.head(staleStorage.original_r2_key)).toBeNull(); + await expect( + env.VIDEOS.resumeMultipartUpload( + staleStorage.original_r2_key, + staleStorage.r2_upload_id, + ).uploadPart(1, new Uint8Array([1]).buffer), + ).rejects.toThrow(/multipart upload does not exist|10024/i); + }); + + it('retries idempotent expiry cleanup without releasing after transient R2 failure', async () => { + const stale = await createUpload(projectId, ownerToken, 10); + const staleUploadId = stale.body.upload.uploadId as string; + await expireUpload(staleUploadId); + const failingBucket = { + resumeMultipartUpload() { + return { + abort: async () => { + throw new Error('temporary R2 outage'); + }, + }; + }, + } as unknown as R2Bucket; + + await expect( + reapExpiredMultipartVideoUploads(env.DB, failingBucket, {projectId, limit: 1}), + ).rejects.toMatchObject({code: 'SERVICE_UNAVAILABLE', status: 503}); + expect( + await env.DB.prepare('SELECT status FROM video_uploads WHERE id = ?') + .bind(staleUploadId) + .first('status'), + ).toBe('expiring'); + expect((await createUpload(projectId, ownerToken, 12)).status).toBe(201); + expect( + await env.DB.prepare('SELECT status FROM video_uploads WHERE id = ?') + .bind(staleUploadId) + .first('status'), + ).toBe('expired'); + }); + + it('handles missing multipart state and simultaneous fresh creates idempotently', async () => { + const stale = await createUpload(projectId, ownerToken, 10); + const staleUploadId = stale.body.upload.uploadId as string; + const staleStorage = await uploadStorage(staleUploadId); + await env.VIDEOS.resumeMultipartUpload( + staleStorage.original_r2_key, + staleStorage.r2_upload_id, + ).abort(); + await expireUpload(staleUploadId); + + const fresh = await Promise.all([ + createUpload(projectId, ownerToken, 12), + createUpload(projectId, ownerToken, 12), + ]); + expect(fresh.map(({status}) => status).sort((a, b) => a - b)).toEqual([201, 409]); + expect( + await env.DB.prepare('SELECT status FROM video_uploads WHERE id = ?') + .bind(staleUploadId) + .first('status'), + ).toBe('expired'); + }); + + it('fences an expired upload before an old completion can publish', async () => { + const stale = await createUpload(projectId, ownerToken, 11); + const staleUploadId = stale.body.upload.uploadId as string; + const part = await putPart( + projectId, + staleUploadId, + 1, + new TextEncoder().encode('stale video'), + ownerToken, + ); + expect(part.status).toBe(200); + await expireUpload(staleUploadId); + + const fresh = createUpload(projectId, ownerToken, 12); + const completion = api( + `/projects/${projectId}/video/upload/${staleUploadId}/complete`, + ownerToken, + { + method: 'POST', + body: {parts: [{partNumber: 1, etag: part.body.part.etag}]}, + }, + ); + const [freshResult, completionResult] = await Promise.all([fresh, completion]); + expect(freshResult.status).toBe(201); + expect(completionResult.status).toBe(409); + expect( + await env.DB.prepare('SELECT status FROM video_uploads WHERE id = ?') + .bind(staleUploadId) + .first('status'), + ).toBe('expired'); + }); + + it('leases an in-flight completion against a concurrent fresh create', async () => { + const created = await createUpload(projectId, ownerToken, 11); + const uploadId = created.body.upload.uploadId as string; + const part = await putPart( + projectId, + uploadId, + 1, + new TextEncoder().encode('final video'), + ownerToken, + ); + expect(part.status).toBe(200); + const upload = await uploadStorage(uploadId); + const completionStart = new Date('2030-01-01T00:00:00.000Z'); + await env.DB.prepare('UPDATE video_uploads SET expires_at = ? WHERE id = ?') + .bind('2030-01-01T00:01:00.000Z', uploadId) + .run(); + + let enteredCompletion!: () => void; + let releaseCompletion!: () => void; + const completionEntered = new Promise((resolve) => { + enteredCompletion = resolve; + }); + const completionReleased = new Promise((resolve) => { + releaseCompletion = resolve; + }); + const multipart = env.VIDEOS.resumeMultipartUpload( + upload.original_r2_key, + upload.r2_upload_id, + ); + const blockingBucket = { + head: env.VIDEOS.head.bind(env.VIDEOS), + resumeMultipartUpload() { + return { + complete: async (parts: R2UploadedPart[]) => { + enteredCompletion(); + await completionReleased; + return multipart.complete(parts); + }, + }; + }, + } as unknown as R2Bucket; + const owner = { + id: ownerId, + email: `video-owner-${suffix}@sentry.io`, + displayName: 'Hackweek Member', + avatarUrl: null, + role: 'member' as const, + actualRole: 'member' as const, + }; + + const completing = completeVideoUpload( + env.DB, + blockingBucket, + null, + projectId, + uploadId, + [{partNumber: 1, etag: part.body.part.etag}], + owner, + completionStart, + ); + await completionEntered; + try { + await expect( + createMultipartVideoUpload( + env.DB, + env.VIDEOS, + projectId, + owner, + {fileName: 'replacement.mp4', fileSize: 12, contentType: 'video/mp4'}, + new Date('2030-01-01T00:02:00.000Z'), + ), + ).rejects.toMatchObject({code: 'CONFLICT', status: 409}); + } finally { + releaseCompletion(); + } + await expect(completing).resolves.toMatchObject({status: 'queued'}); + }); + it('retires only with confirmation, retains the original, and requires a fresh replacement', async () => { const {video, key} = await completeSmallUpload(projectId, ownerToken, 'first video'); const unconfirmed = await api(`/projects/${projectId}/video`, ownerToken, { @@ -563,6 +762,24 @@ async function responseText(response: Response) { return new TextDecoder().decode(await response.arrayBuffer()); } +async function expireUpload(uploadId: string) { + await env.DB.prepare( + `UPDATE video_uploads SET expires_at = '2000-01-01T00:00:00.000Z' WHERE id = ?`, + ) + .bind(uploadId) + .run(); +} + +async function uploadStorage(uploadId: string) { + const upload = await env.DB.prepare( + `SELECT original_r2_key, r2_upload_id FROM video_uploads WHERE id = ?`, + ) + .bind(uploadId) + .first<{original_r2_key: string; r2_upload_id: string}>(); + if (!upload) throw new Error('upload storage fixture is missing'); + return upload; +} + async function completeSmallUpload(project: string, token: string, bytes: string) { const created = await createUpload(project, token, bytes.length); expect(created.status).toBe(201); From 5585fdd537b20f71d51036c2e3cfe882616eb247 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 17:52:40 +0200 Subject: [PATCH 07/18] fix(video): correct peak-limited loudness output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an initial canonical transcode cannot reach the loudness target because dynamic true-peak limiting is required, re-analyze the derivative and run one audio-only corrective pass while copying the encoded video. Revalidate codec, dimensions, fast-start layout, and the existing ±0.7 LU gate after correction. Add a deterministic high-crest fixture that reproduces the rejected short-clip behavior. --- processor/video-processor.mjs | 174 ++++++++++++++++++++------------ scripts/test-video-processor.ts | 43 +++++++- 2 files changed, 151 insertions(+), 66 deletions(-) diff --git a/processor/video-processor.mjs b/processor/video-processor.mjs index 1bf34ec..7aa7257 100644 --- a/processor/video-processor.mjs +++ b/processor/video-processor.mjs @@ -1,6 +1,6 @@ import {createHash} from 'node:crypto'; import {createReadStream, createWriteStream} from 'node:fs'; -import {mkdtemp, open, rm, stat} from 'node:fs/promises'; +import {mkdtemp, open, rename, rm, stat} from 'node:fs/promises'; import {createServer} from 'node:http'; import {tmpdir} from 'node:os'; import path from 'node:path'; @@ -39,62 +39,35 @@ export async function processFile(inputPath, outputPath) { const normalizeAudio = firstPass !== null && firstPass.inputI > -70; await transcode(inputPath, outputPath, firstPass, normalizeAudio); - const output = await probe(outputPath); - const video = output.streams.find((stream) => stream.codec_type === 'video'); - const audio = output.streams.find((stream) => stream.codec_type === 'audio'); - const outputDuration = mediaDuration(output); - if ( - !video || - !audio || - video.codec_name !== 'h264' || - audio.codec_name !== 'aac' || - video.pix_fmt !== 'yuv420p' || - !positive(video.width) || - !positive(video.height) || - video.width > 1920 || - video.height > 1080 || - outputDuration > MAX_DURATION_SECONDS + 0.05 - ) { - throw new ProcessorError( - 'Canonical output failed codec, pixel, size, or duration checks', - ); + let canonical = await validateCanonicalOutput(outputPath, sourceVideo); + let measured = await analyzeLoudness(outputPath); + let loudnessLufs = measured?.inputI ?? null; + if (normalizeAudio && measured && outsideLoudnessTolerance(loudnessLufs)) { + const correctedPath = `${outputPath}.loudness.mp4`; + await correctLoudness(outputPath, correctedPath, measured); + await rename(correctedPath, outputPath); + canonical = await validateCanonicalOutput(outputPath, sourceVideo); + measured = await analyzeLoudness(outputPath); + loudnessLufs = measured?.inputI ?? null; } - - const rotation = sourceRotation(sourceVideo); - const sourceWidth = - Math.abs(rotation) % 180 === 90 ? sourceVideo.height : sourceVideo.width; - const sourceHeight = - Math.abs(rotation) % 180 === 90 ? sourceVideo.width : sourceVideo.height; - if (video.width > sourceWidth || video.height > sourceHeight) { - throw new ProcessorError('Canonical output unexpectedly upscaled the source'); - } - - const fastStart = await hasFastStart(outputPath); - if (!fastStart) throw new ProcessorError('Canonical MP4 is not fast-start enabled'); - const measured = await analyzeLoudness(outputPath); - const loudnessLufs = measured?.inputI ?? null; - if ( - normalizeAudio && - (loudnessLufs === null || - Math.abs(loudnessLufs - TARGET_LUFS) > LOUDNESS_TOLERANCE_LU) - ) { + if (normalizeAudio && outsideLoudnessTolerance(loudnessLufs)) { throw new ProcessorError( `Output loudness ${String(loudnessLufs)} LUFS is outside ${LOUDNESS_TOLERANCE_LU} LU of ${TARGET_LUFS}`, ); } return { - durationSeconds: outputDuration, - width: video.width, - height: video.height, - videoCodec: video.codec_name, - audioCodec: audio.codec_name, - pixelFormat: video.pix_fmt, + durationSeconds: canonical.outputDuration, + width: canonical.video.width, + height: canonical.video.height, + videoCodec: canonical.video.codec_name, + audioCodec: canonical.audio.codec_name, + pixelFormat: canonical.video.pix_fmt, loudnessLufs, loudnessTargetLufs: TARGET_LUFS, loudnessToleranceLu: LOUDNESS_TOLERANCE_LU, audioMode: normalizeAudio ? 'normalized' : 'generated-silence', - fastStart, + fastStart: canonical.fastStart, sha256: await sha256(outputPath), }; } @@ -106,23 +79,7 @@ async function transcode(inputPath, outputPath, firstPass, normalizeAudio) { } args.push('-map', '0:v:0', '-map', normalizeAudio ? '0:a:0' : '1:a:0'); args.push('-vf', SCALE_FILTER); - if (normalizeAudio) { - args.push( - '-af', - [ - `loudnorm=I=${TARGET_LUFS}`, - 'LRA=11', - 'TP=-1.5', - `measured_I=${firstPass.inputI}`, - `measured_LRA=${firstPass.inputLra}`, - `measured_TP=${firstPass.inputTp}`, - `measured_thresh=${firstPass.inputThresh}`, - `offset=${firstPass.targetOffset}`, - 'linear=true', - 'print_format=summary', - ].join(':'), - ); - } + if (normalizeAudio) args.push('-af', loudnormFilter(firstPass)); args.push( '-c:v', 'libx264', @@ -160,6 +117,97 @@ async function transcode(inputPath, outputPath, firstPass, normalizeAudio) { await run('ffmpeg', args); } +async function correctLoudness(inputPath, outputPath, measured) { + await run('ffmpeg', [ + '-hide_banner', + '-nostdin', + '-y', + '-i', + inputPath, + '-map', + '0:v:0', + '-map', + '0:a:0', + '-c:v', + 'copy', + '-af', + loudnormFilter(measured), + '-c:a', + 'aac', + '-b:a', + '192k', + '-ar', + '48000', + '-ac', + '2', + '-map_metadata', + '-1', + '-sn', + '-dn', + '-movflags', + '+faststart', + '-shortest', + outputPath, + ]); +} + +function loudnormFilter(measured) { + return [ + `loudnorm=I=${TARGET_LUFS}`, + 'LRA=11', + 'TP=-1.5', + `measured_I=${measured.inputI}`, + `measured_LRA=${measured.inputLra}`, + `measured_TP=${measured.inputTp}`, + `measured_thresh=${measured.inputThresh}`, + `offset=${measured.targetOffset}`, + 'linear=true', + 'print_format=summary', + ].join(':'); +} + +async function validateCanonicalOutput(outputPath, sourceVideo) { + const output = await probe(outputPath); + const video = output.streams.find((stream) => stream.codec_type === 'video'); + const audio = output.streams.find((stream) => stream.codec_type === 'audio'); + const outputDuration = mediaDuration(output); + if ( + !video || + !audio || + video.codec_name !== 'h264' || + audio.codec_name !== 'aac' || + video.pix_fmt !== 'yuv420p' || + !positive(video.width) || + !positive(video.height) || + video.width > 1920 || + video.height > 1080 || + outputDuration > MAX_DURATION_SECONDS + 0.05 + ) { + throw new ProcessorError( + 'Canonical output failed codec, pixel, size, or duration checks', + ); + } + + const rotation = sourceRotation(sourceVideo); + const sourceWidth = + Math.abs(rotation) % 180 === 90 ? sourceVideo.height : sourceVideo.width; + const sourceHeight = + Math.abs(rotation) % 180 === 90 ? sourceVideo.width : sourceVideo.height; + if (video.width > sourceWidth || video.height > sourceHeight) { + throw new ProcessorError('Canonical output unexpectedly upscaled the source'); + } + + const fastStart = await hasFastStart(outputPath); + if (!fastStart) throw new ProcessorError('Canonical MP4 is not fast-start enabled'); + return {video, audio, outputDuration, fastStart}; +} + +function outsideLoudnessTolerance(loudnessLufs) { + return ( + loudnessLufs === null || Math.abs(loudnessLufs - TARGET_LUFS) > LOUDNESS_TOLERANCE_LU + ); +} + async function analyzeLoudness(file) { const output = await run('ffmpeg', [ '-hide_banner', diff --git a/scripts/test-video-processor.ts b/scripts/test-video-processor.ts index f2bf23f..fd44b37 100644 --- a/scripts/test-video-processor.ts +++ b/scripts/test-video-processor.ts @@ -24,6 +24,7 @@ try { await chmod(work, 0o777); const audible = path.join(work, 'audible.mp4'); + const peakLimited = path.join(work, 'peak-limited.mp4'); const lowSilent = path.join(work, 'low-silent.mp4'); const rotationBase = path.join(work, 'rotation-base.mp4'); const rotated = path.join(work, 'rotated.mp4'); @@ -52,6 +53,26 @@ try { '-shortest', audible, ]); + ffmpeg([ + '-f', + 'lavfi', + '-i', + 'color=size=320x180:rate=15:color=black', + '-f', + 'lavfi', + '-i', + 'aevalsrc=0.003*sin(2*PI*440*t)+if(between(t\\,1\\,1.005)\\,0.5\\,0):s=48000:c=stereo', + '-t', + '3.5', + '-c:v', + 'libx264', + '-preset', + 'ultrafast', + '-c:a', + 'aac', + '-shortest', + peakLimited, + ]); ffmpeg([ '-f', 'lavfi', @@ -97,8 +118,8 @@ try { ]); await writeFile(malformed, 'not a media file'); await Promise.all( - [audible, lowSilent, rotationBase, rotated, overDuration, malformed].map((file) => - chmod(file, 0o644), + [audible, peakLimited, lowSilent, rotationBase, rotated, overDuration, malformed].map( + (file) => chmod(file, 0o644), ), ); @@ -119,6 +140,22 @@ try { 'MP4 moov precedes mdat', ); + const peakLimitedResult = processFixture('peak-limited.mp4', 'peak-limited-output.mp4'); + assertCanonical(probe(path.join(work, 'peak-limited-output.mp4'))); + assert( + peakLimitedResult.audioMode === 'normalized', + 'peak-limited input remains normalized', + ); + assert( + typeof peakLimitedResult.loudnessLufs === 'number' && + Math.abs(peakLimitedResult.loudnessLufs + 16) <= 0.7, + `peak-limited output is ${String(peakLimitedResult.loudnessLufs)} LUFS within ±0.7 LU`, + ); + assert( + await fastStart(path.join(work, 'peak-limited-output.mp4')), + 'corrective audio pass preserves MP4 fast start', + ); + const silentResult = processFixture('low-silent.mp4', 'low-silent-output.mp4'); const silentProbe = probe(path.join(work, 'low-silent-output.mp4')); assertCanonical(silentProbe); @@ -145,7 +182,7 @@ try { expectFixtureFailure('over-duration.mp4', 'over-output.mp4', 'exceeds 600s'); expectFixtureFailure('audible.mp4', 'missing/output.mp4', 'No such file'); - console.log('Video processor: 28 checks passed'); + console.log('Video processor: 36 checks passed'); } finally { await rm(work, {recursive: true, force: true}); } From 3c6248004951c667a21e22e763c8eaca83795ff8 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 17:55:51 +0200 Subject: [PATCH 08/18] fix(video): expose failed processing retry Let project managers restart a failed Workflow attempt against the retained immutable original instead of retiring and re-uploading the video. Refresh the project-video query cache from the retry response and cover the recovery action in the video UI test. --- src/app/queries/videos.ts | 13 +++++++++++++ src/app/video/ProjectVideoPanel.tsx | 14 ++++++++++++-- test/video-ui/video-ui.test.tsx | 10 +++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/app/queries/videos.ts b/src/app/queries/videos.ts index cc4a34e..b878f77 100644 --- a/src/app/queries/videos.ts +++ b/src/app/queries/videos.ts @@ -55,6 +55,19 @@ async function prepareVideoUpload(projectId: string, file: File) { return created; } +export function useRetryVideo(projectId: string) { + const cache = useQueryClient(); + return useMutation({ + mutationFn: () => + apiRequest( + `/projects/${encodeURIComponent(projectId)}/video/retry`, + jsonRequest('POST', {}), + ), + onSuccess: (response) => + cache.setQueryData(['project-video', projectId], response), + }); +} + export function useDeleteVideo(projectId: string) { const cache = useQueryClient(); return useMutation({ diff --git a/src/app/video/ProjectVideoPanel.tsx b/src/app/video/ProjectVideoPanel.tsx index e75a820..9fe2984 100644 --- a/src/app/video/ProjectVideoPanel.tsx +++ b/src/app/video/ProjectVideoPanel.tsx @@ -4,7 +4,7 @@ import {Link} from 'wouter'; import type {PlaybackResponse, ProjectVideo} from '../../shared/videos'; import {IndividualPlayer} from '../player/IndividualPlayer'; -import {useCreateVideoUpload, useDeleteVideo} from '../queries/videos'; +import {useCreateVideoUpload, useDeleteVideo, useRetryVideo} from '../queries/videos'; import {createMultipartUpload, type ResumableUpload, type UploadSnapshot} from './upload'; const INITIAL_UPLOAD: UploadSnapshot = { @@ -38,6 +38,7 @@ export function ProjectVideoPanel(props: { } = props; const cache = useQueryClient(); const createUpload = useCreateVideoUpload(projectId); + const retryProcessing = useRetryVideo(projectId); const remove = useDeleteVideo(projectId); const controller = useRef(null); const [upload, setUpload] = useState(null); @@ -77,7 +78,7 @@ export function ProjectVideoPanel(props: { } const isUploading = upload && upload.phase !== 'complete'; - const actionError = error ?? remove.error?.message; + const actionError = error ?? retryProcessing.error?.message ?? remove.error?.message; return (
@@ -174,6 +175,15 @@ export function ProjectVideoPanel(props: { /> )} + {video?.status === 'failed' && ( + + )} {video && (
- {video?.status === 'ready' && ( - - watch video - - )} {loading ? (

loading video status…

- ) : video ? ( - - ) : ( + ) : !video ? (

no demo video yet. projects remain complete without one.

- )} + ) : video.status !== 'ready' ? ( + + ) : null} {video?.status === 'ready' && playback && ( @@ -207,10 +198,6 @@ export function ProjectVideoPanel(props: { {actionError}

)} -

- uploads are sent in resumable parts to private R2 storage. completed originals are - retained when a video is retired. -

); } diff --git a/test/video-ui/video-ui.test.tsx b/test/video-ui/video-ui.test.tsx index 0297169..e225214 100644 --- a/test/video-ui/video-ui.test.tsx +++ b/test/video-ui/video-ui.test.tsx @@ -1,5 +1,5 @@ import {QueryClient, QueryClientProvider} from '@tanstack/react-query'; -import {render, screen} from '@testing-library/react'; +import {act, render, screen} from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import {Route, Router} from 'wouter'; import {memoryLocation} from 'wouter/memory-location'; @@ -45,25 +45,49 @@ describe('video user experience', () => { } return json({}); }); + let finishUpload: (() => void) | undefined; const uploadFactory = ( file: File, _session: VideoUploadSession, onChange: (snapshot: UploadSnapshot) => void, - ): ResumableUpload => ({ - start: () => - onChange({phase: 'uploading', bytesSent: 2, bytesTotal: file.size, error: null}), - pause: async () => - onChange({phase: 'paused', bytesSent: 2, bytesTotal: file.size, error: null}), - resume: () => - onChange({phase: 'uploading', bytesSent: 2, bytesTotal: file.size, error: null}), - retry: () => - onChange({phase: 'uploading', bytesSent: 2, bytesTotal: file.size, error: null}), - }); + ): ResumableUpload => { + finishUpload = () => + onChange({ + phase: 'complete', + bytesSent: file.size, + bytesTotal: file.size, + error: null, + }); + return { + start: () => + onChange({ + phase: 'uploading', + bytesSent: 2, + bytesTotal: file.size, + error: null, + }), + pause: async () => + onChange({phase: 'paused', bytesSent: 2, bytesTotal: file.size, error: null}), + resume: () => + onChange({ + phase: 'uploading', + bytesSent: 2, + bytesTotal: file.size, + error: null, + }), + retry: () => + onChange({ + phase: 'uploading', + bytesSent: 2, + bytesTotal: file.size, + error: null, + }), + }; + }; renderQuery( { '/api/projects/project/video/upload', expect.objectContaining({method: 'POST'}), ); + + await act(async () => finishUpload?.()); + expect(screen.queryByRole('progressbar')).toBeNull(); }); it('persists multipart resume identity and skips server-confirmed parts', async () => { @@ -115,25 +142,18 @@ describe('video user experience', () => { expect(readResumeRecord('project', file)).toBeNull(); }); - it('keeps ready playback and uploads independent of Stream configuration', () => { + it('removes redundant ready status, watch link, and storage fine print', () => { const ready = renderQuery( - , - ); - expect(screen.getByRole('link', {name: 'watch video'}).getAttribute('href')).toBe( - '/years/2026/projects/project/video', + , ); + expect(screen.queryByRole('link', {name: 'watch video'})).toBeNull(); + expect(screen.queryByText('ready to watch')).toBeNull(); + expect(screen.queryByText(/private R2 storage/i)).toBeNull(); ready.unmount(); - renderQuery( - , - ); + renderQuery(); expect(screen.getByLabelText('select project video')).toBeTruthy(); - expect(screen.getByText(/private R2 storage/i)).toBeTruthy(); + expect(screen.queryByText(/private R2 storage/i)).toBeNull(); }); it('retries failed processing without requiring another upload', async () => { @@ -143,7 +163,6 @@ describe('video user experience', () => { renderQuery( Date: Tue, 11 Aug 2026 18:19:08 +0200 Subject: [PATCH 10/18] fix(video): surface project video deletion Move the existing video-removal action into the panel header and label it clearly as delete video so managers can find it without scrolling past the player. Explain in the confirmation that deletion enables a replacement upload, while preserving the existing retirement-backed storage behavior. --- src/app/video/ProjectVideoPanel.tsx | 34 ++++++++++++++--------------- test/video-ui/video-ui.test.tsx | 7 ++++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/app/video/ProjectVideoPanel.tsx b/src/app/video/ProjectVideoPanel.tsx index f02d7dc..2ab5f46 100644 --- a/src/app/video/ProjectVideoPanel.tsx +++ b/src/app/video/ProjectVideoPanel.tsx @@ -86,6 +86,22 @@ export function ProjectVideoPanel(props: {

demo reel

project video

+ {canManage && video && ( + + )} {loading ? ( @@ -148,7 +164,7 @@ export function ProjectVideoPanel(props: { )} - {canManage && !isUploading && ( + {canManage && !isUploading && (!video || video.status === 'failed') && (
{!video && (
)} {actionError && ( diff --git a/test/video-ui/video-ui.test.tsx b/test/video-ui/video-ui.test.tsx index e225214..7858953 100644 --- a/test/video-ui/video-ui.test.tsx +++ b/test/video-ui/video-ui.test.tsx @@ -144,11 +144,14 @@ describe('video user experience', () => { it('removes redundant ready status, watch link, and storage fine print', () => { const ready = renderQuery( - , + , ); expect(screen.queryByRole('link', {name: 'watch video'})).toBeNull(); expect(screen.queryByText('ready to watch')).toBeNull(); expect(screen.queryByText(/private R2 storage/i)).toBeNull(); + expect( + screen.getByRole('button', {name: 'delete video'}).closest('header'), + ).not.toBeNull(); ready.unmount(); renderQuery(); @@ -179,7 +182,7 @@ describe('video user experience', () => { '/api/projects/project/video/retry', expect.objectContaining({method: 'POST'}), ); - expect(screen.getByRole('button', {name: 'retire video'})).toBeTruthy(); + expect(screen.getByRole('button', {name: 'delete video'})).toBeTruthy(); }); it('attaches authenticated progressive MP4 directly to an HTML video element', async () => { From 945e2d981b8501581c7be9d05252bda5250f657f Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 18:54:45 +0200 Subject: [PATCH 11/18] feat(reel): add resilient screening playback Include every ready project video without requiring a manual screening entry, while preserving curated projects first and enforcing closed-year member access. Add group-aware HTML interludes, full-viewport screening, play-from-here URL recovery, and automatic continuation past broken clips on top of the existing preloaded player. --- scripts/local-readiness.ts | 3 +- src/app/player/ScreeningPlayer.tsx | 99 +++++++++++++++++++++------ src/app/player/controller.ts | 103 +++++++++++++++++++++-------- src/app/routes/AdminPage.tsx | 4 +- src/app/routes/ProjectsPage.tsx | 8 ++- src/app/routes/WatchPage.tsx | 49 ++++++++++++-- src/app/styles.css | 35 +++++++++- src/shared/videos.ts | 1 + src/worker/routes/videos.ts | 4 +- src/worker/services/videos.ts | 37 ++++++++--- test/app/routes.test.tsx | 27 ++++++-- test/player/controller.test.tsx | 22 ++++-- test/video-ui/video-ui.test.tsx | 22 +++++- test/video/video.test.ts | 18 +++-- 14 files changed, 340 insertions(+), 92 deletions(-) diff --git a/scripts/local-readiness.ts b/scripts/local-readiness.ts index 0efda37..c9eb004 100644 --- a/scripts/local-readiness.ts +++ b/scripts/local-readiness.ts @@ -305,8 +305,9 @@ try { playlist.videos.length === 1 && playlist.videos[0].videoId === videoId && playlist.videos[0].projectName === 'Readiness Video' && + playlist.videos[0].groupName === 'Readiness Team' && playlist.videos[0].teamMembers.includes('Local Developer'), - 'ready derivative appears in curated reel order with team overlay data', + 'ready derivative appears in the reel without a manual screening entry', ); await sendJson('DELETE', '/api/projects/readiness-project/video', {confirmed: true}); diff --git a/src/app/player/ScreeningPlayer.tsx b/src/app/player/ScreeningPlayer.tsx index 48baf87..e752302 100644 --- a/src/app/player/ScreeningPlayer.tsx +++ b/src/app/player/ScreeningPlayer.tsx @@ -1,4 +1,11 @@ -import {useCallback, useEffect, useRef, useState} from 'react'; +import { + forwardRef, + useCallback, + useEffect, + useImperativeHandle, + useRef, + useState, +} from 'react'; import type {PlaylistItem, PlaybackResponse} from '../../shared/videos'; import {createPlayerAudioGraph} from './audio'; @@ -8,24 +15,52 @@ import { type ScreeningController, } from './controller'; -const INITIAL_STATE: PlayerState = {phase: 'idle', index: 0, error: null}; +export interface ScreeningPlayerHandle { + playFrom(videoId: string): void; +} + +const initialState = (index: number): PlayerState => ({ + phase: 'idle', + index, + error: null, + countdownSeconds: null, +}); -export function ScreeningPlayer({ - playlist, - getPlayback, -}: { - playlist: PlaylistItem[]; - getPlayback: (videoId: string) => Promise; -}) { +export const ScreeningPlayer = forwardRef< + ScreeningPlayerHandle, + { + playlist: PlaylistItem[]; + getPlayback: (videoId: string) => Promise; + initialVideoId?: string | null; + onActiveVideoChange?: (videoId: string) => void; + } +>(function ScreeningPlayer( + {playlist, getPlayback, initialVideoId, onActiveVideoChange}, + ref, +) { const shell = useRef(null); const videos = [ useRef(null), useRef(null), ] as const; const controller = useRef(null); - const [state, setState] = useState(INITIAL_STATE); + const announcedVideoId = useRef(null); + const requestedIndex = playlist.findIndex((clip) => clip.videoId === initialVideoId); + const [state, setState] = useState(initialState(Math.max(0, requestedIndex))); const clip = playlist[state.index]; + const publishState = useCallback( + (next: PlayerState) => { + setState(next); + const videoId = playlist[next.index]?.videoId; + if (next.phase === 'title' && videoId && announcedVideoId.current !== videoId) { + announcedVideoId.current = videoId; + onActiveVideoChange?.(videoId); + } + }, + [onActiveVideoChange, playlist], + ); + const buildController = useCallback(() => { if (controller.current) return controller.current; const first = videos[0].current; @@ -37,10 +72,21 @@ export function ScreeningPlayer({ elements: [first, second], audio, getPlayback, - onState: setState, + onState: publishState, }); return controller.current; - }, [getPlayback, playlist, videos]); + }, [getPlayback, playlist, publishState, videos]); + + useImperativeHandle( + ref, + () => ({ + playFrom(videoId) { + const index = playlist.findIndex((item) => item.videoId === videoId); + if (index >= 0) void buildController()?.jumpTo(index); + }, + }), + [buildController, playlist], + ); useEffect(() => () => controller.current?.destroy(), []); @@ -61,7 +107,7 @@ export function ScreeningPlayer({
∅

no videos are ready

-

uploading, processing, measuring, and failed videos stay out of the reel.

+

uploading, processing, and failed videos stay out of the reel.

); } @@ -69,6 +115,7 @@ export function ScreeningPlayer({ const activeSlot = state.index % 2; const showingTitle = state.phase === 'title'; const team = clip.teamMembers.join(' · ') || 'Hackweek team'; + const projectMeta = [clip.groupName, team].filter(Boolean).join(' · '); return (
@@ -88,24 +135,33 @@ export function ScreeningPlayer({

#{String(state.index + 1).padStart(2, '0')} / Hackweek

{clip.projectName}

+ {clip.groupName && {clip.groupName}} {team} + {state.countdownSeconds && ( + + {state.countdownSeconds} + + )}
{['playing', 'paused'].includes(state.phase) && (
{clip.projectName} - {team} + {projectMeta}
)} {state.phase === 'idle' && (
#H

the Hackweek reel

-

{playlist.length} ready project videos · private progressive MP4

+

{playlist.length} ready project videos

sound starts only after you press play
@@ -114,18 +170,19 @@ export function ScreeningPlayer({
✓

that’s the reel

-

all {playlist.length} ready videos played.

+

all remaining ready videos played.

)} {state.phase === 'error' && (
- playback stopped + {clip.projectName} could not be played

{state.error}

+ continuing automatically…
)} @@ -157,7 +214,7 @@ export function ScreeningPlayer({
); -} +}); export function handleScreeningShortcut( event: Pick, diff --git a/src/app/player/controller.ts b/src/app/player/controller.ts index 52361e2..003e840 100644 --- a/src/app/player/controller.ts +++ b/src/app/player/controller.ts @@ -8,10 +8,12 @@ export interface PlayerState { phase: PlayerPhase; index: number; error: string | null; + countdownSeconds: number | null; } export interface ScreeningController { - start(): Promise; + start(index?: number): Promise; + jumpTo(index: number): Promise; togglePause(): Promise; skip(): Promise; destroy(): void; @@ -24,6 +26,7 @@ export function createScreeningController({ getPlayback, onState, titleDurationMs = 1_800, + errorDurationMs = 1_800, attach = attachMp4, }: { playlist: PlaylistItem[]; @@ -32,18 +35,22 @@ export function createScreeningController({ getPlayback: (videoId: string) => Promise; onState: (state: PlayerState) => void; titleDurationMs?: number; + errorDurationMs?: number; attach?: typeof attachMp4; }): ScreeningController { let index = 0; let active: 0 | 1 = 0; let phase: PlayerPhase = 'idle'; + let countdownSeconds: number | null = null; let destroyed = false; let operation = 0; - let titleTimer: ReturnType | null = null; + let transitionTimer: ReturnType | null = null; + let countdownTimer: ReturnType | null = null; const attachments: [MediaAttachment | null, MediaAttachment | null] = [null, null]; const attachedVideoIds: [string | null, string | null] = [null, null]; const slotOperations: [number, number] = [0, 0]; - const notify = (error: string | null = null) => onState({phase, index, error}); + const notify = (error: string | null = null) => + onState({phase, index, error, countdownSeconds}); const endedHandlers = elements.map((_element, slot) => () => { if (phase === 'playing' && slot === active) void advance(); @@ -52,6 +59,14 @@ export function createScreeningController({ element.addEventListener('ended', endedHandlers[slot]), ); + function clearTransitionTimers() { + if (transitionTimer) clearTimeout(transitionTimer); + if (countdownTimer) clearInterval(countdownTimer); + transitionTimer = null; + countdownTimer = null; + countdownSeconds = null; + } + async function prepare(clipIndex: number, slot: 0 | 1) { const clip = playlist[clipIndex]; if (!clip || attachedVideoIds[slot] === clip.videoId) return; @@ -86,18 +101,20 @@ export function createScreeningController({ audio.setGain(slot, clip.gainDb); } - async function playCurrent() { - if (destroyed) return; - const currentOperation = ++operation; - phase = 'title'; + function beginTitleCountdown(currentOperation: number) { + const endsAt = Date.now() + titleDurationMs; + countdownSeconds = Math.max(1, Math.ceil(titleDurationMs / 1_000)); notify(); - await prepare(index, active); - if (destroyed || currentOperation !== operation || phase !== 'title') return; - - const nextSlot = active === 0 ? 1 : 0; - void prepare(index + 1, nextSlot).catch(() => undefined); - titleTimer = setTimeout(() => { - titleTimer = null; + countdownTimer = setInterval(() => { + if (destroyed || currentOperation !== operation || phase !== 'title') return; + const next = Math.max(1, Math.ceil((endsAt - Date.now()) / 1_000)); + if (next !== countdownSeconds) { + countdownSeconds = next; + notify(); + } + }, 100); + transitionTimer = setTimeout(() => { + clearTransitionTimers(); if (destroyed || currentOperation !== operation || phase !== 'title') return; phase = 'playing'; notify(); @@ -109,12 +126,24 @@ export function createScreeningController({ }, titleDurationMs); } + async function playCurrent() { + if (destroyed) return; + clearTransitionTimers(); + const currentOperation = ++operation; + phase = 'title'; + notify(); + await prepare(index, active); + if (destroyed || currentOperation !== operation || phase !== 'title') return; + elements[active].currentTime = 0; + + const nextSlot = active === 0 ? 1 : 0; + void prepare(index + 1, nextSlot).catch(() => undefined); + beginTitleCountdown(currentOperation); + } + async function advance() { operation += 1; - if (titleTimer) { - clearTimeout(titleTimer); - titleTimer = null; - } + clearTransitionTimers(); elements[active].pause(); if (index >= playlist.length - 1) { phase = 'complete'; @@ -130,22 +159,40 @@ export function createScreeningController({ function fail(message: string) { operation += 1; - if (titleTimer) { - clearTimeout(titleTimer); - titleTimer = null; - } + clearTransitionTimers(); elements[active].pause(); phase = 'error'; notify(message); + const failedOperation = operation; + transitionTimer = setTimeout(() => { + transitionTimer = null; + if (destroyed || failedOperation !== operation || phase !== 'error') return; + void advance(); + }, errorDurationMs); + } + + async function playFrom(nextIndex: number) { + if (!Number.isInteger(nextIndex) || nextIndex < 0 || nextIndex >= playlist.length) + return; + operation += 1; + clearTransitionTimers(); + elements[active].pause(); + index = nextIndex; + active = (nextIndex % 2) as 0 | 1; + await audio.resume(); + await playCurrent().catch((error: unknown) => + fail(error instanceof Error ? error.message : 'playback could not start'), + ); } return { - async start() { + async start(startIndex = 0) { if (!playlist.length || phase !== 'idle') return; - await audio.resume(); - await playCurrent().catch((error: unknown) => - fail(error instanceof Error ? error.message : 'playback could not start'), - ); + await playFrom(startIndex); + }, + async jumpTo(nextIndex) { + if (!playlist.length) return; + await playFrom(nextIndex); }, async togglePause() { if (phase === 'playing') { @@ -172,7 +219,7 @@ export function createScreeningController({ operation += 1; slotOperations[0] += 1; slotOperations[1] += 1; - if (titleTimer) clearTimeout(titleTimer); + clearTransitionTimers(); elements.forEach((element, slot) => { element.pause(); element.removeEventListener('ended', endedHandlers[slot]); diff --git a/src/app/routes/AdminPage.tsx b/src/app/routes/AdminPage.tsx index 2473cb9..8d100d5 100644 --- a/src/app/routes/AdminPage.tsx +++ b/src/app/routes/AdminPage.tsx @@ -196,8 +196,8 @@ export function AdminPage() {

demo screening

project order

- choose the order projects will appear during the Hackweek screening. only - ready videos play in the reel. + choose which ready projects play first. any other ready videos follow in + project-name order.

preview ready reel diff --git a/src/app/routes/ProjectsPage.tsx b/src/app/routes/ProjectsPage.tsx index cc9e9da..efceb57 100644 --- a/src/app/routes/ProjectsPage.tsx +++ b/src/app/routes/ProjectsPage.tsx @@ -76,9 +76,11 @@ export function ProjectsPage({isAdmin = false}: {isAdmin?: boolean}) {

- - watch reel - + {(isAdmin || year.data.year.submissionsClosed) && ( + + watch reel + + )} {year.data.year.votingEnabled && ( vote diff --git a/src/app/routes/WatchPage.tsx b/src/app/routes/WatchPage.tsx index f5defa9..6a0b74f 100644 --- a/src/app/routes/WatchPage.tsx +++ b/src/app/routes/WatchPage.tsx @@ -1,14 +1,31 @@ +import {useCallback, useRef} from 'react'; import {useQuery} from '@tanstack/react-query'; -import {Link, useParams} from 'wouter'; +import {Link, useParams, useSearchParams} from 'wouter'; import {IndividualPlayer} from '../player/IndividualPlayer'; -import {ScreeningPlayer} from '../player/ScreeningPlayer'; +import {ScreeningPlayer, type ScreeningPlayerHandle} from '../player/ScreeningPlayer'; import {getPlayback, usePlaylist, useProjectVideo} from '../queries/videos'; import {PageState, QueryState} from '../components/AppLayout'; export function WatchPage() { const {yearId} = useParams<{yearId: string}>(); const playlist = usePlaylist(yearId); + const player = useRef(null); + const [search, setSearch] = useSearchParams(); + const initialVideoId = search.get('from'); + const trackActiveVideo = useCallback( + (videoId: string) => { + setSearch( + (current) => { + const next = new URLSearchParams(current); + next.set('from', videoId); + return next; + }, + {replace: true}, + ); + }, + [setSearch], + ); return ( {playlist.data && ( @@ -23,7 +40,13 @@ export function WatchPage() {

private progressive MP4 playback in the curated screening order.

- + {playlist.data.videos.length > 0 && (

on demand

@@ -32,10 +55,22 @@ export function WatchPage() { {playlist.data.videos.map((clip) => (
  • {String(clip.position + 1).padStart(2, '0')} - - {clip.projectName} - {formatDuration(clip.durationSeconds)} - +
    + + {clip.projectName} + + {[clip.groupName, formatDuration(clip.durationSeconds)] + .filter(Boolean) + .join(' · ')} + + + +
    • ))} diff --git a/src/app/styles.css b/src/app/styles.css index 467f7ef..7db8792 100644 --- a/src/app/styles.css +++ b/src/app/styles.css @@ -1871,13 +1871,18 @@ main { box-shadow: 0 22px 55px rgba(29, 17, 39, 0.22); } .screeningPlayer:fullscreen { - display: grid; - place-content: center; + width: 100vw; + height: 100vh; border: 0; border-radius: 0; } .screeningPlayer:fullscreen .screeningStage { width: 100vw; + height: 100vh; + aspect-ratio: auto; +} +.screeningPlayer:fullscreen .screeningControls { + display: none; } .screeningStage { position: relative; @@ -1964,10 +1969,26 @@ main { line-height: 0.95; letter-spacing: -0.065em; } +.titleCard strong, .titleCard span { + display: block; color: #ffd1ea; font-weight: 500; } +.titleCard strong { + margin-bottom: 0.25rem; + color: #fff; +} +.titleCountdown { + display: grid; + width: 2.4rem; + margin-top: 1rem; + aspect-ratio: 1; + place-items: center; + color: var(--ink); + border-radius: 50%; + background: var(--green); +} .startCard { z-index: 3; background: radial-gradient(circle at 50% 35%, #4e2a9a, #1d1127 62%); @@ -2093,10 +2114,20 @@ kbd { font-size: 1.3rem; font-weight: 700; } +.reelIndex li > div { + display: grid; + min-width: 0; + gap: 0.5rem; +} .reelIndex a { min-width: 0; text-decoration: none; } +.reelIndex .textAction { + width: fit-content; + padding: 0; + text-align: left; +} .reelIndex strong, .reelIndex small { display: block; diff --git a/src/shared/videos.ts b/src/shared/videos.ts index b859163..dd1e3ba 100644 --- a/src/shared/videos.ts +++ b/src/shared/videos.ts @@ -68,6 +68,7 @@ export interface PlaylistItem { videoId: string; projectId: string; projectName: string; + groupName: string | null; teamMembers: string[]; durationSeconds: number; gainDb: number; diff --git a/src/worker/routes/videos.ts b/src/worker/routes/videos.ts index ba08d14..1ddd0eb 100644 --- a/src/worker/routes/videos.ts +++ b/src/worker/routes/videos.ts @@ -33,7 +33,9 @@ videosRoutes.get('/playlist', async (c) => { try { const year = c.req.query('year'); if (!year) invalid('Year is required'); - const response: PlaylistResponse = {videos: await listPlaylist(c.env.DB, year)}; + const response: PlaylistResponse = { + videos: await listPlaylist(c.env.DB, year, c.get('user')), + }; return c.json(response, 200, {'Cache-Control': 'private, no-store'}); } catch (error) { return respondError(c, error); diff --git a/src/worker/services/videos.ts b/src/worker/services/videos.ts index 99c81c2..8c8006b 100644 --- a/src/worker/services/videos.ts +++ b/src/worker/services/videos.ts @@ -6,7 +6,11 @@ import type { VideoUploadPart, VideoUploadSession, } from '../../shared/videos'; -import {currentYearIdSql, effectiveYearFlags} from '../repositories/years'; +import { + currentYearIdSql, + effectiveYearFlags, + getEffectiveYearFlags, +} from '../repositories/years'; import type {VideoProcessingParams, VideoProcessorResult} from '../video-processing'; import {videoWorkflowInstanceId} from '../video-processing'; import {ServiceError} from './errors'; @@ -88,27 +92,41 @@ export async function getProjectVideo(db: D1Database, projectId: string) { export async function listPlaylist( db: D1Database, yearId: string, + user: SessionUser, ): Promise { + const year = await getEffectiveYearFlags(db, yearId); + if (!year) throw new ServiceError('NOT_FOUND', 'Year not found', 404); + if (!year.submissionsClosed && user.role !== 'admin') { + throw new ServiceError( + 'AUTH_FORBIDDEN', + 'The screening reel is available after submissions close', + 403, + ); + } + const {results} = await db .prepare( `SELECT pv.id video_id, p.id project_id, p.name project_name, - pv.duration_seconds, pv.gain_db, so.position - FROM screening_order so - JOIN projects p ON p.id = so.project_id AND p.status = 'active' + g.name group_name, pv.duration_seconds, pv.gain_db, so.position + FROM projects p JOIN video_submissions pv ON pv.project_id = p.id - WHERE so.year_id = ? AND pv.status = 'ready' AND pv.retired_at IS NULL + LEFT JOIN groups g ON g.id = p.group_id + LEFT JOIN screening_order so ON so.project_id = p.id AND so.year_id = p.year_id + WHERE p.year_id = ? AND p.status = 'active' AND p.kind = 'project' + AND pv.status = 'ready' AND pv.retired_at IS NULL AND pv.processed_r2_key IS NOT NULL AND pv.duration_seconds IS NOT NULL AND pv.gain_db IS NOT NULL - ORDER BY so.position, p.id`, + ORDER BY so.position IS NULL, so.position, p.name COLLATE NOCASE, p.id`, ) .bind(yearId) .all<{ video_id: string; project_id: string; project_name: string; + group_name: string | null; duration_seconds: number; gain_db: number; - position: number; + position: number | null; }>(); if (!results.length) return []; @@ -129,14 +147,15 @@ export async function listPlaylist( membersByProject.set(member.project_id, names); } - return results.map((row) => ({ + return results.map((row, position) => ({ videoId: row.video_id, projectId: row.project_id, projectName: row.project_name, + groupName: row.group_name, teamMembers: membersByProject.get(row.project_id) ?? [], durationSeconds: row.duration_seconds, gainDb: row.gain_db, - position: row.position, + position, })); } diff --git a/test/app/routes.test.tsx b/test/app/routes.test.tsx index ac62dce..9e2e9a4 100644 --- a/test/app/routes.test.tsx +++ b/test/app/routes.test.tsx @@ -164,7 +164,8 @@ describe('clickable project routes', () => { expect(within(archives).queryByText('2025')).toBeNull(); }); - it('links every year to its private R2 screening reel', async () => { + it('reveals the screening reel to admins or after submissions close', async () => { + let submissionsClosed = false; fetchMock.mockImplementation(async (input) => { const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; @@ -173,7 +174,7 @@ describe('clickable project routes', () => { year: { id: '2026', votingEnabled: false, - submissionsClosed: false, + submissionsClosed, projectCount: 0, ideaCount: 0, groupCount: 0, @@ -186,12 +187,26 @@ describe('clickable project routes', () => { return json({projects: [], nextCursor: null}); }); - renderRoute(, '/years/2026/projects', '/years/:yearId/projects'); - + const member = renderRoute( + , + '/years/2026/projects', + '/years/:yearId/projects', + ); expect(await screen.findByRole('heading', {name: 'projects & ideas'})).toBeTruthy(); - expect(screen.getByRole('link', {name: 'watch reel'}).getAttribute('href')).toBe( - '/years/2026/watch', + expect(screen.queryByRole('link', {name: 'watch reel'})).toBeNull(); + member.unmount(); + + const admin = renderRoute( + , + '/years/2026/projects', + '/years/:yearId/projects', ); + expect(await screen.findByRole('link', {name: 'watch reel'})).toBeTruthy(); + admin.unmount(); + + submissionsClosed = true; + renderRoute(, '/years/2026/projects', '/years/:yearId/projects'); + expect(await screen.findByRole('link', {name: 'watch reel'})).toBeTruthy(); }); it('defaults to the grid view when storage is unavailable', async () => { diff --git a/test/player/controller.test.tsx b/test/player/controller.test.tsx index 3a2415b..5caade3 100644 --- a/test/player/controller.test.tsx +++ b/test/player/controller.test.tsx @@ -34,7 +34,7 @@ describe('dual screening controller', () => { await controller.start(); await Promise.resolve(); - expect(states.at(-1)?.phase).toBe('title'); + expect(states.at(-1)).toMatchObject({phase: 'title', countdownSeconds: 1}); expect(attached).toEqual([ '/api/videos/video-1/content', '/api/videos/video-2/content', @@ -89,25 +89,33 @@ describe('dual screening controller', () => { expect(states.at(-1)?.phase).toBe('playing'); await controller.skip(); expect(states.at(-1)?.index).toBe(1); + await controller.jumpTo(0); + expect(states.at(-1)).toMatchObject({phase: 'title', index: 0}); const errors: PlayerState[] = []; const errorController = createScreeningController({ - playlist: [playlist[0]], + playlist, elements: [fakeVideo(), fakeVideo()], audio: fakeAudio(), - getPlayback: async () => { - throw new Error('private source unavailable'); + getPlayback: async (videoId) => { + if (videoId === 'video-1') throw new Error('private source unavailable'); + return { + source: {kind: 'mp4' as const, url: '/api/videos/video-2/content'}, + expiresAt: null, + }; }, + attach: () => ({destroy: vi.fn()}), onState: (state) => errors.push(state), titleDurationMs: 1, + errorDurationMs: 10, }); await errorController.start(); expect(errors.at(-1)).toMatchObject({ phase: 'error', error: 'private source unavailable', }); - await errorController.skip(); - expect(errors.at(-1)?.phase).toBe('complete'); + await vi.advanceTimersByTimeAsync(10); + expect(errors.at(-1)).toMatchObject({phase: 'title', index: 1}); }); }); @@ -134,6 +142,7 @@ const playlist: PlaylistItem[] = [ videoId: 'video-1', projectId: 'project-1', projectName: 'First', + groupName: 'Europe', teamMembers: ['Ada', 'Grace'], durationSeconds: 10, gainDb: 6, @@ -143,6 +152,7 @@ const playlist: PlaylistItem[] = [ videoId: 'video-2', projectId: 'project-2', projectName: 'Second', + groupName: 'Americas', teamMembers: ['Linus'], durationSeconds: 20, gainDb: -3, diff --git a/test/video-ui/video-ui.test.tsx b/test/video-ui/video-ui.test.tsx index 7858953..af9f3a6 100644 --- a/test/video-ui/video-ui.test.tsx +++ b/test/video-ui/video-ui.test.tsx @@ -218,14 +218,21 @@ describe('video user experience', () => { }); it('renders accessible empty reel and individual ready-video permalinks', async () => { - fetchMock.mockResolvedValue(json({videos: playlist})); - renderRoute(, '/years/2026/watch', '/years/:yearId/watch'); + fetchMock.mockImplementation(async () => json({videos: playlist})); + const reel = renderRoute(, '/years/2026/watch', '/years/:yearId/watch'); expect(await screen.findByRole('heading', {name: 'play the reel'})).toBeTruthy(); expect(screen.getByRole('button', {name: 'play all'})).toBeTruthy(); expect(screen.getByText('Ada Lovelace · Grace Hopper')).toBeTruthy(); expect(screen.getByRole('link', {name: /First project/}).getAttribute('href')).toBe( '/years/2026/watch/video-1', ); + expect(screen.getAllByRole('button', {name: 'play from here'})).toHaveLength(2); + reel.unmount(); + + renderRoute(, '/years/2026/watch?from=video-2', '/years/:yearId/watch'); + expect( + await screen.findByRole('button', {name: 'play from Second project'}), + ).toBeTruthy(); renderQuery(); expect(screen.getByRole('heading', {name: 'no videos are ready'})).toBeTruthy(); @@ -322,9 +329,20 @@ const playlist: PlaylistItem[] = [ videoId: 'video-1', projectId: 'project', projectName: 'First project', + groupName: 'Europe', teamMembers: ['Ada Lovelace', 'Grace Hopper'], durationSeconds: 30, gainDb: 0, position: 0, }, + { + videoId: 'video-2', + projectId: 'project-2', + projectName: 'Second project', + groupName: 'Americas', + teamMembers: ['Linus Torvalds'], + durationSeconds: 45, + gainDb: -1, + position: 1, + }, ]; diff --git a/test/video/video.test.ts b/test/video/video.test.ts index 342c3ef..bbc6ebb 100644 --- a/test/video/video.test.ts +++ b/test/video/video.test.ts @@ -650,11 +650,12 @@ describe('R2 multipart video lifecycle', () => { env.DB.prepare( 'INSERT INTO screening_order (year_id, project_id, position) VALUES (?, ?, 1)', ).bind(yearId, unreadyProject), - env.DB.prepare( - 'INSERT INTO screening_order (year_id, project_id, position) VALUES (?, ?, 2)', - ).bind(yearId, firstProject), + env.DB.prepare('UPDATE users SET is_admin = 1 WHERE id = ?').bind(ownerId), ]); + const hiddenWhileOpen = await api(`/videos/playlist?year=${yearId}`, memberToken); + expect(hiddenWhileOpen.status).toBe(403); + const playlist = await api(`/videos/playlist?year=${yearId}`, ownerToken); expect(playlist.status).toBe(200); expect(playlist.body.videos.map((item: {videoId: string}) => item.videoId)).toEqual([ @@ -663,12 +664,14 @@ describe('R2 multipart video lifecycle', () => { ]); expect(playlist.body.videos[0]).toMatchObject({ projectName: 'Curated second', + groupName: 'Video group', position: 0, teamMembers: ['Hackweek Member'], }); expect(playlist.body.videos[1]).toMatchObject({ projectName: 'Curated first', - position: 2, + groupName: 'Video group', + position: 1, teamMembers: ['Hackweek Member', 'Hackweek Member'], }); expect( @@ -693,6 +696,13 @@ describe('R2 multipart video lifecycle', () => { expect( afterRetirement.body.videos.map((item: {videoId: string}) => item.videoId), ).toEqual([first.video.id]); + + await env.DB.prepare('UPDATE years SET submissions_closed = 1 WHERE id = ?') + .bind(yearId) + .run(); + const memberArchive = await api(`/videos/playlist?year=${yearId}`, memberToken); + expect(memberArchive.status).toBe(200); + expect(memberArchive.body.videos).toHaveLength(1); }); it('limits local processing to one while independent projects remain queued', async () => { From 3bb7e10e0c4142a41641de2c68c3a931420f929b Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 19:02:18 +0200 Subject: [PATCH 12/18] refactor(reel): reuse project list rows Render reel entries with the same project row, group tag, and contributor stack used by the overview list, with the row title starting playback from that project. Remove the implementation-detail playback copy and the now-redundant reel-specific card styling. --- src/app/components/ProjectCard.tsx | 79 ++++++++++++++++++++++++++---- src/app/routes/WatchPage.tsx | 42 +++++++--------- src/app/styles.css | 65 +++++------------------- test/video-ui/video-ui.test.tsx | 17 +++++-- 4 files changed, 113 insertions(+), 90 deletions(-) diff --git a/src/app/components/ProjectCard.tsx b/src/app/components/ProjectCard.tsx index 5d3dd06..37c4b96 100644 --- a/src/app/components/ProjectCard.tsx +++ b/src/app/components/ProjectCard.tsx @@ -3,6 +3,11 @@ import {Link} from 'wouter'; import type {ProjectSummary} from '../../shared/projects'; import {Markdown} from './Markdown'; +interface ProjectListMember { + id: string; + displayName: string; +} + export function ProjectCard({ project, view = 'grid', @@ -14,13 +19,14 @@ export function ProjectCard({ if (view === 'list') { return ( -
    -

    - {project.name} -

    - - -
    + ); } @@ -38,6 +44,55 @@ export function ProjectCard({ ); } +export function ProjectListItem({ + name, + href, + onSelect, + actionLabel, + kind = 'project', + groupName, + detail, + members, + needsHelp = false, + emptyMemberLabel = 'up for grabs', +}: { + name: string; + href?: string; + onSelect?: () => void; + actionLabel?: string; + kind?: ProjectSummary['kind']; + groupName: string; + detail?: string; + members: ProjectListMember[]; + needsHelp?: boolean; + emptyMemberLabel?: string; +}) { + return ( +
    +

    + {href ? ( + {name} + ) : ( + + )} +

    +
    + {groupName} + {detail && {detail}} + {needsHelp && looking for help} +
    + +
    + ); +} + function ProjectTags({project, className}: {project: ProjectSummary; className: string}) { return (
    @@ -49,8 +104,14 @@ function ProjectTags({project, className}: {project: ProjectSummary; className: ); } -function MemberStack({members}: {members: ProjectSummary['members']}) { - if (!members.length) return up for grabs; +function MemberStack({ + members, + emptyLabel = 'up for grabs', +}: { + members: ProjectListMember[]; + emptyLabel?: string; +}) { + if (!members.length) return {emptyLabel}; return ( Hackweek {yearId} / screening

    play the reel

    -

    private progressive MP4 playback in the curated screening order.

    {playlist.data.videos.length > 0 && (
    -

    on demand

    -

    watch one project

    -
      +

      screening order

      +

      playlist

      +
      {playlist.data.videos.map((clip) => ( -
    1. - {String(clip.position + 1).padStart(2, '0')} -
      - - {clip.projectName} - - {[clip.groupName, formatDuration(clip.durationSeconds)] - .filter(Boolean) - .join(' · ')} - - - -
      -
      2. + ({ + id: `${clip.videoId}:${index}`, + displayName, + }))} + emptyMemberLabel="Hackweek team" + actionLabel={`start reel from ${clip.projectName}`} + onSelect={() => player.current?.playFrom(clip.videoId)} + /> ))} -
    +
    )} diff --git a/src/app/styles.css b/src/app/styles.css index 7db8792..7666d1c 100644 --- a/src/app/styles.css +++ b/src/app/styles.css @@ -723,12 +723,22 @@ main { line-height: 1.25; letter-spacing: -0.015em; } -.projectRow h2 a { +.projectRow h2 a, +.projectRowTitle { display: block; + width: 100%; overflow: hidden; + padding: 0; + color: inherit; + text-align: left; text-decoration: none; text-overflow: ellipsis; white-space: nowrap; + border: 0; + background: none; +} +.projectRowTitle:hover { + color: var(--blurple); } .projectRowTags { display: flex; @@ -2093,51 +2103,8 @@ kbd { .reelIndex { padding-top: 4rem; } -.reelIndex ol { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 0.75rem; - padding: 0; - list-style: none; -} -.reelIndex li { - display: flex; - gap: 1rem; - align-items: center; - min-width: 0; - padding: 1rem; - border: 1px solid var(--line); - border-radius: 0.65rem; -} -.reelIndex li > span { - color: var(--blurple); - font-size: 1.3rem; - font-weight: 700; -} -.reelIndex li > div { - display: grid; - min-width: 0; - gap: 0.5rem; -} -.reelIndex a { - min-width: 0; - text-decoration: none; -} -.reelIndex .textAction { - width: fit-content; - padding: 0; - text-align: left; -} -.reelIndex strong, -.reelIndex small { - display: block; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} -.reelIndex small { - margin-top: 0.25rem; - color: var(--muted); +.reelPlaylist { + margin-top: 1rem; } .individualPlayer video { display: block; @@ -2184,9 +2151,6 @@ kbd { .screeningControls kbd { display: none; } - .reelIndex ol { - grid-template-columns: 1fr 1fr; - } } @media (max-width: 560px) { .videoPanel > header { @@ -2199,9 +2163,6 @@ kbd { .screeningControls button:last-child { grid-column: 1 / -1; } - .reelIndex ol { - grid-template-columns: 1fr; - } .titleCard h2, .startCard h2 { font-size: clamp(1.8rem, 10vw, 3rem); diff --git a/test/video-ui/video-ui.test.tsx b/test/video-ui/video-ui.test.tsx index af9f3a6..f278a7d 100644 --- a/test/video-ui/video-ui.test.tsx +++ b/test/video-ui/video-ui.test.tsx @@ -217,16 +217,23 @@ describe('video user experience', () => { pause.mockRestore(); }); - it('renders accessible empty reel and individual ready-video permalinks', async () => { + it('renders the reel playlist with shared project rows and resume controls', async () => { fetchMock.mockImplementation(async () => json({videos: playlist})); const reel = renderRoute(, '/years/2026/watch', '/years/:yearId/watch'); expect(await screen.findByRole('heading', {name: 'play the reel'})).toBeTruthy(); expect(screen.getByRole('button', {name: 'play all'})).toBeTruthy(); + expect(screen.getByRole('heading', {name: 'playlist'})).toBeTruthy(); expect(screen.getByText('Ada Lovelace · Grace Hopper')).toBeTruthy(); - expect(screen.getByRole('link', {name: /First project/}).getAttribute('href')).toBe( - '/years/2026/watch/video-1', - ); - expect(screen.getAllByRole('button', {name: 'play from here'})).toHaveLength(2); + expect( + screen + .getByRole('button', {name: 'start reel from First project'}) + .closest('.projectRow'), + ).not.toBeNull(); + expect( + screen.queryByText( + 'private progressive MP4 playback in the curated screening order.', + ), + ).toBeNull(); reel.unmount(); renderRoute(, '/years/2026/watch?from=video-2', '/years/:yearId/watch'); From a59737eb10c2792450f0319dd6e226f736cd5c48 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 20:46:09 +0200 Subject: [PATCH 13/18] chore(video): remove standalone test tooling Drop the benchmark, processor fixture runner, local readiness journey, and rollout E2E script from the pull request. Simplify the verification command and documentation accordingly, and remove the rollout runbook from the repository. --- README.md | 21 +- VIDEO_ROLLOUT.md | 165 ------ package.json | 6 +- scripts/benchmark-video-processor.ts | 135 ----- scripts/local-readiness.ts | 723 --------------------------- scripts/test-video-processor.ts | 302 ----------- test/e2e/video-rollout.test.ts | 93 ---- 7 files changed, 2 insertions(+), 1443 deletions(-) delete mode 100644 VIDEO_ROLLOUT.md delete mode 100644 scripts/benchmark-video-processor.ts delete mode 100644 scripts/local-readiness.ts delete mode 100644 scripts/test-video-processor.ts delete mode 100644 test/e2e/video-rollout.test.ts diff --git a/README.md b/README.md index 475764e..29d84c9 100644 --- a/README.md +++ b/README.md @@ -52,23 +52,6 @@ Never run that command with `--remote`. - **Stale local data:** stop the app and remove only `.wrangler/state`, then repeat the local migrations. This never touches remote resources. - **Playback fails:** confirm the video is ready and signed-in playback returns `200` or `206`; unready, retired, and anonymous reads are intentionally rejected. -## Automated real-byte readiness - -```bash -npm run test:video-processor -npm run test:readiness -``` - -`test:video-processor` builds the pinned image and covers loud, silent, rotated, low-resolution, malformed, over-duration, and forced-failure fixtures generated at runtime. `test:readiness` creates isolated temporary D1/R2 state, preserves the developer’s `.dev.vars`, uploads generated MP4 bytes through multipart R2, resumes the upload, runs the local Workflow and Container, probes the canonical output, verifies authenticated full/range playback and curated playlist inclusion, retires the submission, proves retained objects, then removes its processes and state. - -For a manual browser companion pass, run `npm run dev:video` and check: - -1. pause/reload during upload and resume from the recorded part; -2. queued → processing → ready status; -3. project playback seek (range delivery); -4. curated reel overlay, pause, skip, fullscreen, and advance; -5. retirement removes playback/reel visibility without deleting stored bytes. - ## Authentication Google OAuth is the only browser authentication path. It uses Authorization Code with PKCE, state and nonce validation, Google JWKS verification, exact verified `@sentry.io` enforcement, hashed opaque D1 sessions, and HttpOnly cookies. D1 is the sole role authority. Authenticated mutations require the exact same-origin `Origin` header. @@ -80,6 +63,4 @@ npm run verify npm audit --omit=dev --audit-level=high ``` -The deterministic gate generates binding types, typechecks, checks formatting/lint, runs Worker/frontend/migration/player tests, builds, performs a credential-free production dry run, builds and exercises the real pinned processor, and runs the isolated real-byte local E2E. It does not deploy, provision, access remote resources, or prove real Google OAuth. - -Production resource names, benchmark evidence, observability, rollout, smoke, rollback, and retained-storage policy are documented in [`VIDEO_ROLLOUT.md`](VIDEO_ROLLOUT.md). Every production mutation remains behind explicit later human approval. +The gate generates binding types, typechecks, checks formatting/lint, runs the standard test suites, builds, and performs a credential-free production dry run. It does not deploy, provision, access remote resources, or prove real Google OAuth. diff --git a/VIDEO_ROLLOUT.md b/VIDEO_ROLLOUT.md deleted file mode 100644 index a2de46d..0000000 --- a/VIDEO_ROLLOUT.md +++ /dev/null @@ -1,165 +0,0 @@ -# Video processing production rollout - -This runbook prepares the R2 + Workflow + Container path. It does not authorize or perform any Cloudflare production mutation. - -## Prepared resource contract - -The production declaration in `wrangler.production.json` uses these isolated future video resources: - -| Binding | Proposed resource | Purpose | -| --------------------------- | -------------------------------------------------------- | -------------------------------------------------------------- | -| `VIDEOS` | R2 bucket `hackweek-video-media-production` | Private immutable originals and canonical derivatives | -| `VIDEO_PROCESSING_WORKFLOW` | Workflow `hackweek-video-processing-production` | One durable instance per video attempt | -| `VIDEO_PROCESSOR` | Container Durable Object class `VideoProcessorContainer` | Attempt-isolated FFmpeg invocation | -| Container application | `hackweek-video-processor-production` | Digest-pinned `Dockerfile.video-processor` image | -| `DB` | Existing `hackweek-db` binding | Upload, attempt, fencing, state, and retained-object inventory | - -Migration 0007 is an expand-only transition. The R2 Worker uses `video_submissions`, `video_uploads`, `video_upload_parts`, and `video_processing_attempts`; the legacy `project_videos` columns and `stream_events` table remain unchanged solely so the recorded pre-release Worker can run before deployment or after rollback. The R2 Worker has no active Stream path. - -Production declares both `max_instances: 2` and `VIDEO_PROCESSOR_CONCURRENCY=2`. Keep the two values equal. Local development uses one. The Container has no Internet access and receives no R2 account credential; its outbound handler scopes source/output access to the current D1 attempt. - -Required non-secret variables are `APP_ORIGIN`, `GOOGLE_REDIRECT_URI`, `GOOGLE_CLIENT_ID`, `ALLOWED_EMAIL_DOMAIN`, `VIDEO_PROCESSOR_CONCURRENCY`, and `VIDEO_PROCESSING_AUTOSTART`. `GOOGLE_CLIENT_SECRET` remains the required Worker secret. There is no Stream, HLS, public-R2, service-token, Queue, or general R2 credential binding. - -## Explicit approval boundary - -A human production owner must approve the exact commit, account, names, expected storage growth, benchmark/tuning decision, D1 backup window, smoke/rollback operators, and recorded pre-release Worker version before any command below that uses `--remote`, `r2 bucket create`, `secret put`, `deploy`, or `rollback` is run. The release record must include the passing populated-legacy expand/old-query/new-query/FK migration test for that exact commit. - -Until that approval, only these non-mutating local checks are allowed: - -```bash -npm ci -npm run verify -npm audit --omit=dev --audit-level=high -npm run video:benchmark -npm run build -npm run deploy:dry-run -``` - -`deploy:dry-run` compiles the Worker, validates bindings, and builds the Container locally. It does not upload, create, list, or mutate a Cloudflare resource. - -## Local benchmark evidence - -Command: `npm run video:benchmark` - -Environment recorded 2026-08-11: OrbStack Linux ARM64 Docker engine 29.4.0 on an ARM64 development host; digest-pinned FFmpeg 8.0.1 image. CPU is the delta of cgroup `usage_usec` for the running Container; wall time wraps the `process-file` invocation. Fixtures are generated at runtime with synthetic motion plus audible 48 kHz audio. They are deliberately short and bounded, so these numbers validate profiles and provide a tuning baseline—not a production cost, throughput, or ten-minute latency promise. - -| Profile | Source | Input | Output | Wall | Container CPU | Output loudness | -| ---------------- | ---------------------- | ----------: | ----------: | ------: | ------------: | --------------: | -| correctness-360p | 640×360, 2 s, 24 fps | 435,740 B | 263,556 B | 0.805 s | 1.052 s | -15.95 LUFS | -| bounded-720p | 1280×720, 4 s, 24 fps | 3,252,808 B | 1,829,962 B | 0.657 s | 2.181 s | -15.96 LUFS | -| bounded-1080p | 1920×1080, 4 s, 24 fps | 7,263,137 B | 3,620,283 B | 1.008 s | 4.464 s | -15.96 LUFS | - -Before production approval, repeat the benchmark on the release commit and run a separately approved bounded staging sample representative of expected durations. Start with concurrency two only if p95 processing wall time, Container CPU/memory, scratch disk, Workflow retries, and queued wait remain within the agreed event window. Reduce both concurrency declarations together if account/container pressure appears; increasing beyond two requires a new review and benchmark. - -## Later provisioning order (mutating; approval required) - -The following is an operator checklist, not deployment automation. Stop if the configured Cloudflare account is not `773afa1f62ff86c80db4f24f7ff1e9c8` or any proposed resource name is already owned for another purpose. - -1. Record the release commit and current pre-release Worker version ID for rollback. Confirm it is the version covered by the legacy `project_videos`/`stream_events` SQL contract in the migration test. Obtain an explicit go/no-go from the production owner. -2. Run all local checks from the approval section and archive their output with the release record, including `npm run test:migration -- test/migration/migration.test.ts test/e2e/video-rollout.test.ts`. -3. Create the isolated private video bucket: - - ```bash - npx wrangler r2 bucket create hackweek-video-media-production \ - --config wrangler.production.json - ``` - -4. Set the existing Worker’s Google OAuth secret if it is not already present. The command prompts securely; never place the value in shell history: - - ```bash - npx wrangler secret put GOOGLE_CLIENT_SECRET \ - --config wrangler.production.json - ``` - -5. Apply reviewed D1 migrations during the approved backup window: - - ```bash - npx wrangler d1 migrations apply hackweek-db \ - --remote --config wrangler.production.json - ``` - - Migration 0007 only expands the schema. The still-deployed pre-release Worker continues to read and write its unchanged `project_videos` columns and `stream_events`; the new R2 tables can be queried independently. If migration application or the following deploy fails, leave the pre-release Worker serving and investigate—do not attempt a destructive schema reversal. - -6. Deploy the reviewed declaration: - - ```bash - npx wrangler deploy --config wrangler.production.json \ - --containers-rollout gradual - ``` - - Wrangler materializes the declared `hackweek-video-processing-production` Workflow, `hackweek-video-processor-production` Container application/image, and `VideoProcessorContainer` Durable Object migration as part of this approved deploy. Do not create similarly named resources by hand. - -7. Perform the smoke checklist below with one small generated/approved non-sensitive clip before allowing event uploads. - -The existing manual GitHub deployment workflow remains an alternative controlled entry point only after its environment approval and typed confirmation. Do not run both paths for one release. - -## Production smoke criteria - -Use a test project and authenticated creator/member/admin accounts. The release is healthy only when all checks pass: - -1. Anonymous create, playback descriptor, and content requests return `401`; a non-member upload returns `403`. -2. Multipart create → part upload → refresh/resume → completion succeeds; duplicate completion returns the same video/attempt. -3. State advances queued → processing → ready, and Workflow instance `video--attempt-1` completes all named steps. -4. D1 records distinct original/processed keys, duration ≤600 s, and loudness within -16 ±0.7 LUFS; the downloaded derivative probes as H.264/AAC `yuv420p`. -5. Authenticated content returns `200`, a seek returns exact `206`/`Content-Range`, and an unsatisfiable range returns `416`. -6. Only the ready video appears in saved screening order with the correct project/team overlay. Pause, skip, fullscreen, ended advance, and recoverable error advance work in the event browser. -7. Confirmed retirement removes project/reel playback while both R2 objects remain present. Do not delete the smoke objects. -8. Two concurrent independent project jobs can run; a third waits/retries. A same-project second active upload conflicts. - -Useful read-only diagnostics after deployment: - -```bash -npx wrangler tail hackweek --config wrangler.production.json --format json -npx wrangler workflows instances describe \ - hackweek-video-processing-production video--attempt- \ - --config wrangler.production.json -``` - -## Observability and alerts - -`observability.enabled` is set in production. Workflow logs emit JSON with `component=video-processing`, an event name, `videoId`, attempt, and bounded failure text; they never include object keys, media bytes, cookies, OAuth values, or R2 credentials. - -Create dashboard/alert ownership before rollout for: - -- Workflow failed/terminated instance count >0 over 5 minutes; -- `processor_failed` or `claim_failed` events >0, grouped by attempt and bounded error; -- oldest queued/running D1 attempt >10 minutes; -- queued depth above 2 for 10 minutes (capacity pressure at cap two); -- Worker `/api/projects/*/video*` and `/api/videos/*/content` 5xx rate >1% over 5 minutes; -- Container CPU, memory, scratch disk, restart, and timeout pressure; -- `video_submissions.status='failed'` growth and retries per video; -- R2 object count/bytes and monthly growth for `hackweek-video-media-production`. - -Never place full request headers, session cookies, source/output object keys, or media payloads in an alert. Link alerts to this runbook and name an event-time operator. - -## Rollback - -Rollback is state-preserving. Do not delete R2 objects, Workflow instances, D1 rows, or the Container application during incident response. - -1. Pause new R2 video completion/processing by preparing `VIDEO_PROCESSING_AUTOSTART=false` on a reviewed incident commit when the Worker is healthy enough to deploy that change. New completed R2 uploads remain queued rather than being published incorrectly. -2. If the Worker release itself is faulty, roll back to the recorded pre-release Worker version that was captured and compatibility-tested before rollout: - - ```bash - npx wrangler rollback \ - --name hackweek --message "Rollback video rollout: " - ``` - - This target remains schema-compatible after 0007 because the migration does not alter `project_videos` or `stream_events`. Do not select an older unrecorded version. The rollback version may use its original Stream lifecycle; no Stream behavior is present in the new R2 Worker. - -3. Do not reverse migration 0007. Leave legacy rows/events, R2 queued/running/failed attempt rows, and all original/derivative objects intact. A late R2 result cannot publish over a retired or newer attempt. -4. Restore the R2 release only after `npm run verify`, the production smoke subset, reconciliation of any legacy writes made during rollback, and incident-owner approval pass on the corrective release. Re-enable autostart and keep concurrency at or below two. -5. Reconcile status and inventory; do not manually mark a video ready and do not copy an unprobed object into a canonical key. - -D1 migration 0007 is a forward-only expansion. Worker rollback does not reverse it, and no database downgrade is required. - -## Future contraction (separate approval required) - -Do not drop, rename, or repurpose `project_videos`, its legacy columns/indexes, or `stream_events` in this release. They define the tested pre-release rollback contract. - -A later contraction requires a separate production-owner approval and release after this rollback target is retired. Before contraction, inventory and reconcile legacy rows/events—including writes made during any rollback—record a new R2-compatible rollback target, prove no deployed Worker queries the legacy schema, and take the approved D1 backup. Only then may a new migration remove the legacy tables. Renaming `video_submissions` is not part of 0007; if desired, it requires its own expand/deploy/contract sequence rather than an in-place destructive rename. - -## Retained-storage policy - -No automatic deletion is permitted for completed originals or derivatives, including retired submissions and stale completed derivatives. Only incomplete expired multipart uploads may be aborted. The retained bytes support recovery and later delivery changes, but storage growth is an accepted operational risk. - -Track per-video original/derivative keys and sizes in the monthly inventory, alert on growth, and review retention with the data owner after the event. Any future deletion policy requires separate human approval, an inventory/export plan, and a new implementation; it is not part of this rollout. diff --git a/package.json b/package.json index e6fe791..e50b9a6 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,7 @@ "test:worker": "node scripts/run-worker-tests.mjs", "test:app": "vp test run --config vitest.app.config.ts", "test:migration": "vp test run --config vitest.migration.config.ts", - "test:readiness": "tsx scripts/local-readiness.ts", - "verify": "npm run cf-typegen && npm run typecheck && npm run format:check && npm run lint && npm test && npm run build && npm run deploy:dry-run && npm run test:video-processor && npm run test:readiness", + "verify": "npm run cf-typegen && npm run typecheck && npm run format:check && npm run lint && npm test && npm run build && npm run deploy:dry-run", "deploy:dry-run": "wrangler deploy --dry-run --config wrangler.production.json --outdir .wrangler/deploy-dry-run", "build": "vp build", "preview": "vp preview", @@ -22,9 +21,6 @@ "db:migrate:local": "wrangler d1 migrations apply hackweek-db --local", "dev:video": "npm run db:migrate:local && vp dev", "video:processor:build": "docker build --file Dockerfile.video-processor --tag hackweek-video-processor:local .", - "video:benchmark": "tsx scripts/benchmark-video-processor.ts", - "test:video-processor": "tsx scripts/test-video-processor.ts", - "test:video-workflow": "npm run test:readiness", "migrate:validate": "tsx scripts/migrate/cli.ts validate", "migrate:dry-run": "tsx scripts/migrate/cli.ts dry-run", "migrate:local": "tsx scripts/migrate/cli.ts import --target local", diff --git a/scripts/benchmark-video-processor.ts b/scripts/benchmark-video-processor.ts deleted file mode 100644 index 387ccd6..0000000 --- a/scripts/benchmark-video-processor.ts +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env node -import {execFileSync, spawnSync} from 'node:child_process'; -import {chmod, mkdtemp, rm, stat} from 'node:fs/promises'; -import {tmpdir} from 'node:os'; -import path from 'node:path'; - -const root = process.cwd(); -const image = 'hackweek-video-processor:local'; -const work = await mkdtemp(path.join(tmpdir(), 'hackweek-video-benchmark-')); -const container = `hackweek-video-benchmark-${process.pid}`; -const profiles = [ - {name: 'correctness-360p', width: 640, height: 360, duration: 2}, - {name: 'bounded-720p', width: 1280, height: 720, duration: 4}, - {name: 'bounded-1080p', width: 1920, height: 1080, duration: 4}, -]; -let containerStarted = false; - -try { - run('docker', ['build', '--file', 'Dockerfile.video-processor', '--tag', image, '.']); - await chmod(work, 0o777); - for (const profile of profiles) { - ffmpeg([ - '-f', - 'lavfi', - '-i', - `testsrc2=size=${profile.width}x${profile.height}:rate=24`, - '-f', - 'lavfi', - '-i', - 'sine=frequency=440:sample_rate=48000', - '-t', - String(profile.duration), - '-af', - 'volume=0.05', - '-c:v', - 'libx264', - '-preset', - 'ultrafast', - '-c:a', - 'aac', - '-shortest', - path.join(work, `${profile.name}-input.mp4`), - ]); - } - await Promise.all( - profiles.map((profile) => chmod(path.join(work, `${profile.name}-input.mp4`), 0o644)), - ); - - run('docker', [ - 'run', - '--detach', - '--rm', - '--name', - container, - '--volume', - `${work}:/work`, - image, - ]); - containerStarted = true; - const results = []; - for (const profile of profiles) { - const input = `/work/${profile.name}-input.mp4`; - const output = `/work/${profile.name}-output.mp4`; - const cpuBefore = containerCpuUsec(); - const wallStarted = performance.now(); - const metadata = JSON.parse( - outputOf('docker', [ - 'exec', - container, - 'node', - '/app/video-processor.mjs', - 'process-file', - input, - output, - ]) - .trim() - .split('\n') - .at(-1)!, - ) as {width: number; height: number; loudnessLufs: number | null}; - const wallSeconds = (performance.now() - wallStarted) / 1000; - const cpuSeconds = (containerCpuUsec() - cpuBefore) / 1_000_000; - const inputBytes = (await stat(path.join(work, `${profile.name}-input.mp4`))).size; - const outputBytes = (await stat(path.join(work, `${profile.name}-output.mp4`))).size; - results.push({ - profile: profile.name, - source: `${profile.width}x${profile.height}, ${profile.duration}s, 24fps`, - inputBytes, - outputBytes, - wallSeconds: Number(wallSeconds.toFixed(3)), - cpuSeconds: Number(cpuSeconds.toFixed(3)), - outputResolution: `${metadata.width}x${metadata.height}`, - loudnessLufs: metadata.loudnessLufs, - }); - } - - console.table(results); - console.log(JSON.stringify({image, profiles: results}, null, 2)); -} finally { - if (containerStarted) { - const removed = spawnSync('docker', ['rm', '--force', container], { - cwd: root, - stdio: 'ignore', - }); - if (removed.status !== 0) { - console.error(`Could not remove benchmark container ${container}`); - process.exitCode = 1; - } - } - await rm(work, {recursive: true, force: true}); -} - -function containerCpuUsec() { - const stats = outputOf('docker', [ - 'exec', - container, - 'sh', - '-c', - "awk '/^usage_usec / {print $2}' /sys/fs/cgroup/cpu.stat", - ]).trim(); - const value = Number(stats); - if (!Number.isFinite(value)) throw new Error(`Invalid container CPU usage: ${stats}`); - return value; -} - -function ffmpeg(args: string[]) { - run('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-nostdin', '-y', ...args]); -} - -function run(command: string, args: string[]) { - execFileSync(command, args, {cwd: root, stdio: 'inherit'}); -} - -function outputOf(command: string, args: string[]) { - return execFileSync(command, args, {cwd: root, encoding: 'utf8'}); -} diff --git a/scripts/local-readiness.ts b/scripts/local-readiness.ts deleted file mode 100644 index c9eb004..0000000 --- a/scripts/local-readiness.ts +++ /dev/null @@ -1,723 +0,0 @@ -#!/usr/bin/env node -import {execFileSync, spawn, spawnSync, type ChildProcess} from 'node:child_process'; -import {createHash} from 'node:crypto'; -import {mkdtemp, readFile, rm, stat, writeFile} from 'node:fs/promises'; -import {tmpdir} from 'node:os'; -import path from 'node:path'; - -const root = process.cwd(); -const fixture = path.join(root, 'test/fixtures/firebase'); -const state = await mkdtemp(path.join(tmpdir(), 'hackweek-readiness-')); -const port = Number(process.env.READINESS_PORT ?? 5199); -const origin = `http://127.0.0.1:${port}`; -const config = path.join(state, 'wrangler.readiness.json'); -const source = path.join(state, 'readiness-source.mp4'); -const original = path.join(state, 'readiness-original.mp4'); -const derivative = path.join(state, 'readiness-derivative.mp4'); -const googleClientId = 'local-readiness.apps.googleusercontent.com'; -const googleClientSecret = 'synthetic-readiness-value'; -const sessionToken = createHash('sha256') - .update('hackweek-local-readiness') - .digest('base64url'); -const sessionTokenHash = createHash('sha256').update(sessionToken).digest('hex'); -const rootDevVars = path.join(root, '.dev.vars'); -const devVarsBefore = await optionalFile(rootDevVars); -const dockerContainersBefore = new Set(dockerContainerNames()); -let server: ChildProcess | undefined; -const serverLog: string[] = []; - -try { - await writeFile(config, JSON.stringify(localConfig()), {mode: 0o600}); - await writeFile(path.join(state, '.dev.vars'), localDevVars(), {mode: 0o600}); - - run('npx', [ - 'wrangler', - 'd1', - 'migrations', - 'apply', - 'hackweek-db', - '--local', - '--persist-to', - state, - '--config', - config, - ]); - run('npm', [ - 'run', - 'migrate:local', - '--', - '--database', - path.join(fixture, 'database.json'), - '--storage-manifest', - path.join(fixture, 'storage-manifest.json'), - '--storage-root', - path.join(fixture, 'storage'), - '--bucket-name', - 'hackweek-attachments-readiness', - '--config', - config, - '--persist-to', - state, - ]); - run('npm', [ - 'run', - 'migrate:reconcile', - '--', - '--source', - path.join(fixture, 'database.json'), - '--storage-manifest', - path.join(fixture, 'storage-manifest.json'), - '--storage-root', - path.join(fixture, 'storage'), - '--target', - 'local', - '--bucket-name', - 'hackweek-attachments-readiness', - '--config', - config, - '--persist-to', - state, - ]); - - const now = Math.floor(Date.now() / 1000); - sql(` - INSERT INTO users - (id, source_uid, google_subject, email, display_name, avatar_url, is_admin) - VALUES - ('readiness-user', 'readiness-user', 'google-readiness-user', - 'developer@sentry.io', 'Local Developer', NULL, 1); - INSERT INTO user_sessions - (token_hash, user_id, expires_at, created_at, last_used_at) - VALUES ('${sessionTokenHash}', 'readiness-user', ${now + 28_800}, ${now}, ${now}); - INSERT INTO years (id) VALUES ('9999'); - INSERT INTO groups (id, source_id, year_id, name, creator_id) - VALUES ('readiness-group', 'readiness-group', '9999', 'Readiness Team', 'readiness-user'); - INSERT INTO projects - (id, source_id, year_id, creator_id, group_id, name, summary, kind) - VALUES - ('readiness-project', 'readiness-project', '9999', 'readiness-user', - 'readiness-group', 'Readiness Video', 'Real local video E2E', 'project'); - INSERT INTO project_members (project_id, user_id) - VALUES ('readiness-project', 'readiness-user'); - `); - - ffmpeg([ - '-f', - 'lavfi', - '-i', - 'testsrc2=size=640x360:rate=24', - '-f', - 'lavfi', - '-i', - 'sine=frequency=440:sample_rate=48000', - '-t', - '2', - '-af', - 'volume=0.05', - '-c:v', - 'libx264', - '-preset', - 'ultrafast', - '-c:a', - 'aac', - '-shortest', - source, - ]); - - server = spawn( - process.execPath, - [ - path.join(root, 'node_modules/vite-plus/bin/vp'), - 'dev', - '--host', - '127.0.0.1', - '--port', - String(port), - '--strictPort', - ], - { - cwd: root, - detached: true, - env: localEnvironment(), - stdio: ['ignore', 'pipe', 'pipe'], - }, - ); - server.stdout?.on('data', (chunk) => serverLog.push(String(chunk))); - server.stderr?.on('data', (chunk) => serverLog.push(String(chunk))); - await waitForServer(); - - const login = await request('/api/auth/login', {redirect: 'manual'}); - const authorization = new URL(login.headers.get('Location')!); - assert( - authorization.searchParams.get('client_id') === googleClientId, - 'loopback Google OAuth configuration is active', - ); - const session = await get('/api/session'); - assert( - session.user.role === 'admin' && session.user.email === 'developer@sentry.io', - 'seeded D1 session authenticates the readiness administrator', - ); - const unauthorized = await fetch(`${origin}/api/projects/readiness-project/video`); - assert(unauthorized.status === 401, 'video APIs reject unauthenticated requests'); - - const projects = await get('/api/projects?year=2024&limit=50'); - assert( - projects.projects.some( - (project: {name: string}) => project.name === 'Historical Telescope', - ), - 'isolated D1 contains the migrated archive fixture', - ); - const historical = await get('/api/projects/project-history'); - const media = await request(`/api/media/${historical.project.media[0].id}/content`); - assert( - media.status === 200 && (await media.text()).includes('Synthetic'), - 'isolated attachment R2 contains reconciled fixture bytes', - ); - - await sendJson('PUT', '/api/admin/years/9999/screening-order', { - projectIds: ['readiness-project'], - }); - const bytes = await readFile(source); - const created = await sendJson( - 'POST', - '/api/projects/readiness-project/video/upload', - { - fileName: 'readiness-source.mp4', - fileSize: bytes.byteLength, - contentType: 'video/mp4', - }, - 201, - ); - assert( - created.upload.status === 'uploading' && created.upload.completedParts.length === 0, - 'real local R2 multipart upload is created', - ); - const uploadId = created.upload.uploadId as string; - const interrupted = await get( - `/api/projects/readiness-project/video/upload/${uploadId}`, - ); - assert( - interrupted.upload.completedParts.length === 0, - 'an interrupted upload resumes from durable server state', - ); - - const partResponse = await request( - `/api/projects/readiness-project/video/upload/${uploadId}/parts/1`, - { - method: 'PUT', - headers: { - Origin: origin, - 'Content-Type': 'application/octet-stream', - 'Content-Length': String(bytes.byteLength), - }, - body: bytes, - }, - ); - const partBody = await responseJson(partResponse); - assert(partResponse.status === 200, 'generated media bytes stream into multipart R2'); - const resumed = await get(`/api/projects/readiness-project/video/upload/${uploadId}`); - assert( - resumed.upload.completedParts[0]?.etag === partBody.part.etag, - 'uploaded part ETag survives a resume lookup', - ); - - const completionPath = `/api/projects/readiness-project/video/upload/${uploadId}/complete`; - const completionInput = { - parts: [{partNumber: 1, etag: partBody.part.etag}], - }; - const completed = await sendJson('POST', completionPath, completionInput); - assert(completed.video.status === 'queued', 'multipart completion queues processing'); - const videoId = completed.video.id as string; - const duplicate = await sendJson('POST', completionPath, completionInput); - assert( - duplicate.video.id === videoId && duplicate.video.processingAttempt === 1, - 'duplicate completion reuses the fenced Workflow attempt', - ); - - const ready = await waitForReady(); - assert(ready.status === 'ready', 'local Workflow conditionally publishes ready state'); - assert( - Math.abs(ready.loudnessLufs + 16) <= 0.7, - 'ready metadata records normalized loudness within ±0.7 LU', - ); - - const workflowEvidence = output('npx', [ - 'wrangler', - 'workflows', - 'instances', - 'describe', - 'hackweek-video-processing-readiness', - `video-${videoId}-attempt-1`, - '--local', - '--port', - String(port), - '--config', - config, - ]); - assert( - workflowEvidence.includes('run pinned ffmpeg processor') && - workflowEvidence.toLowerCase().includes('complete'), - 'local Workflow records a completed pinned FFmpeg Container step', - ); - - const descriptor = await get(`/api/videos/${videoId}/playback`); - assert( - descriptor.source.kind === 'mp4' && - descriptor.source.url === `/api/videos/${videoId}/content`, - 'playback returns a storage-neutral authenticated MP4 descriptor', - ); - const unauthorizedContent = await fetch(`${origin}/api/videos/${videoId}/content`); - assert( - unauthorizedContent.status === 401, - 'private derivative rejects anonymous reads', - ); - const full = await request(`/api/videos/${videoId}/content`); - const fullBytes = Buffer.from(await full.arrayBuffer()); - assert( - full.status === 200 && - full.headers.get('accept-ranges') === 'bytes' && - fullBytes.byteLength > 0, - 'authenticated full playback returns real derivative bytes', - ); - const rangeEnd = Math.min(1023, fullBytes.byteLength - 1); - const partial = await request(`/api/videos/${videoId}/content`, { - headers: {Range: `bytes=0-${rangeEnd}`}, - }); - const partialBytes = Buffer.from(await partial.arrayBuffer()); - assert( - partial.status === 206 && - partial.headers.get('content-range') === - `bytes 0-${rangeEnd}/${fullBytes.byteLength}` && - partialBytes.equals(fullBytes.subarray(0, rangeEnd + 1)), - 'authenticated range playback returns the exact derivative slice', - ); - const unsatisfiable = await request(`/api/videos/${videoId}/content`, { - headers: {Range: `bytes=${fullBytes.byteLength}-`}, - }); - assert( - unsatisfiable.status === 416 && - unsatisfiable.headers.get('content-range') === `bytes */${fullBytes.byteLength}`, - 'unsatisfiable playback range returns deterministic 416 metadata', - ); - - const playlist = await get('/api/videos/playlist?year=9999'); - assert( - playlist.videos.length === 1 && - playlist.videos[0].videoId === videoId && - playlist.videos[0].projectName === 'Readiness Video' && - playlist.videos[0].groupName === 'Readiness Team' && - playlist.videos[0].teamMembers.includes('Local Developer'), - 'ready derivative appears in the reel without a manual screening entry', - ); - - await sendJson('DELETE', '/api/projects/readiness-project/video', {confirmed: true}); - const afterRetirement = await get('/api/videos/playlist?year=9999'); - assert(afterRetirement.videos.length === 0, 'retired video leaves the curated reel'); - const retiredPlayback = await request(`/api/videos/${videoId}/content`); - assert(retiredPlayback.status === 409, 'retired derivative is no longer playable'); - - await stopServer(); - const row = query<{ - original_r2_key: string; - processed_r2_key: string; - status: string; - }>( - `SELECT original_r2_key, processed_r2_key, status FROM video_submissions WHERE id = '${escapeSql(videoId)}'`, - ); - assert( - row.status === 'retired' && row.original_r2_key !== row.processed_r2_key, - 'D1 retains distinct immutable original and derivative keys after retirement', - ); - getR2Object(row.original_r2_key, original); - getR2Object(row.processed_r2_key, derivative); - assert( - createHash('sha256') - .update(await readFile(original)) - .digest('hex') === createHash('sha256').update(bytes).digest('hex'), - 'retained R2 original matches the generated upload bytes', - ); - assert( - (await stat(derivative)).size === fullBytes.byteLength, - 'retained R2 derivative matches playback bytes', - ); - - const probe = JSON.parse( - output('ffprobe', [ - '-v', - 'error', - '-show_entries', - 'format=duration:stream=codec_type,codec_name,width,height,pix_fmt', - '-of', - 'json', - derivative, - ]), - ) as Probe; - const video = probe.streams.find((stream) => stream.codec_type === 'video'); - const audio = probe.streams.find((stream) => stream.codec_type === 'audio'); - assert( - video?.codec_name === 'h264' && - video.pix_fmt === 'yuv420p' && - (video.width ?? 0) <= 1920 && - (video.height ?? 0) <= 1080, - 'ffprobe confirms H.264 yuv420p output at or below 1080p', - ); - assert(audio?.codec_name === 'aac', 'ffprobe confirms canonical AAC audio'); - assert(Number(probe.format.duration) <= 600, 'ffprobe confirms bounded duration'); - assert(await hasFastStart(derivative), 'canonical MP4 places moov before mdat'); - const measuredLoudness = measureLoudness(derivative); - assert( - Math.abs(measuredLoudness + 16) <= 0.7, - `ffmpeg measures canonical output at ${measuredLoudness} LUFS`, - ); - - console.log('Local video readiness: 30 checks passed'); -} finally { - await stopServer(); - cleanupReadinessContainers(); - const devVarsAfter = await optionalFile(rootDevVars); - const devVarsPreserved = sameOptionalBytes(devVarsBefore, devVarsAfter); - await rm(state, {recursive: true, force: true}); - if (!devVarsPreserved) { - console.error('Developer .dev.vars changed during isolated readiness'); - process.exitCode = 1; - } -} - -interface Probe { - streams: Array<{ - codec_type: string; - codec_name?: string; - width?: number; - height?: number; - pix_fmt?: string; - }>; - format: {duration: string}; -} - -function localConfig() { - return { - name: 'hackweek-video-readiness', - main: path.join(root, 'src/worker/index.ts'), - compatibility_date: '2026-08-03', - assets: { - directory: path.join(root, 'public'), - not_found_handling: 'single-page-application', - binding: 'ASSETS', - run_worker_first: true, - }, - d1_databases: [ - { - binding: 'DB', - database_name: 'hackweek-db', - database_id: 'local', - migrations_dir: path.join(root, 'migrations'), - }, - ], - r2_buckets: [ - {binding: 'ATTACHMENTS', bucket_name: 'hackweek-attachments-readiness'}, - {binding: 'VIDEOS', bucket_name: 'hackweek-videos-readiness'}, - ], - workflows: [ - { - binding: 'VIDEO_PROCESSING_WORKFLOW', - name: 'hackweek-video-processing-readiness', - class_name: 'VideoProcessingWorkflow', - }, - ], - containers: [ - { - name: 'hackweek-video-processor-readiness', - class_name: 'VideoProcessorContainer', - image: path.join(root, 'Dockerfile.video-processor'), - image_build_context: root, - max_instances: 1, - instance_type: 'standard-2', - }, - ], - durable_objects: { - bindings: [{name: 'VIDEO_PROCESSOR', class_name: 'VideoProcessorContainer'}], - }, - migrations: [ - {tag: 'video-processor-v1', new_sqlite_classes: ['VideoProcessorContainer']}, - ], - vars: { - APP_ORIGIN: origin, - GOOGLE_CLIENT_ID: googleClientId, - GOOGLE_CLIENT_SECRET: googleClientSecret, - GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, - ALLOWED_EMAIL_DOMAIN: 'sentry.io', - VIDEO_PROCESSOR_CONCURRENCY: '1', - VIDEO_PROCESSING_AUTOSTART: 'true', - }, - observability: {enabled: true}, - }; -} - -function localDevVars() { - return `APP_ORIGIN="${origin}"\nGOOGLE_CLIENT_ID="${googleClientId}"\nGOOGLE_CLIENT_SECRET="${googleClientSecret}"\nGOOGLE_REDIRECT_URI="${origin}/api/auth/callback"\nALLOWED_EMAIL_DOMAIN="sentry.io"\nVIDEO_PROCESSOR_CONCURRENCY="1"\nVIDEO_PROCESSING_AUTOSTART="true"\n`; -} - -function localEnvironment() { - return { - ...process.env, - HACKWEEK_LOCAL_STATE_PATH: state, - HACKWEEK_WRANGLER_CONFIG: config, - APP_ORIGIN: origin, - GOOGLE_CLIENT_ID: googleClientId, - GOOGLE_CLIENT_SECRET: googleClientSecret, - GOOGLE_REDIRECT_URI: `${origin}/api/auth/callback`, - ALLOWED_EMAIL_DOMAIN: 'sentry.io', - VIDEO_PROCESSOR_CONCURRENCY: '1', - VIDEO_PROCESSING_AUTOSTART: 'true', - }; -} - -async function waitForServer() { - for (let attempt = 0; attempt < 1_200; attempt += 1) { - if (server?.exitCode !== null) { - throw new Error(`Local server exited early:\n${serverLog.join('')}`); - } - try { - if ((await fetch(`${origin}/api/health`)).ok) return; - } catch { - // Worker and Container image are still starting. - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - throw new Error(`Timed out waiting for local server:\n${serverLog.join('')}`); -} - -async function waitForReady() { - for (let attempt = 0; attempt < 600; attempt += 1) { - const response = await get('/api/projects/readiness-project/video'); - const video = response.video as { - status: string; - loudnessLufs: number; - errorMessage: string | null; - }; - if (video.status === 'ready') return video; - if (video.status === 'failed') { - throw new Error( - `Local video processing failed: ${video.errorMessage}\n${serverLog.join('')}`, - ); - } - await new Promise((resolve) => setTimeout(resolve, 250)); - } - throw new Error(`Timed out waiting for Workflow:\n${serverLog.join('')}`); -} - -async function get(pathname: string) { - const response = await request(pathname); - return responseJson(response); -} - -async function sendJson( - method: 'POST' | 'PUT' | 'DELETE', - pathname: string, - body: unknown, - expectedStatus = method === 'DELETE' ? 204 : 200, -) { - const response = await request(pathname, { - method, - headers: {'Content-Type': 'application/json', Origin: origin}, - body: JSON.stringify(body), - }); - if (response.status !== expectedStatus) { - throw new Error( - `${pathname} returned ${response.status}, expected ${expectedStatus}: ${await response.text()}\n${serverLog.join('')}`, - ); - } - return response.status === 204 ? null : ((await response.json()) as any); -} - -async function responseJson(response: Response) { - if (!response.ok) { - throw new Error( - `${new URL(response.url).pathname} returned ${response.status}: ${await response.text()}\n${serverLog.join('')}`, - ); - } - return response.json() as Promise; -} - -function request(pathname: string, init: RequestInit = {}) { - const headers = new Headers(init.headers); - headers.set('Cookie', `sentry-hackweek-session=${sessionToken}`); - return fetch(`${origin}${pathname}`, {...init, headers}); -} - -function sql(command: string) { - run('npx', [ - 'wrangler', - 'd1', - 'execute', - 'hackweek-db', - '--local', - '--persist-to', - state, - '--config', - config, - '--command', - command, - ]); -} - -function query(command: string) { - const parsed = JSON.parse( - output('npx', [ - 'wrangler', - 'd1', - 'execute', - 'hackweek-db', - '--local', - '--persist-to', - state, - '--config', - config, - '--command', - command, - '--json', - ]), - ) as Array<{results: T[]}>; - const row = parsed[0]?.results[0]; - if (!row) throw new Error(`D1 query returned no rows: ${command}`); - return row; -} - -function getR2Object(key: string, destination: string) { - run('npx', [ - 'wrangler', - 'r2', - 'object', - 'get', - `hackweek-videos-readiness/${key}`, - '--file', - destination, - '--local', - '--persist-to', - state, - '--config', - config, - ]); -} - -function ffmpeg(args: string[]) { - run('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-nostdin', '-y', ...args]); -} - -function measureLoudness(file: string) { - const result = spawnSync( - 'ffmpeg', - [ - '-hide_banner', - '-nostdin', - '-i', - file, - '-map', - '0:a:0', - '-af', - 'loudnorm=I=-16:LRA=11:TP=-1.5:print_format=json', - '-f', - 'null', - '-', - ], - {cwd: root, encoding: 'utf8'}, - ); - if (result.status !== 0) { - throw new Error(`Loudness probe failed:\n${result.stdout}\n${result.stderr}`); - } - const blocks = [...result.stderr.matchAll(/\{[\s\S]*?"input_i"[\s\S]*?\}/g)]; - const input = blocks.at(-1)?.[0]; - const loudness = input ? Number(JSON.parse(input).input_i) : Number.NaN; - if (!Number.isFinite(loudness)) throw new Error('Loudness probe returned no value'); - return loudness; -} - -async function hasFastStart(file: string) { - const bytes = await readFile(file); - const moov = bytes.indexOf(Buffer.from('moov')); - const mdat = bytes.indexOf(Buffer.from('mdat')); - return moov >= 0 && mdat >= 0 && moov < mdat; -} - -async function stopServer() { - if (!server?.pid || server.exitCode !== null) return; - try { - process.kill(-server.pid, 'SIGTERM'); - } catch { - return; - } - const exited = await Promise.race([ - new Promise((resolve) => server!.once('exit', () => resolve(true))), - new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), - ]); - if (!exited) { - try { - process.kill(-server.pid, 'SIGKILL'); - } catch { - // The detached process group exited between checks. - } - } -} - -function run(command: string, args: string[]) { - execFileSync(command, args, { - cwd: root, - env: localEnvironment(), - stdio: ['ignore', 'inherit', 'inherit'], - }); -} - -function output(command: string, args: string[]) { - return execFileSync(command, args, { - cwd: root, - env: localEnvironment(), - encoding: 'utf8', - }); -} - -function assert(value: unknown, message: string): asserts value { - if (!value) - throw new Error(`Readiness check failed: ${message}\n${serverLog.join('')}`); - console.log(`✓ ${message}`); -} - -function escapeSql(value: string) { - return value.replaceAll("'", "''"); -} - -function optionalFile(file: string) { - return readFile(file).then( - (contents) => contents, - (error: NodeJS.ErrnoException) => { - if (error.code === 'ENOENT') return null; - throw error; - }, - ); -} - -function sameOptionalBytes(left: Buffer | null, right: Buffer | null) { - return left === null ? right === null : right !== null && left.equals(right); -} - -function dockerContainerNames() { - const result = spawnSync('docker', ['ps', '--all', '--format', '{{.Names}}'], { - cwd: root, - encoding: 'utf8', - }); - if (result.status !== 0) return []; - return result.stdout - .split('\n') - .map((name) => name.trim()) - .filter(Boolean); -} - -function cleanupReadinessContainers() { - for (const name of dockerContainerNames()) { - if ( - !dockerContainersBefore.has(name) && - name.startsWith('workerd-hackweek-video-readiness-') - ) { - spawnSync('docker', ['rm', '--force', name], {cwd: root, stdio: 'ignore'}); - } - } -} diff --git a/scripts/test-video-processor.ts b/scripts/test-video-processor.ts deleted file mode 100644 index fd44b37..0000000 --- a/scripts/test-video-processor.ts +++ /dev/null @@ -1,302 +0,0 @@ -#!/usr/bin/env node -import {execFileSync, spawnSync} from 'node:child_process'; -import {chmod, mkdtemp, readFile, rm, writeFile} from 'node:fs/promises'; -import {tmpdir} from 'node:os'; -import path from 'node:path'; - -const root = process.cwd(); -const image = 'hackweek-video-processor:local'; -const work = await mkdtemp(path.join(tmpdir(), 'hackweek-processor-test-')); -const uid = process.getuid?.() ?? 1000; -const gid = process.getgid?.() ?? 1000; - -try { - run('docker', ['build', '--file', 'Dockerfile.video-processor', '--tag', image, '.']); - const version = output('docker', [ - 'run', - '--rm', - '--entrypoint', - 'ffmpeg', - image, - '-version', - ]); - assert(version.startsWith('ffmpeg version 8.0.1 '), 'pinned FFmpeg 8.0.1 runs'); - - await chmod(work, 0o777); - const audible = path.join(work, 'audible.mp4'); - const peakLimited = path.join(work, 'peak-limited.mp4'); - const lowSilent = path.join(work, 'low-silent.mp4'); - const rotationBase = path.join(work, 'rotation-base.mp4'); - const rotated = path.join(work, 'rotated.mp4'); - const overDuration = path.join(work, 'over-duration.mp4'); - const malformed = path.join(work, 'malformed.mp4'); - - ffmpeg([ - '-f', - 'lavfi', - '-i', - 'testsrc2=size=1280x720:rate=24', - '-f', - 'lavfi', - '-i', - 'sine=frequency=1000:sample_rate=48000', - '-t', - '2', - '-af', - 'volume=0.05', - '-c:v', - 'libx264', - '-preset', - 'ultrafast', - '-c:a', - 'aac', - '-shortest', - audible, - ]); - ffmpeg([ - '-f', - 'lavfi', - '-i', - 'color=size=320x180:rate=15:color=black', - '-f', - 'lavfi', - '-i', - 'aevalsrc=0.003*sin(2*PI*440*t)+if(between(t\\,1\\,1.005)\\,0.5\\,0):s=48000:c=stereo', - '-t', - '3.5', - '-c:v', - 'libx264', - '-preset', - 'ultrafast', - '-c:a', - 'aac', - '-shortest', - peakLimited, - ]); - ffmpeg([ - '-f', - 'lavfi', - '-i', - 'testsrc2=size=320x180:rate=15', - '-t', - '1.5', - '-c:v', - 'libx264', - '-preset', - 'ultrafast', - '-an', - lowSilent, - ]); - ffmpeg([ - '-f', - 'lavfi', - '-i', - 'testsrc2=size=320x180:rate=15', - '-t', - '1', - '-c:v', - 'libx264', - '-preset', - 'ultrafast', - '-an', - rotationBase, - ]); - ffmpeg(['-display_rotation:v:0', '90', '-i', rotationBase, '-c', 'copy', rotated]); - ffmpeg([ - '-f', - 'lavfi', - '-i', - 'color=size=64x64:rate=1:color=black', - '-t', - '601', - '-c:v', - 'libx264', - '-preset', - 'ultrafast', - '-an', - overDuration, - ]); - await writeFile(malformed, 'not a media file'); - await Promise.all( - [audible, peakLimited, lowSilent, rotationBase, rotated, overDuration, malformed].map( - (file) => chmod(file, 0o644), - ), - ); - - const audibleResult = processFixture('audible.mp4', 'audible-output.mp4'); - const audibleProbe = probe(path.join(work, 'audible-output.mp4')); - assertCanonical(audibleProbe); - assert( - audibleResult.audioMode === 'normalized', - 'audible input uses two-pass loudnorm', - ); - assert( - typeof audibleResult.loudnessLufs === 'number' && - Math.abs(audibleResult.loudnessLufs + 16) <= 0.7, - `audible output is ${String(audibleResult.loudnessLufs)} LUFS within ±0.7 LU`, - ); - assert( - await fastStart(path.join(work, 'audible-output.mp4')), - 'MP4 moov precedes mdat', - ); - - const peakLimitedResult = processFixture('peak-limited.mp4', 'peak-limited-output.mp4'); - assertCanonical(probe(path.join(work, 'peak-limited-output.mp4'))); - assert( - peakLimitedResult.audioMode === 'normalized', - 'peak-limited input remains normalized', - ); - assert( - typeof peakLimitedResult.loudnessLufs === 'number' && - Math.abs(peakLimitedResult.loudnessLufs + 16) <= 0.7, - `peak-limited output is ${String(peakLimitedResult.loudnessLufs)} LUFS within ±0.7 LU`, - ); - assert( - await fastStart(path.join(work, 'peak-limited-output.mp4')), - 'corrective audio pass preserves MP4 fast start', - ); - - const silentResult = processFixture('low-silent.mp4', 'low-silent-output.mp4'); - const silentProbe = probe(path.join(work, 'low-silent-output.mp4')); - assertCanonical(silentProbe); - const silentVideo = silentProbe.streams.find( - (stream) => stream.codec_type === 'video', - )!; - assert( - silentVideo.width === 320 && silentVideo.height === 180, - 'low-resolution input is not upscaled', - ); - assert( - silentResult.audioMode === 'generated-silence', - 'input without audio receives deterministic AAC silence', - ); - - const rotatedResult = processFixture('rotated.mp4', 'rotated-output.mp4'); - assert( - rotatedResult.width === 180 && rotatedResult.height === 320, - 'rotation metadata is applied without upscaling', - ); - assertCanonical(probe(path.join(work, 'rotated-output.mp4'))); - - expectFixtureFailure('malformed.mp4', 'malformed-output.mp4', 'ffprobe exited'); - expectFixtureFailure('over-duration.mp4', 'over-output.mp4', 'exceeds 600s'); - expectFixtureFailure('audible.mp4', 'missing/output.mp4', 'No such file'); - - console.log('Video processor: 36 checks passed'); -} finally { - await rm(work, {recursive: true, force: true}); -} - -function processFixture(input: string, outputFile: string) { - const result = spawnSync( - 'docker', - [ - 'run', - '--rm', - '--user', - `${uid}:${gid}`, - '--volume', - `${work}:/work`, - image, - 'process-file', - `/work/${input}`, - `/work/${outputFile}`, - ], - {cwd: root, encoding: 'utf8'}, - ); - if (result.status !== 0) { - throw new Error(`Processor failed:\n${result.stdout}\n${result.stderr}`); - } - return JSON.parse(result.stdout.trim().split('\n').at(-1)!) as { - width: number; - height: number; - loudnessLufs: number | null; - audioMode: string; - }; -} - -function expectFixtureFailure(input: string, outputFile: string, message: string) { - const result = spawnSync( - 'docker', - [ - 'run', - '--rm', - '--user', - `${uid}:${gid}`, - '--volume', - `${work}:/work`, - image, - 'process-file', - `/work/${input}`, - `/work/${outputFile}`, - ], - {cwd: root, encoding: 'utf8'}, - ); - assert(result.status !== 0, `${input} is rejected deterministically`); - assert( - `${result.stdout}\n${result.stderr}`.includes(message), - `${input} reports ${message}`, - ); -} - -function assertCanonical(result: Probe) { - const video = result.streams.find((stream) => stream.codec_type === 'video'); - const audio = result.streams.find((stream) => stream.codec_type === 'audio'); - assert(video?.codec_name === 'h264', 'output video codec is H.264'); - assert(video?.pix_fmt === 'yuv420p', 'output pixel format is yuv420p'); - assert( - (video?.width ?? 0) <= 1920 && (video?.height ?? 0) <= 1080, - 'output is <=1080p', - ); - assert(audio?.codec_name === 'aac', 'output audio codec is AAC'); - assert(Number(result.format.duration) <= 600.05, 'output duration is <=600 seconds'); -} - -function probe(file: string) { - return JSON.parse( - output('ffprobe', [ - '-v', - 'error', - '-show_entries', - 'format=duration:stream=codec_type,codec_name,width,height,pix_fmt', - '-of', - 'json', - file, - ]), - ) as Probe; -} - -interface Probe { - streams: Array<{ - codec_type: string; - codec_name?: string; - width?: number; - height?: number; - pix_fmt?: string; - }>; - format: {duration: string}; -} - -async function fastStart(file: string) { - const bytes = await readFile(file); - const moov = bytes.indexOf(Buffer.from('moov')); - const mdat = bytes.indexOf(Buffer.from('mdat')); - return moov >= 0 && mdat >= 0 && moov < mdat; -} - -function ffmpeg(args: string[]) { - run('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-nostdin', '-y', ...args]); -} - -function run(command: string, args: string[]) { - execFileSync(command, args, {cwd: root, stdio: 'inherit'}); -} - -function output(command: string, args: string[]) { - return execFileSync(command, args, {cwd: root, encoding: 'utf8'}); -} - -function assert(value: unknown, message: string): asserts value { - if (!value) throw new Error(`Video processor check failed: ${message}`); - console.log(`✓ ${message}`); -} diff --git a/test/e2e/video-rollout.test.ts b/test/e2e/video-rollout.test.ts deleted file mode 100644 index aa8ffc1..0000000 --- a/test/e2e/video-rollout.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import {readFile} from 'node:fs/promises'; - -import {describe, expect, it} from 'vitest'; - -interface WranglerConfig { - r2_buckets: Array<{binding: string; bucket_name: string}>; - workflows: Array<{binding: string; name: string; class_name: string}>; - containers: Array<{ - name: string; - class_name: string; - image: string; - max_instances: number; - }>; - vars: Record; - observability: {enabled: boolean}; -} - -describe('video rollout preparation', () => { - it('declares isolated production video resources with concurrency capped at two', async () => { - const config = JSON.parse( - await readFile('wrangler.production.json', 'utf8'), - ) as WranglerConfig; - - expect(config.r2_buckets.find(({binding}) => binding === 'VIDEOS')).toEqual({ - binding: 'VIDEOS', - bucket_name: 'hackweek-video-media-production', - }); - expect( - config.workflows.find(({binding}) => binding === 'VIDEO_PROCESSING_WORKFLOW'), - ).toMatchObject({ - name: 'hackweek-video-processing-production', - class_name: 'VideoProcessingWorkflow', - }); - expect( - config.containers.find(({class_name}) => class_name === 'VideoProcessorContainer'), - ).toMatchObject({ - name: 'hackweek-video-processor-production', - image: './Dockerfile.video-processor', - max_instances: 2, - }); - expect(config.vars).toMatchObject({ - VIDEO_PROCESSOR_CONCURRENCY: '2', - VIDEO_PROCESSING_AUTOSTART: 'true', - }); - expect(config.vars).not.toHaveProperty('STREAM_MODE'); - expect(config.observability.enabled).toBe(true); - }); - - it('pins both processor image stages by digest', async () => { - const dockerfile = await readFile('Dockerfile.video-processor', 'utf8'); - const stages = dockerfile.match(/^FROM .+@sha256:[a-f0-9]{64}.*$/gm) ?? []; - expect(stages).toHaveLength(2); - expect(dockerfile).toContain('mwader/static-ffmpeg:8.0.1@sha256:'); - expect(dockerfile).toContain('node:24.11.0-bookworm-slim@sha256:'); - }); - - it('keeps readiness on real bytes and lifecycle APIs rather than fake readiness', async () => { - const readiness = await readFile('scripts/local-readiness.ts', 'utf8'); - expect(readiness).toContain("'wrangler',\n 'workflows'"); - expect(readiness).toContain('/parts/1'); - expect(readiness).toContain('headers: {Range:'); - expect(readiness).toContain('/api/videos/playlist?year=9999'); - expect(readiness).toContain("output('ffprobe'"); - expect(readiness).not.toContain('STREAM_MODE'); - expect(readiness).not.toMatch(/UPDATE video_submissions SET status\s*=\s*'ready'/); - }); - - it('documents retained storage and the explicit production approval boundary', async () => { - const runbook = await readFile('VIDEO_ROLLOUT.md', 'utf8'); - expect(runbook).toContain('Explicit approval boundary'); - expect(runbook).toContain('max_instances: 2'); - expect(runbook).toContain('No automatic deletion'); - expect(runbook).toContain('hackweek-video-media-production'); - }); - - it('keeps migration, deployment, rollback, and future contraction compatible', async () => { - const [migration, workflow, runbook] = await Promise.all([ - readFile('migrations/0007_r2_video_lifecycle.sql', 'utf8'), - readFile('.github/workflows/deploy.yml', 'utf8'), - readFile('VIDEO_ROLLOUT.md', 'utf8'), - ]); - - expect(migration).toContain('CREATE TABLE video_submissions'); - expect(migration).not.toMatch(/ALTER TABLE project_videos|DROP TABLE stream_events/); - expect(workflow).toContain('Apply expand-compatible D1 migrations'); - expect(workflow.indexOf('Apply expand-compatible D1 migrations')).toBeLessThan( - workflow.indexOf('Deploy Worker and static assets'), - ); - expect(runbook).toContain('recorded pre-release Worker version'); - expect(runbook).toContain('does not alter `project_videos` or `stream_events`'); - expect(runbook).toContain('Future contraction (separate approval required)'); - }); -}); From ef88ad859c90297021df874df90809003282794d Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 20:54:55 +0200 Subject: [PATCH 14/18] fix(reel): restore project row sizing Limit the large reel heading typography to the section heading instead of every nested project title. Keep contributor initials aligned at the end of each playlist row and add regression coverage for the shared member bubbles. --- src/app/styles.css | 5 ++++- test/video-ui/video-ui.test.tsx | 15 +++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/app/styles.css b/src/app/styles.css index 7666d1c..6ba1b71 100644 --- a/src/app/styles.css +++ b/src/app/styles.css @@ -1779,7 +1779,7 @@ main { gap: 1rem; } .videoPanel h2, -.reelIndex h2 { +.reelIndex > h2 { margin: 0; font-size: clamp(1.6rem, 3vw, 2.4rem); letter-spacing: -0.035em; @@ -2106,6 +2106,9 @@ kbd { .reelPlaylist { margin-top: 1rem; } +.reelPlaylist .memberStack { + justify-self: end; +} .individualPlayer video { display: block; width: 100%; diff --git a/test/video-ui/video-ui.test.tsx b/test/video-ui/video-ui.test.tsx index f278a7d..654288b 100644 --- a/test/video-ui/video-ui.test.tsx +++ b/test/video-ui/video-ui.test.tsx @@ -1,5 +1,5 @@ import {QueryClient, QueryClientProvider} from '@tanstack/react-query'; -import {act, render, screen} from '@testing-library/react'; +import {act, render, screen, within} from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import {Route, Router} from 'wouter'; import {memoryLocation} from 'wouter/memory-location'; @@ -224,11 +224,14 @@ describe('video user experience', () => { expect(screen.getByRole('button', {name: 'play all'})).toBeTruthy(); expect(screen.getByRole('heading', {name: 'playlist'})).toBeTruthy(); expect(screen.getByText('Ada Lovelace · Grace Hopper')).toBeTruthy(); - expect( - screen - .getByRole('button', {name: 'start reel from First project'}) - .closest('.projectRow'), - ).not.toBeNull(); + const firstRow = screen + .getByRole('button', {name: 'start reel from First project'}) + .closest('.projectRow'); + if (!(firstRow instanceof HTMLElement)) throw new Error('Expected a project row'); + expect(within(firstRow).getByRole('heading', {name: 'First project'})).toBeTruthy(); + expect(within(firstRow).getByLabelText('Ada Lovelace, Grace Hopper')).toBeTruthy(); + expect(within(firstRow).getByText('AL')).toBeTruthy(); + expect(within(firstRow).getByText('GH')).toBeTruthy(); expect( screen.queryByText( 'private progressive MP4 playback in the curated screening order.', From f96158929b521eb93144c109da7678a0928592f9 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 21:03:12 +0200 Subject: [PATCH 15/18] fix(reel): add seeking and transient metadata Add a controller-backed timeline that remains usable in normal and fullscreen screening modes, without exposing native fullscreen controls that bypass HTML interludes. Fade each project overlay after its introduction and return stable member IDs in playlist data so shared project rows render the same contributor bubbles as the overview. --- src/app/player/ScreeningPlayer.tsx | 37 ++++++++++++++++++++++++--- src/app/player/controller.ts | 39 ++++++++++++++++++++++++++--- src/app/routes/WatchPage.tsx | 5 +--- src/app/styles.css | 40 +++++++++++++++++++++++++++++- src/shared/videos.ts | 2 +- src/worker/services/videos.ts | 12 ++++----- test/player/controller.test.tsx | 11 ++++++-- test/video-ui/video-ui.test.tsx | 10 ++++++-- test/video/video.test.ts | 4 +-- 9 files changed, 134 insertions(+), 26 deletions(-) diff --git a/src/app/player/ScreeningPlayer.tsx b/src/app/player/ScreeningPlayer.tsx index e752302..f2e1d0c 100644 --- a/src/app/player/ScreeningPlayer.tsx +++ b/src/app/player/ScreeningPlayer.tsx @@ -19,11 +19,13 @@ export interface ScreeningPlayerHandle { playFrom(videoId: string): void; } -const initialState = (index: number): PlayerState => ({ +const initialState = (index: number, durationSeconds: number): PlayerState => ({ phase: 'idle', index, error: null, countdownSeconds: null, + currentTime: 0, + durationSeconds, }); export const ScreeningPlayer = forwardRef< @@ -46,7 +48,10 @@ export const ScreeningPlayer = forwardRef< const controller = useRef(null); const announcedVideoId = useRef(null); const requestedIndex = playlist.findIndex((clip) => clip.videoId === initialVideoId); - const [state, setState] = useState(initialState(Math.max(0, requestedIndex))); + const initialIndex = Math.max(0, requestedIndex); + const [state, setState] = useState( + initialState(initialIndex, playlist[initialIndex]?.durationSeconds ?? 0), + ); const clip = playlist[state.index]; const publishState = useCallback( @@ -114,8 +119,11 @@ export const ScreeningPlayer = forwardRef< const activeSlot = state.index % 2; const showingTitle = state.phase === 'title'; - const team = clip.teamMembers.join(' · ') || 'Hackweek team'; + const team = + clip.teamMembers.map(({displayName}) => displayName).join(' · ') || 'Hackweek team'; const projectMeta = [clip.groupName, team].filter(Boolean).join(' · '); + const canSeek = ['playing', 'paused'].includes(state.phase); + const timelineDuration = state.durationSeconds || clip.durationSeconds; return (
    @@ -147,7 +155,7 @@ export const ScreeningPlayer = forwardRef< )}
    {['playing', 'paused'].includes(state.phase) && ( -
    +
    {clip.projectName} {projectMeta}
    @@ -188,6 +196,22 @@ export const ScreeningPlayer = forwardRef< )}
    +
    + {formatPlaybackTime(state.currentTime)} + + controller.current?.seek(event.currentTarget.valueAsNumber) + } + /> + {formatPlaybackTime(timelineDuration)} +
    sound starts only after you press play
    @@ -190,7 +191,7 @@ export const ScreeningPlayer = forwardRef< className="screeningStart" onClick={() => void controller.current?.skip()} > - {state.index < playlist.length - 1 ? 'skip now' : 'finish reel'} + {activeIndex < playlist.length - 1 ? 'skip now' : 'finish reel'} )} @@ -218,10 +219,10 @@ export const ScreeningPlayer = forwardRef< > {state.phase === 'paused' ? '▶ resume' : 'Ⅱ pause'} space -
    +
    {clip.projectName} - {state.index + 1} of {playlist.length} + {activeIndex + 1} of {playlist.length}
    videoId).join(':')} ref={player} playlist={playlist.data.videos} getPlayback={getPlayback} diff --git a/src/app/styles.css b/src/app/styles.css index 666f0e0..5a6fe74 100644 --- a/src/app/styles.css +++ b/src/app/styles.css @@ -2164,10 +2164,16 @@ kbd { .screeningControls { grid-template-columns: repeat(3, 1fr); } - .screeningControls > div { - grid-column: 1 / -1; + .screeningTimeline { grid-row: 1; } + .screeningNowPlaying { + grid-column: 1 / -1; + grid-row: 2; + } + .screeningControls button { + grid-row: 3; + } .screeningControls button:last-child { grid-column: auto; } @@ -2184,6 +2190,7 @@ kbd { grid-template-columns: 1fr 1fr; } .screeningControls button:last-child { + grid-row: 4; grid-column: 1 / -1; } .titleCard h2, diff --git a/src/app/video/upload.ts b/src/app/video/upload.ts index 2f5fec3..766d1e2 100644 --- a/src/app/video/upload.ts +++ b/src/app/video/upload.ts @@ -40,6 +40,7 @@ export function createMultipartUpload( const bytesSent = () => parts.reduce((total, part) => total + part.sizeBytes, 0); const notify = () => onChange({phase, bytesSent: bytesSent(), bytesTotal: file.size, error}); + const isPaused = () => phase === 'paused'; async function run() { if (running || phase === 'complete') return; @@ -50,6 +51,7 @@ export function createMultipartUpload( try { const partCount = Math.ceil(file.size / session.partSize); for (let partNumber = 1; partNumber <= partCount; partNumber += 1) { + if (isPaused()) return; if (parts.some((part) => part.partNumber === partNumber)) continue; active = new AbortController(); const start = (partNumber - 1) * session.partSize; @@ -69,6 +71,7 @@ export function createMultipartUpload( notify(); } + if (isPaused()) return; const completed = await fetch(`${uploadUrl(session)}/complete`, { method: 'POST', headers: {'Content-Type': 'application/json'}, diff --git a/src/worker/services/videos.ts b/src/worker/services/videos.ts index b1eaaa6..a766e51 100644 --- a/src/worker/services/videos.ts +++ b/src/worker/services/videos.ts @@ -46,8 +46,6 @@ interface ProcessingAttemptRow { attempt_status: string; } -export class ProcessingCapacityError extends Error {} - interface UploadRow { id: string; video_id: string; @@ -622,7 +620,9 @@ export async function claimVideoProcessingAttempt( videoId: string, attempt: number, concurrency: number, -): Promise<{status: 'claimed'; outputKey: string} | {status: 'stale'}> { +): Promise< + {status: 'claimed'; outputKey: string} | {status: 'stale'} | {status: 'capacity'} +> { const row = await processingAttempt(db, videoId, attempt); if ( !row || @@ -637,9 +637,7 @@ export async function claimVideoProcessingAttempt( `SELECT COUNT(*) count FROM video_processing_attempts WHERE status = 'running'`, ) .first<{count: number}>(); - if ((running?.count ?? 0) >= concurrency) { - throw new ProcessingCapacityError('Video processor concurrency is currently full'); - } + if ((running?.count ?? 0) >= concurrency) return {status: 'capacity'}; const outputKey = videoProcessedKey(row.project_id, videoId, attempt); const results = await db.batch([ db @@ -673,7 +671,7 @@ export async function claimVideoProcessingAttempt( } const current = await processingAttempt(db, videoId, attempt); if (current?.video_status === 'queued' && current.attempt_status === 'queued') { - throw new ProcessingCapacityError('Video processor concurrency is currently full'); + return {status: 'capacity'}; } return {status: 'stale'}; } diff --git a/src/worker/workflows/video-processing.ts b/src/worker/workflows/video-processing.ts index f96ef6f..30c028a 100644 --- a/src/worker/workflows/video-processing.ts +++ b/src/worker/workflows/video-processing.ts @@ -32,29 +32,42 @@ export class VideoProcessingWorkflow extends WorkflowEntrypoint< > { async run(event: WorkflowEvent, step: WorkflowStep) { const {videoId, attempt} = event.payload; - let claim: Awaited>; - try { - claim = await step.do( - 'claim current processing attempt', - { - retries: {limit: 360, delay: '5 seconds', backoff: 'constant'}, - timeout: '30 seconds', - }, - () => - claimVideoProcessingAttempt( - this.env.DB, - videoId, - attempt, - processingConcurrency(this.env.VIDEO_PROCESSOR_CONCURRENCY), - ), - ); - } catch (error) { - const message = errorMessage(error); - logVideoProcessing('error', 'claim_failed', {videoId, attempt, message}); - await step.do('record claim failure', () => - failVideoProcessingAttempt(this.env.DB, videoId, attempt, message), - ); - return {status: 'failed', stage: 'claim'}; + let claim: Exclude< + Awaited>, + {status: 'capacity'} + >; + for (;;) { + let candidate: Awaited>; + try { + candidate = await step.do( + 'claim current processing attempt', + { + retries: {limit: 5, delay: '2 seconds', backoff: 'constant'}, + timeout: '30 seconds', + }, + () => + claimVideoProcessingAttempt( + this.env.DB, + videoId, + attempt, + processingConcurrency(this.env.VIDEO_PROCESSOR_CONCURRENCY), + ), + ); + } catch (error) { + const message = errorMessage(error); + logVideoProcessing('error', 'claim_failed', {videoId, attempt, message}); + await step.do('record claim failure', () => + failVideoProcessingAttempt(this.env.DB, videoId, attempt, message), + ); + return {status: 'failed', stage: 'claim'}; + } + if (candidate.status === 'capacity') { + logVideoProcessing('info', 'waiting_for_capacity', {videoId, attempt}); + await step.sleep('wait for processing capacity', '15 seconds'); + continue; + } + claim = candidate; + break; } if (claim.status === 'stale') { logVideoProcessing('info', 'stale_before_processing', {videoId, attempt}); diff --git a/test/player/controller.test.tsx b/test/player/controller.test.tsx index e69e80f..879ceaf 100644 --- a/test/player/controller.test.tsx +++ b/test/player/controller.test.tsx @@ -60,6 +60,7 @@ describe('dual screening controller', () => { videos[0].dispatchEvent(new Event('ended')); await Promise.resolve(); + expect(audio.resume).toHaveBeenCalledTimes(2); expect(states.at(-1)?.phase).toBe('title'); expect(states.at(-1)?.index).toBe(1); await vi.advanceTimersByTimeAsync(10); @@ -97,9 +98,10 @@ describe('dual screening controller', () => { expect(states.at(-1)).toMatchObject({phase: 'title', index: 0}); const errors: PlayerState[] = []; + const errorVideos: [HTMLVideoElement, HTMLVideoElement] = [fakeVideo(), fakeVideo()]; const errorController = createScreeningController({ playlist, - elements: [fakeVideo(), fakeVideo()], + elements: errorVideos, audio: fakeAudio(), getPlayback: async (videoId) => { if (videoId === 'video-1') throw new Error('private source unavailable'); @@ -118,6 +120,11 @@ describe('dual screening controller', () => { phase: 'error', error: 'private source unavailable', }); + errorVideos[0].dispatchEvent(new Event('durationchange')); + expect(errors.at(-1)).toMatchObject({ + phase: 'error', + error: 'private source unavailable', + }); await vi.advanceTimersByTimeAsync(10); expect(errors.at(-1)).toMatchObject({phase: 'title', index: 1}); }); diff --git a/test/video-ui/video-ui.test.tsx b/test/video-ui/video-ui.test.tsx index 4fb593e..09ed45d 100644 --- a/test/video-ui/video-ui.test.tsx +++ b/test/video-ui/video-ui.test.tsx @@ -109,6 +109,34 @@ describe('video user experience', () => { expect(screen.queryByRole('progressbar')).toBeNull(); }); + it('does not start another part when pause races a completed request', async () => { + const file = new File(['abcdef'], 'pause.mp4', {type: 'video/mp4'}); + const session = {...uploadSession, fileSize: file.size, partSize: 3}; + let resolveFirstPart: ((response: Response) => void) | undefined; + fetchMock.mockImplementation( + () => + new Promise((resolve) => { + resolveFirstPart = resolve; + }), + ); + const snapshots: UploadSnapshot[] = []; + const upload = createMultipartUpload(file, session, (snapshot) => + snapshots.push(snapshot), + ); + + upload.start(); + await vi.waitFor(() => expect(resolveFirstPart).toBeTypeOf('function')); + resolveFirstPart?.( + json({part: {partNumber: 1, etag: 'first', sizeBytes: session.partSize}}), + ); + await upload.pause(); + + await vi.waitFor(() => + expect(snapshots.at(-1)).toMatchObject({phase: 'paused', bytesSent: 3}), + ); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + it('persists multipart resume identity and skips server-confirmed parts', async () => { const file = new File(['abcde'], 'resume.mp4', { type: 'video/mp4', @@ -248,6 +276,26 @@ describe('video user experience', () => { expect(screen.getByRole('heading', {name: 'no videos are ready'})).toBeTruthy(); }); + it('falls back safely when a refreshed playlist removes the selected clip', () => { + const view = render( + , + ); + expect(screen.getByRole('button', {name: 'play from Second project'})).toBeTruthy(); + + view.rerender( + , + ); + expect(screen.getByRole('button', {name: 'play all'})).toBeTruthy(); + }); + it('exposes visible pause, skip, fullscreen controls and keyboard shortcuts', async () => { const actions = {togglePause: vi.fn(), skip: vi.fn(), fullscreen: vi.fn()}; for (const [code, key] of [ diff --git a/test/video/video.test.ts b/test/video/video.test.ts index 68b7b6b..e3431ea 100644 --- a/test/video/video.test.ts +++ b/test/video/video.test.ts @@ -8,7 +8,6 @@ import { createMultipartVideoUpload, failVideoProcessingAttempt, MAX_VIDEO_BYTES, - ProcessingCapacityError, publishVideoProcessingAttempt, reapExpiredMultipartVideoUploads, VIDEO_PART_SIZE, @@ -715,7 +714,7 @@ describe('R2 multipart video lifecycle', () => { expect(claimed.status).toBe('claimed'); await expect( claimVideoProcessingAttempt(env.DB, right.video.id, 1, 1), - ).rejects.toBeInstanceOf(ProcessingCapacityError); + ).resolves.toEqual({status: 'capacity'}); expect( await env.DB.prepare('SELECT status FROM video_submissions WHERE id = ?') .bind(right.video.id) From e24089d1545d1aa74ff5cfebab29289a48cc3c7c Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 11 Aug 2026 21:49:25 +0200 Subject: [PATCH 18/18] fix(reel): avoid blocking automatic advance Treat AudioContext resume as best-effort during ended and error-driven autoplay transitions so a browser-held resume promise cannot stall the reel. User-triggered start and resume actions still await audio activation, and the controller test covers a permanently pending automatic resume. --- src/app/player/controller.ts | 2 +- test/player/controller.test.tsx | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/app/player/controller.ts b/src/app/player/controller.ts index 0ad3b58..170d650 100644 --- a/src/app/player/controller.ts +++ b/src/app/player/controller.ts @@ -181,7 +181,7 @@ export function createScreeningController({ index += 1; active = active === 0 ? 1 : 0; try { - await audio.resume(); + void audio.resume().catch(() => undefined); await playCurrent(); } catch (error) { fail(error instanceof Error ? error.message : 'playback could not start'); diff --git a/test/player/controller.test.tsx b/test/player/controller.test.tsx index 879ceaf..58bc2a6 100644 --- a/test/player/controller.test.tsx +++ b/test/player/controller.test.tsx @@ -16,6 +16,9 @@ describe('dual screening controller', () => { const states: PlayerState[] = []; const attached: string[] = []; const audio = fakeAudio(); + vi.mocked(audio.resume) + .mockResolvedValueOnce(undefined) + .mockImplementationOnce(() => new Promise(() => undefined)); const controller = createScreeningController({ playlist, elements: videos,