diff --git a/docs/docs/commands.md b/docs/docs/commands.md index 599080b9..06af7c12 100644 --- a/docs/docs/commands.md +++ b/docs/docs/commands.md @@ -1,7 +1,7 @@ ## Show player Opens the PodNotes pane and brings it into focus. -If the pane already exists but is hidden — for example in a collapsed sidebar or out of view because the right sidebar has too many icons — this command reveals it. If it does not exist yet, the command creates it in the right sidebar. You can run it from the command palette or bind it to a hotkey. +If the pane already exists but is hidden - for example in a collapsed sidebar or out of view because the right sidebar has too many icons - this command reveals it. If it does not exist yet, the command creates it in the right sidebar. You can run it from the command palette or bind it to a hotkey. PodNotes also adds a **podcast icon to the left ribbon** as a reliable way to reopen the pane. On mobile it appears in the ribbon menu, and you can hide it via Obsidian's *Manage ribbon actions* if you prefer to use the command instead. @@ -25,7 +25,7 @@ This will skip the current episode forward by the amount of seconds specified in ## Download Playing Episode This will download the currently playing episode. -Downloads are stored in the location specified by the **Episode download path** setting. This path is a template and **must include a per-episode token** such as `{{title}}` (the default is `PodNotes/{{podcast}}/{{title}}`). A path without `{{title}}` makes every episode resolve to the same file, so downloads overwrite each other or fail. The file extension is added automatically — do not include one. +Downloads are stored in the location specified by the **Episode download path** setting. This path is a template and **must include a per-episode token** such as `{{title}}` (the default is `PodNotes/{{podcast}}/{{title}}`). A path without `{{title}}` makes every episode resolve to the same file, so downloads overwrite each other or fail. The file extension is added automatically - do not include one. ## Capture Timestamp This will capture the current timestamp of the currently playing episode. @@ -85,8 +85,16 @@ Episode links look like this: [https://pod.link/1138055739/episode/1732808e781cc They can be used to share the episode with others, no matter what podcast app they use. ## Transcribe current episode -This command will transcribe the currently playing episode using OpenAI's Whisper model. -The transcription will be saved in the location specified in the transcript settings. +This command transcribes the currently playing audio episode and saves the +result at the path configured in **Transcript settings**. -Note: This feature requires an OpenAI API key to be set in the settings. +Plain Whisper transcription and OpenAI speaker diarization require an available +OpenAI secret on the current device. Deepgram speaker diarization requires an +available Deepgram secret instead. In **Settings -> PodNotes -> Transcript +settings**, select an existing Obsidian secret or create one for the provider +you use. + +The command remains available when a required secret is missing so PodNotes can +tell you which provider needs to be configured. If a selected secret is not +available on this device, open PodNotes settings and select or create it again. diff --git a/docs/docs/import_export.md b/docs/docs/import_export.md index cd9a890d..4e076ce9 100644 --- a/docs/docs/import_export.md +++ b/docs/docs/import_export.md @@ -19,33 +19,79 @@ First designate a file path to save to (or use the default), and click _Export_. ## Settings & templates -Under the **Settings & templates** heading you can move your PodNotes +Under the **Settings & templates** heading, you can move your PodNotes configuration between vaults or back it up. This covers your preferences, note/timestamp/transcript templates, file paths, saved feeds, and playlists. ![Settings & templates import/export](resources/settings_import_export.png) -Playback progress, downloaded-episode bookkeeping, the currently-playing +Playback progress, downloaded-episode bookkeeping, the currently playing episode, and the episode-to-note mapping are **not** included, because they are specific to a single vault. +PodNotes settings exports use format v2. API key values and the names of the +Obsidian secrets selected in PodNotes are omitted by default. This keeps a +destination vault's existing OpenAI and Deepgram selections unchanged when you +import the file. + ### Exporting settings -1. (Optional) Enable **Include OpenAI API key** if you also want your key in the - file. The key is written in plaintext, so only do this for files you keep - private — the file lives in your vault and may sync to other devices or be - read by other plugins. +1. Leave **Include API keys** off for a normal settings export. To transfer the + OpenAI and Deepgram keys that are available on this device, enable it + explicitly. 2. Set a file name (or keep the default `PodNotes_Settings.json`). 3. Click **Export**. The settings file is written to your vault. +When **Include API keys** is enabled, available values are added in plaintext +under a separate top-level `secrets` payload: + +```json +{ + "type": "podnotes-settings", + "version": 2, + "settings": {}, + "secrets": { + "openAI": "your OpenAI API key", + "deepgram": "your Deepgram API key" + } +} +``` + +Only configured keys are included. If a selected secret name exists in PodNotes +settings but its value is unavailable on this device, the export stops instead +of silently creating an incomplete credential backup. Open **Transcript +settings**, select or create the missing secret on this device, and export +again. + +An export containing `secrets` is sensitive. The values are plaintext in a file +inside your vault, where they may sync to other devices or be read by other +plugins. Keep the file private and delete it when it is no longer needed. + ### Importing settings -1. Click **Import** next to _Import settings_ and choose a settings file. Both a - PodNotes export file and a raw `data.json` are accepted. +1. Click **Import** next to **Import settings** and choose a settings file. + PodNotes accepts current v2 exports, legacy v1 exports, and a raw PodNotes + `data.json`. 2. Confirm the import. Your current preferences, templates, feeds, and playlists are replaced with the imported values; playback progress and downloads are kept. +An import without API key values preserves the OpenAI and Deepgram secret names +already selected in the destination vault. For a legacy v1 export or raw +`data.json`, PodNotes recognizes the old plaintext `openAIApiKey` and +`diarizationApiKey` fields and imports them as secrets instead of putting them +back into plugin settings. + +When an import contains API keys, the confirmation names the affected +providers. PodNotes stores each value under a PodNotes-owned name in Obsidian's +vault-local secret storage, then updates PodNotes to reference it. Existing +Obsidian secrets are never overwritten. If a PodNotes-owned name already holds +a different value, the imported key is stored under a new suffixed name. + +Imported secret values are available only in the current vault on the current +device. Repeat the import or select/create the appropriate secrets separately +on another device. + A file exported by a newer version of PodNotes is rejected until you update the -plugin. If an episode is already open when you import, a changed default playback -rate applies to the next episode you open. \ No newline at end of file +plugin. If an episode is already open when you import, a changed default +playback rate applies to the next episode you open. diff --git a/docs/docs/resources/diarization_settings.png b/docs/docs/resources/diarization_settings.png index 8228fb42..59baab79 100644 Binary files a/docs/docs/resources/diarization_settings.png and b/docs/docs/resources/diarization_settings.png differ diff --git a/docs/docs/resources/settings_import_export.png b/docs/docs/resources/settings_import_export.png index 40436014..5c5f7650 100644 Binary files a/docs/docs/resources/settings_import_export.png and b/docs/docs/resources/settings_import_export.png differ diff --git a/docs/docs/transcripts.md b/docs/docs/transcripts.md index 917981bd..62e144e9 100644 --- a/docs/docs/transcripts.md +++ b/docs/docs/transcripts.md @@ -1,25 +1,60 @@ # Transcripts -PodNotes allows you to create transcripts of podcast episodes using OpenAI's Whisper model. +PodNotes can create transcript notes from podcast episodes. Plain transcription +uses OpenAI's Whisper model, while optional speaker diarization can use OpenAI +or Deepgram. ## Setting Up -Before you can use the transcription feature, you need to set up a few things: +Before you can use transcription, set up the following: -1. **OpenAI API Key**: You need to have an OpenAI API key. You can get one by signing up at [OpenAI's website](https://openai.com/). Once you have the key, enter it in the PodNotes settings under the "Transcript settings" section. +1. **OpenAI API key**: Create a key at [OpenAI's website](https://openai.com/). + In **Settings -> PodNotes -> Transcript settings**, use **OpenAI API key** to + select an existing Obsidian secret or create one. Plain Whisper transcription + and OpenAI diarization both use this secret. -2. **Transcript File Path**: In the settings, you can specify where you want the transcript files to be saved. You can use placeholders like `{{podcast}}` and `{{title}}` in the path. +2. **Transcript file path**: Choose where transcript files are saved. You can + use placeholders such as `{{podcast}}` and `{{title}}` in the path. -3. **Transcript Template**: You can also customize how the transcript content is formatted using a template. +3. **Transcript template**: Customize how the transcript note is formatted. + +### How API keys are stored + +PodNotes uses Obsidian's native secret picker. The API key value is kept in +Obsidian's vault-local secret storage; PodNotes stores only the selected secret +name in its `data.json`, through the `openAISecretId` and `deepgramSecretId` +references. Obsidian secrets are centralized and can be selected by other +plugins, so this is not plugin-specific isolation. + +Secret values are local to the current vault on the current device. A selected +secret name can sync with the rest of the vault configuration while its value +does not. If PodNotes says that a selected secret is unavailable on this device, +open **Transcript settings** on that device and select or create the secret +again. + +When upgrading from a version that stored API keys directly in `data.json`, +PodNotes moves existing OpenAI and Deepgram keys into Obsidian's secret storage +before rewriting the settings file. The plaintext fields are then removed. If +that migration cannot be completed and verified, PodNotes leaves the existing +settings file unchanged and shows a persistent notice so you can restart the +plugin and retry. ## Creating a Transcript To create a transcript: -1. Start playing the podcast episode you want to transcribe. -2. Use the "Transcribe current episode" command in Obsidian. -3. PodNotes will fetch the audio for the episode you are playing (reusing an already-downloaded copy when one exists), split it into chunks, and send these chunks to OpenAI for transcription. The transcription always uses the currently playing episode's own audio, regardless of your download path settings. -4. Once the transcription is complete, a new file will be created at the specified location with the transcribed content. +1. Start playing the audio episode you want to transcribe. +2. Run **PodNotes: Transcribe current episode** from the command palette. +3. PodNotes fetches the currently playing episode's audio, reusing an existing + download when available. Plain Whisper and OpenAI diarization split large + audio into chunks; Deepgram diarization sends the episode in one request. +4. When transcription finishes, PodNotes creates a note at the configured + transcript path. + +The command always uses the currently playing episode's own audio, regardless +of your episode download path setting. It remains available when a required +secret is missing so it can tell you which provider must be configured on the +current device. Generated transcript notes are also available to workflow plugins through the [PodNotes API](./api.md#transcript), so tools such as QuickAdd or Templater can @@ -27,11 +62,15 @@ read the text and send it to the AI provider configured in your own macro. ## Transcript Template -The transcript template works similarly to the [note template](./templates.md#note-template), but with the added `{{transcript}}` placeholder. +The transcript template works similarly to the +[note template](./templates.md#note-template), with the additional +`{{transcript}}` placeholder. ## Speaker Diarization -By default the transcription uses OpenAI's Whisper model, which produces plain text with **no speaker labels**. Speaker diarization is an opt-in setting that instead labels each segment of the transcript by speaker, turning a wall of text into a readable, speaker-by-speaker conversation: +By default, transcription uses OpenAI's Whisper model, which produces plain +text with **no speaker labels**. Speaker diarization is an opt-in setting that +labels each transcript segment by speaker: ![Transcript with speaker labels](resources/transcript_diarization.png) @@ -41,8 +80,16 @@ In the **Transcript settings** section, turn on **Speaker diarization** and choo ![Speaker diarization settings](resources/diarization_settings.png) -- **OpenAI** (`gpt-4o-transcribe-diarize`): reuses the OpenAI API key you already entered above, so there is nothing else to configure. Because each request is capped at ~20 MB (a conservative margin under OpenAI's 25 MB request cap), a long episode is split into chunks that are diarized independently — so on long episodes the speaker labels can change across chunk boundaries (the same person may be labelled `A` in one chunk and `B` in the next). A typical-length episode fits in a single request and is fully consistent. -- **Deepgram**: sends the whole episode in one request, so speaker labels stay consistent across the entire episode. This requires a separate **Deepgram API key**, which you can create at [deepgram.com](https://deepgram.com) (new accounts include free credit). Your Deepgram key is stored separately from your OpenAI key and is only used for diarization. +- **OpenAI** (`gpt-4o-transcribe-diarize`) reuses the OpenAI secret selected + above. Because each request is capped at about 20 MB, a conservative margin + under OpenAI's 25 MB request cap, a long episode is diarized in independent + chunks. Speaker labels can therefore reset across chunk boundaries. A typical + episode fits in one request and keeps consistent labels. +- **Deepgram** sends the whole episode in one request, so speaker labels remain + consistent throughout the episode. Create a separate key at + [deepgram.com](https://deepgram.com), then select an existing Obsidian secret + or create one under **Deepgram API key**. The Deepgram secret is used only for + Deepgram diarization and is local to the current vault and device. Diarization is off by default, so existing transcripts and the plain-Whisper workflow are unchanged unless you enable it. @@ -53,7 +100,9 @@ The **Speaker label format** setting controls the prefix added before each speak - OpenAI labels speakers `A`, `B`, `C`, … - Deepgram labels speakers `1`, `2`, `3`, … -The default is `**{{speaker}}:** `, which renders as `**A:**`-style bold prefixes. To spell out the word "Speaker", set it to `**Speaker {{speaker}}:** ` (rendering `**Speaker A:**`); `> {{speaker}}: ` would instead put each turn in a blockquote. +The default is `**{{speaker}}:** `, which renders as an `**A:**`-style bold +prefix. To spell out the word "Speaker", use `**Speaker {{speaker}}:** `. To put +each turn in a blockquote, use `> {{speaker}}: `. The labelled transcript replaces the usual `{{transcript}}` value in your [transcript template](#transcript-template), so you don't need to change your template to use diarization. diff --git a/manifest.json b/manifest.json index 2719c32b..ab8be03c 100644 --- a/manifest.json +++ b/manifest.json @@ -2,7 +2,7 @@ "id": "podnotes", "name": "PodNotes", "version": "2.17.5", - "minAppVersion": "0.15.9", + "minAppVersion": "1.11.4", "description": "Helps you write notes on podcasts.", "author": "Christian B. B. Houmann", "authorUrl": "https://bagerbach.com", diff --git a/package-lock.json b/package-lock.json index f9d91ed0..5721c5ac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "2.17.5", "license": "MIT", "dependencies": { + "@noble/hashes": "^2.2.0", "fuse.js": "^7.4.2", "openai": "^6.45.0" }, @@ -1927,6 +1928,18 @@ "@emnapi/runtime": "^1.7.1" } }, + "node_modules/@noble/hashes": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.2.0.tgz", + "integrity": "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@octokit/auth-token": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-6.0.0.tgz", diff --git a/package.json b/package.json index 1fa02e47..d5899165 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "format": "oxfmt src tests scripts .github package.json manifest.json versions.json tsconfig.base.json tsconfig.json tsconfig.test.json .oxlintrc.json .oxfmtrc.json svelte.config.mjs vite.config.ts vitest.config.ts vitest.e2e.config.ts vitest.setup.ts wrangler.jsonc orca.yaml docs/mkdocs.yml" }, "dependencies": { + "@noble/hashes": "^2.2.0", "fuse.js": "^7.4.2", "openai": "^6.45.0" }, diff --git a/scripts/provision-obsidian-e2e-vault.mjs b/scripts/provision-obsidian-e2e-vault.mjs index f700cd8b..4a0dc5fb 100644 --- a/scripts/provision-obsidian-e2e-vault.mjs +++ b/scripts/provision-obsidian-e2e-vault.mjs @@ -21,12 +21,13 @@ const PLUGIN_ID = "podnotes"; // "=> true" so an echoed command can't be mistaken for a positive result. export const PODNOTES_READY_EVAL = `Boolean(app.plugins.plugins[${JSON.stringify(PLUGIN_ID)}])`; -// A valid, empty PodNotes settings document. Mirrors DEFAULT_SETTINGS in -// src/constants.ts so a freshly provisioned vault loads with clean state instead -// of QuickAdd's { choices, migrations } shape. Keep in sync with constants.ts. -// (currentEpisode is intentionally omitted — DEFAULT_SETTINGS sets it to +// A valid, empty PodNotes schema-v1 document. Mirrors DEFAULT_SETTINGS plus the +// persistence marker so a freshly provisioned vault loads with clean state +// instead of QuickAdd's { choices, migrations } shape. Keep in sync with +// constants.ts. (currentEpisode is intentionally omitted - DEFAULT_SETTINGS sets it to // undefined, which JSON cannot represent and PodNotes treats as absent.) export const DEFAULT_PODNOTES_DATA = { + schemaVersion: 2, savedFeeds: {}, podNotes: {}, defaultPlaybackRate: 1, @@ -103,8 +104,8 @@ export const DEFAULT_PODNOTES_DATA = { shouldRepeat: false, episodes: [], }, - openAIApiKey: "", - diarizationApiKey: "", + openAISecretId: "", + deepgramSecretId: "", transcript: { path: "transcripts/{{podcast}}/{{title}}.md", template: "# {{title}}\n\nPodcast: {{podcast}}\nDate: {{date}}\n\n{{transcript}}", diff --git a/scripts/provision-obsidian-e2e-vault.test.ts b/scripts/provision-obsidian-e2e-vault.test.ts index b95fa449..40889e2e 100644 --- a/scripts/provision-obsidian-e2e-vault.test.ts +++ b/scripts/provision-obsidian-e2e-vault.test.ts @@ -49,14 +49,15 @@ describe("provision-obsidian-e2e-vault", () => { expect(options.vaultPath).toBe("/tmp/podnotes-repo/vaults/podnotes-a"); }); - it("seeds a data.json that mirrors the real DEFAULT_SETTINGS", () => { + it("seeds schema v2 data that mirrors the real DEFAULT_SETTINGS", () => { // JSON cannot represent currentEpisode: undefined, so compare the - // serialized forms — this is exactly what lands in the vault's data.json + // serialized forms - this is exactly what lands in the vault's data.json // and it fails if a new setting is added to src/constants.ts without // updating DEFAULT_PODNOTES_DATA. - expect(JSON.parse(JSON.stringify(DEFAULT_PODNOTES_DATA))).toEqual( - JSON.parse(JSON.stringify(DEFAULT_SETTINGS)), - ); + expect(JSON.parse(JSON.stringify(DEFAULT_PODNOTES_DATA))).toEqual({ + schemaVersion: 2, + ...JSON.parse(JSON.stringify(DEFAULT_SETTINGS)), + }); }); it("defaults the vault name to podnotes-", () => { diff --git a/src/TemplateEngine.test.ts b/src/TemplateEngine.test.ts index a540d7c2..b5289bed 100644 --- a/src/TemplateEngine.test.ts +++ b/src/TemplateEngine.test.ts @@ -164,6 +164,15 @@ describe("empty tag arguments (NT-05/CH-09)", () => { plugin.set({ settings: { feedNote: { path: "" }, savedFeeds: {} } } as never); expect(NoteTemplateEngine("{{date:YYYY}}", demoEpisode)).toBe("2024"); }); + + it("renders a JSON-restored episode date without throwing", () => { + const restoredEpisode = { + ...demoEpisode, + episodeDate: "2024-01-01T00:00:00.000Z" as unknown as Date, + }; + + expect(NoteTemplateEngine("{{date:YYYY-MM-DD}}", restoredEpisode)).toBe("2024-01-01"); + }); }); describe("NoteTemplateEngine feed-scoped tags (#163)", () => { diff --git a/src/constants.ts b/src/constants.ts index d11f9793..5ac40069 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -154,8 +154,8 @@ export const DEFAULT_SETTINGS: IPodNotesSettings = { ...LOCAL_FILES_SETTINGS, episodes: [], }, - openAIApiKey: "", - diarizationApiKey: "", + openAISecretId: "", + deepgramSecretId: "", transcript: { path: "transcripts/{{podcast}}/{{title}}.md", template: "# {{title}}\n\nPodcast: {{podcast}}\nDate: {{date}}\n\n{{transcript}}", diff --git a/src/main.activateView.test.ts b/src/main.activateView.test.ts index e4ef0a60..863e311e 100644 --- a/src/main.activateView.test.ts +++ b/src/main.activateView.test.ts @@ -254,6 +254,11 @@ describe("PodNotes onload wiring (#55)", () => { storeUnsubscribers: [], views: new Set(), app: { + secretStorage: { + getSecret: vi.fn(() => null), + setSecret: vi.fn(), + listSecrets: vi.fn(() => []), + }, workspace: { onLayoutReady: vi.fn(), on: vi.fn(() => ({})), @@ -398,7 +403,7 @@ describe("PodNotes onload wiring (#55)", () => { mediaType: "video", }; const { commands } = await loadPlugin({ - openAIApiKey: "sk-test", + openAISecretId: "podnotes-openai-api-key", currentEpisode: videoEpisode, }); diff --git a/src/main.persistence.test.ts b/src/main.persistence.test.ts new file mode 100644 index 00000000..36952ea0 --- /dev/null +++ b/src/main.persistence.test.ts @@ -0,0 +1,336 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { SecretStorage } from "obsidian"; +import { DEFAULT_SETTINGS } from "./constants"; +import PodNotes from "./main"; +import { CredentialRepository } from "./services/CredentialRepository"; + +function memorySecretStorage(initial: Record = {}) { + const values = new Map(Object.entries(initial)); + const storage = { + getSecret: vi.fn((id: string) => values.get(id) ?? null), + setSecret: vi.fn((id: string, value: string) => values.set(id, value)), + listSecrets: vi.fn(() => [...values.keys()]), + } as unknown as SecretStorage; + return { storage, values }; +} + +function makePlugin( + saveData: (data: unknown) => Promise = vi.fn().mockResolvedValue(undefined), +): PodNotes { + const { storage } = memorySecretStorage(); + const plugin = Object.create(PodNotes.prototype) as PodNotes; + Object.assign(plugin, { + app: { secretStorage: storage }, + credentials: new CredentialRepository(storage), + isReady: true, + settings: structuredClone(DEFAULT_SETTINGS), + pendingSave: null, + pendingSaveWaiters: [], + saveScheduled: false, + saveChain: Promise.resolve(), + persistenceUnknownFields: {}, + saveData, + }); + return plugin; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("PodNotes persistence integration", () => { + it("loads legacy JSON dates as Date instances", async () => { + const plugin = makePlugin(); + Object.assign(plugin, { + loadData: vi.fn().mockResolvedValue({ + currentEpisode: { + title: "Restored", + streamUrl: "restored.mp3", + url: "", + description: "", + content: "", + podcastName: "Podcast", + episodeDate: "2024-03-01T10:05:03.000Z", + }, + }), + }); + + await plugin.loadSettings(); + + expect(plugin.settings.currentEpisode?.episodeDate).toEqual( + new Date("2024-03-01T10:05:03.000Z"), + ); + }); + + it("refuses future data before any save can overwrite it", async () => { + const saveData = vi.fn().mockResolvedValue(undefined); + const plugin = makePlugin(saveData); + Object.assign(plugin, { + loadData: vi.fn().mockResolvedValue({ schemaVersion: 3 }), + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(plugin.loadSettings()).rejects.toThrow(/schema v3/); + expect(saveData).not.toHaveBeenCalled(); + }); + + it("writes schema v2, canonical dates, and preserved unknown fields", async () => { + const saveData = vi.fn().mockResolvedValue(undefined); + const plugin = makePlugin(saveData); + plugin.settings.currentEpisode = { + title: "Current", + streamUrl: "current.mp3", + url: "", + description: "", + content: "", + podcastName: "Podcast", + episodeDate: new Date("2024-03-01T10:05:03.000Z"), + }; + Object.assign(plugin, { persistenceUnknownFields: { retained: { enabled: true } } }); + + await plugin.saveSettings(); + + expect(saveData).toHaveBeenCalledWith( + expect.objectContaining({ + schemaVersion: 2, + retained: { enabled: true }, + currentEpisode: expect.objectContaining({ + episodeDate: "2024-03-01T10:05:03.000Z", + }), + }), + ); + }); + + it("moves legacy credentials into SecretStorage before writing schema v2", async () => { + const saveData = vi.fn().mockResolvedValue(undefined); + const { storage, values } = memorySecretStorage(); + const plugin = makePlugin(saveData); + Object.assign(plugin, { + app: { secretStorage: storage }, + credentials: new CredentialRepository(storage), + loadData: vi.fn().mockResolvedValue({ + schemaVersion: 1, + openAIApiKey: "sk-legacy", + diarizationApiKey: "dg-legacy", + retained: true, + }), + }); + + await plugin.loadSettings(); + + expect(values.get("podnotes-openai-api-key")).toBe("sk-legacy"); + expect(values.get("podnotes-deepgram-api-key")).toBe("dg-legacy"); + expect(plugin.settings.openAISecretId).toBe("podnotes-openai-api-key"); + expect(plugin.settings.deepgramSecretId).toBe("podnotes-deepgram-api-key"); + expect(saveData).toHaveBeenCalledTimes(1); + const persisted = saveData.mock.calls[0][0] as Record; + expect(persisted).toEqual( + expect.objectContaining({ + schemaVersion: 2, + openAISecretId: "podnotes-openai-api-key", + deepgramSecretId: "podnotes-deepgram-api-key", + retained: true, + }), + ); + expect(persisted).not.toHaveProperty("openAIApiKey"); + expect(persisted).not.toHaveProperty("diarizationApiKey"); + }); + + it("leaves legacy data untouched when SecretStorage migration fails", async () => { + const saveData = vi.fn().mockResolvedValue(undefined); + const raw = { schemaVersion: 1, openAIApiKey: "sk", defaultVolume: 0.3 }; + const storage = { + getSecret: vi.fn(() => null), + setSecret: vi.fn(() => { + throw new Error("keychain unavailable"); + }), + listSecrets: vi.fn(() => []), + } as unknown as SecretStorage; + const plugin = makePlugin(saveData); + Object.assign(plugin, { + app: { secretStorage: storage }, + credentials: new CredentialRepository(storage), + loadData: vi.fn().mockResolvedValue(raw), + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(plugin.loadSettings()).rejects.toThrow("keychain unavailable"); + expect(saveData).not.toHaveBeenCalled(); + expect(raw).toEqual({ schemaVersion: 1, openAIApiKey: "sk", defaultVolume: 0.3 }); + }); + + it("does not save v2 when the second credential write fails", async () => { + const saveData = vi.fn().mockResolvedValue(undefined); + const values = new Map(); + const storage = { + getSecret: vi.fn((id: string) => values.get(id) ?? null), + setSecret: vi.fn((id: string, value: string) => { + if (id.startsWith("podnotes-deepgram")) throw new Error("Deepgram write failed"); + values.set(id, value); + }), + listSecrets: vi.fn(() => [...values.keys()]), + } as unknown as SecretStorage; + const plugin = makePlugin(saveData); + Object.assign(plugin, { + app: { secretStorage: storage }, + credentials: new CredentialRepository(storage), + loadData: vi.fn().mockResolvedValue({ + schemaVersion: 1, + openAIApiKey: "sk-created", + diarizationApiKey: "dg-fails", + }), + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(plugin.loadSettings()).rejects.toThrow("Deepgram write failed"); + expect(values.get("podnotes-openai-api-key")).toBe("sk-created"); + expect(saveData).not.toHaveBeenCalled(); + }); + + it("reuses an already-created secret after the v2 save fails and is retried", async () => { + const { storage, values } = memorySecretStorage(); + const raw = { schemaVersion: 1, openAIApiKey: "sk-retry" }; + const first = makePlugin(vi.fn().mockRejectedValue(new Error("disk full"))); + Object.assign(first, { + app: { secretStorage: storage }, + credentials: new CredentialRepository(storage), + loadData: vi.fn().mockResolvedValue(raw), + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(first.loadSettings()).rejects.toThrow("disk full"); + expect(values.get("podnotes-openai-api-key")).toBe("sk-retry"); + + const retrySave = vi.fn().mockResolvedValue(undefined); + const retry = makePlugin(retrySave); + Object.assign(retry, { + app: { secretStorage: storage }, + credentials: new CredentialRepository(storage), + loadData: vi.fn().mockResolvedValue(raw), + }); + await retry.loadSettings(); + + expect(retry.settings.openAISecretId).toBe("podnotes-openai-api-key"); + expect(values.has("podnotes-openai-api-key-2")).toBe(false); + expect(retrySave).toHaveBeenCalledWith( + expect.objectContaining({ + schemaVersion: 2, + openAISecretId: "podnotes-openai-api-key", + }), + ); + }); + + it("does not rewrite data that is already schema v2", async () => { + const saveData = vi.fn().mockResolvedValue(undefined); + const plugin = makePlugin(saveData); + Object.assign(plugin, { + loadData: vi.fn().mockResolvedValue({ schemaVersion: 2, defaultVolume: 0.4 }), + }); + + await plugin.loadSettings(); + + expect(plugin.settings.defaultVolume).toBe(0.4); + expect(saveData).not.toHaveBeenCalled(); + }); + + it("scrubs retired plaintext fields from schema v2 without importing them", async () => { + const saveData = vi.fn().mockResolvedValue(undefined); + const { storage, values } = memorySecretStorage(); + const plugin = makePlugin(saveData); + Object.assign(plugin, { + app: { secretStorage: storage }, + credentials: new CredentialRepository(storage), + loadData: vi.fn().mockResolvedValue({ + schemaVersion: 2, + openAIApiKey: "must-not-import", + defaultVolume: 0.4, + }), + }); + + await plugin.loadSettings(); + + expect(values.size).toBe(0); + expect(plugin.settings.openAISecretId).toBe(""); + expect(saveData).toHaveBeenCalledTimes(1); + expect(saveData.mock.calls[0][0]).toEqual( + expect.objectContaining({ schemaVersion: 2, defaultVolume: 0.4 }), + ); + expect(saveData.mock.calls[0][0]).not.toHaveProperty("openAIApiKey"); + }); + + it("fails closed if a v2 retired-field scrub cannot be persisted", async () => { + const plugin = makePlugin(vi.fn().mockRejectedValue(new Error("disk full"))); + Object.assign(plugin, { + loadData: vi + .fn() + .mockResolvedValue({ schemaVersion: 2, openAIApiKey: "must-stay-unread" }), + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(plugin.loadSettings()).rejects.toThrow("disk full"); + expect(plugin.settings).toEqual(DEFAULT_SETTINGS); + }); + + it("rejects a strict caller when saveData fails", async () => { + const failure = new Error("disk full"); + const plugin = makePlugin(vi.fn().mockRejectedValue(failure)); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(plugin.saveSettingsStrict()).rejects.toBe(failure); + }); + + it("keeps best-effort saves nonrejecting while logging disk failure", async () => { + const failure = new Error("disk full"); + const plugin = makePlugin(vi.fn().mockRejectedValue(failure)); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect(plugin.saveSettings()).resolves.toBeUndefined(); + expect(consoleError).toHaveBeenCalledWith("PodNotes: failed to save settings", failure); + }); + + it("normalizes a synchronous snapshot failure into the two save contracts", async () => { + const failure = new Error("cannot clone"); + const plugin = makePlugin(); + vi.spyOn(globalThis, "structuredClone").mockImplementation(() => { + throw failure; + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + let strict: Promise | undefined; + expect(() => { + strict = plugin.saveSettingsStrict(); + }).not.toThrow(); + await expect(strict).rejects.toBe(failure); + await expect(plugin.saveSettings()).resolves.toBeUndefined(); + }); + + it("keeps later callers pending until their newer snapshot is durable", async () => { + const resolvers: Array<() => void> = []; + const writes: unknown[] = []; + const plugin = makePlugin( + vi.fn((data: unknown) => { + writes.push(data); + return new Promise((resolve) => resolvers.push(resolve)); + }), + ); + + const first = plugin.saveSettingsStrict(); + await vi.waitFor(() => expect(writes).toHaveLength(1)); + plugin.settings.defaultVolume = 0.25; + const second = plugin.saveSettingsStrict(); + let secondResolved = false; + void second.then(() => { + secondResolved = true; + }); + + resolvers[0](); + await first; + await vi.waitFor(() => expect(writes).toHaveLength(2)); + expect(secondResolved).toBe(false); + expect(writes[1]).toEqual(expect.objectContaining({ defaultVolume: 0.25 })); + + resolvers[1](); + await second; + expect(secondResolved).toBe(true); + }); +}); diff --git a/src/main.ts b/src/main.ts index eabfc63c..8b450b89 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,7 +9,6 @@ import { queue, savedFeeds, hidePlayedEpisodes, - sanitizeEpisodeListLimit, playbackRate, plugin, volume, @@ -20,14 +19,7 @@ import { registerCommands } from "src/commands"; import { Notice, Platform, Plugin, type Editor, type WorkspaceLeaf } from "obsidian"; import { API } from "src/API/API"; import type { IAPI } from "src/API/IAPI"; -import { DEFAULT_SETTINGS, VIEW_TYPE } from "src/constants"; -import { - migrateDownloadPath, - migrateFeedNoteSettings, - migrateNoteSettings, - migrateSkipLength, - migrateTranscriptSettings, -} from "src/settingsMigrations"; +import { VIEW_TYPE } from "src/constants"; import { PodNotesSettingsTab } from "src/ui/settings/PodNotesSettingsTab"; import { MainView } from "src/ui/PodcastView"; import type { IPodNotesSettings } from "./types/IPodNotesSettings"; @@ -42,6 +34,13 @@ import type { IconType } from "./types/IconType"; import { TranscriptionService } from "./services/TranscriptionService"; import { type Unsubscriber } from "svelte/store"; import { normalizePlaybackRate } from "./utility/playbackRate"; +import { + decodePodNotesData, + encodePodNotesData, + PODNOTES_DATA_SCHEMA_VERSION, + PodNotesDataError, +} from "./persistence/podNotesData"; +import { CredentialRepository } from "./services/CredentialRepository"; type MediaSessionActionName = | "previoustrack" @@ -54,10 +53,26 @@ type MediaSessionActionName = | "seekto" | "skipad"; +interface SaveWaiter { + resolve: () => void; + reject: (error: unknown) => void; +} + +class PodNotesSchemaMigrationError extends Error { + constructor( + public readonly originalCause: unknown, + public readonly includedCredentials: boolean, + ) { + super(originalCause instanceof Error ? originalCause.message : String(originalCause)); + this.name = "PodNotesSchemaMigrationError"; + } +} + export default class PodNotes extends Plugin implements IPodNotes { public api!: IAPI; public override settings!: IPodNotesSettings; public override app!: PartialAppExtension; + public credentials!: CredentialRepository; private views = new Set(); @@ -76,14 +91,17 @@ export default class PodNotes extends Plugin implements IPodNotes { private podcastViewMountEnabled = true; private isReady = false; private pendingSave: IPodNotesSettings | null = null; + private pendingSaveWaiters: SaveWaiter[] = []; private saveScheduled = false; private saveChain: Promise = Promise.resolve(); + private persistenceUnknownFields: Record = {}; private mediaSessionActions: MediaSessionActionName[] = []; override async onload() { this.isUnloaded = false; this.podcastViewMountEnabled = !this.isMobileRuntime(); plugin.set(this); + this.credentials = new CredentialRepository(this.app.secretStorage); await this.loadSettings(); @@ -286,6 +304,10 @@ export default class PodNotes extends Plugin implements IPodNotes { return this.transcriptionService; } + invalidateTranscriptionCredentialCache(): void { + this.transcriptionService?.clearCredentialCache(); + } + captureTimestamp(editor: Editor | null | undefined): boolean { if (!editor || !this.api.podcast || !this.settings.timestamp.template) { return false; @@ -397,6 +419,8 @@ export default class PodNotes extends Plugin implements IPodNotes { this.storeUnsubscribers = []; this.volumeUnsubscribe?.(); this.localFilesMirrorUnsubscribe?.(); + this.transcriptionService?.dispose(); + this.transcriptionService = undefined; this.views.clear(); // Intentionally do NOT detach the view's leaves here. Obsidian persists and @@ -413,84 +437,127 @@ export default class PodNotes extends Plugin implements IPodNotes { } async loadSettings() { - const loadedData = await this.loadData(); - - this.settings = Object.assign({}, DEFAULT_SETTINGS, loadedData); - this.settings.timestamp = { - ...DEFAULT_SETTINGS.timestamp, - ...loadedData?.timestamp, - }; - // Build a fresh download object so we never mutate the shared - // DEFAULT_SETTINGS.download, then migrate the legacy empty default (#183). - this.settings.download = { - ...DEFAULT_SETTINGS.download, - ...loadedData?.download, - }; - this.settings.download.path = migrateDownloadPath(this.settings.download.path); - // Normalise the persisted limit so a malformed value (e.g. 0 from an older - // data.json) is repaired in the settings object too, not just clamped for - // runtime behaviour, and so a later saveSettings() can't re-persist it (#114). - this.settings.episodeListLimit = sanitizeEpisodeListLimit(this.settings.episodeListLimit); - // Repair persisted skip lengths so a cleared field (NaN -> null in JSON) - // can't feed the skip arithmetic and corrupt the playback position (PB-02). - this.settings.skipBackwardLength = migrateSkipLength( - this.settings.skipBackwardLength, - DEFAULT_SETTINGS.skipBackwardLength, - ); - this.settings.skipForwardLength = migrateSkipLength( - this.settings.skipForwardLength, - DEFAULT_SETTINGS.skipForwardLength, - ); - // Self-heal a corrupt persisted default playback rate so the loaded store - // and the settings slider both read a clamped value (ST-02). - this.settings.defaultPlaybackRate = normalizePlaybackRate( - this.settings.defaultPlaybackRate, - ); - // Upgrade the legacy empty episode-note default to the Bases-friendly - // default, preserving any path/template the user configured (#160). Returns - // a fresh object, so DEFAULT_SETTINGS.note is never mutated. - this.settings.note = migrateNoteSettings(loadedData?.note); - // Backfill the diarization defaults onto the stored transcript object so an - // existing user (who has only { path, template } persisted) gets a valid - // transcript.diarization instead of undefined (#168). - this.settings.transcript = migrateTranscriptSettings(loadedData?.transcript); - // Backfill a partial/legacy feedNote so a missing template can't crash - // createFeedNote's `template.replace(...)` (ST-08). - this.settings.feedNote = migrateFeedNoteSettings(loadedData?.feedNote); + try { + const decoded = decodePodNotesData(await this.loadData()); + let settings = decoded.settings; + this.credentials ??= new CredentialRepository(this.app.secretStorage); + + if ( + decoded.sourceVersion < PODNOTES_DATA_SCHEMA_VERSION || + decoded.retiredPlaintextPresent + ) { + // SecretStorage writes are synchronous. Store every legacy value and + // verify it first, then write the v2 snapshot directly because the normal + // save queue intentionally stays dormant until onload finishes. A v2 file + // carrying retired fields is scrubbed without importing their values. If + // any step fails, the old data.json is left untouched and loading stops. + try { + const references = + decoded.sourceVersion < PODNOTES_DATA_SCHEMA_VERSION + ? this.credentials.storeValues(decoded.legacySecrets) + : {}; + settings = { ...settings, ...references }; + await this.saveData(encodePodNotesData(settings, decoded.unknownFields)); + } catch (error) { + throw new PodNotesSchemaMigrationError( + error, + Boolean(decoded.legacySecrets.openAI || decoded.legacySecrets.deepgram), + ); + } + const migratedProviders = [ + decoded.legacySecrets.openAI ? "OpenAI" : null, + decoded.legacySecrets.deepgram ? "Deepgram" : null, + ].filter((provider): provider is string => provider !== null); + if (migratedProviders.length > 0) { + new Notice( + `Moved your ${migratedProviders.join(" and ")} API ${migratedProviders.length === 1 ? "key" : "keys"} into Obsidian SecretStorage.`, + ); + } + } + + this.settings = settings; + this.persistenceUnknownFields = decoded.unknownFields; + + if (decoded.warnings.length > 0) { + console.warn("PodNotes repaired invalid persisted values:", decoded.warnings); + } + } catch (error) { + if (error instanceof PodNotesDataError) { + new Notice(error.message, 0); + console.error("PodNotes refused to load unsafe persisted data", error); + } else if (error instanceof PodNotesSchemaMigrationError) { + new Notice( + error.includedCredentials + ? "PodNotes could not securely migrate its API keys. Your existing data was kept unchanged. Restart PodNotes to retry." + : "PodNotes could not upgrade its settings data. Your existing data was kept unchanged. Restart PodNotes to retry.", + 0, + ); + console.error("PodNotes failed to migrate persisted settings", error.originalCause); + } + throw error; + } + } + + saveSettings(): Promise { + return this.requestSettingsSave().catch(() => undefined); } - async saveSettings() { - if (!this.isReady) return; + /** Awaitable save for migrations/imports that must not report success on failure. */ + saveSettingsStrict(): Promise { + return this.requestSettingsSave(); + } + + private requestSettingsSave(): Promise { + if (!this.isReady) return Promise.resolve(); + + try { + this.pendingSave = this.cloneSettings(); + } catch (error) { + console.error("PodNotes: failed to snapshot settings", error); + const failure = Promise.reject(error); + void failure.catch(() => undefined); + return failure; + } - this.pendingSave = this.cloneSettings(); + const completion = new Promise((resolve, reject) => { + this.pendingSaveWaiters.push({ resolve, reject }); + }); + // Attach a handler so even a mistakenly ignored strict save cannot become an + // unhandled rejection. Awaiting the original promise still receives failure. + void completion.catch(() => undefined); - if (this.saveScheduled) { - return this.saveChain; + if (!this.saveScheduled) { + this.saveScheduled = true; + this.saveChain = this.runSaveLoop(); } - this.saveScheduled = true; + return completion; + } - this.saveChain = this.saveChain - .then(async () => { - while (this.pendingSave) { - const snapshot = this.pendingSave; - this.pendingSave = null; - await this.saveData(snapshot); - } - }) - .catch((error) => { - console.error("PodNotes: failed to save settings", error); - }) - .finally(() => { - this.saveScheduled = false; - - // If a save was requested while we were saving, run again. - if (this.pendingSave) { - void this.saveSettings(); + private async runSaveLoop(): Promise { + try { + while (this.pendingSave) { + const snapshot = this.pendingSave; + const waiters = this.pendingSaveWaiters; + this.pendingSave = null; + this.pendingSaveWaiters = []; + + try { + await this.saveData( + encodePodNotesData(snapshot, this.persistenceUnknownFields), + ); + for (const waiter of waiters) waiter.resolve(); + } catch (error) { + console.error("PodNotes: failed to save settings", error); + for (const waiter of waiters) waiter.reject(error); } - }); - - return this.saveChain; + } + } finally { + // No await occurs between the loop's empty check and this assignment, so a + // request cannot land in a gap where it sees a scheduled loop that has + // already decided to exit. + this.saveScheduled = false; + } } private cloneSettings(): IPodNotesSettings { diff --git a/src/parser/feedParser.test.ts b/src/parser/feedParser.test.ts index b3f8ad66..9acaaf93 100644 --- a/src/parser/feedParser.test.ts +++ b/src/parser/feedParser.test.ts @@ -227,6 +227,42 @@ describe("FeedParser", () => { expect(feed.title).toBe("Test Podcast"); expect(feed.url).toBe("https://example.com/feed.xml"); + expect(feed.feedId).toMatch(/^pnf1_[A-Za-z0-9_-]{43}$/); + }); + + test("retains the direct channel GUID as evidence without using it as feed identity", async () => { + const feedWithGuid = ` + + + GUID Podcast + channel-guid + +`; + mockRequestWithTimeout + .mockResolvedValueOnce(feedResponse(feedWithGuid)) + .mockResolvedValueOnce(feedResponse(feedWithGuid)); + + const first = await new FeedParser().getFeed("https://example.com/one.xml"); + const second = await new FeedParser().getFeed("https://example.com/two.xml"); + + expect(first.guid).toBe("channel-guid"); + expect(second.guid).toBe("channel-guid"); + expect(first.feedId).not.toBe(second.feedId); + }); + + test("preserves a constructor feed ID across a confirmed URL and title refresh", async () => { + mockRequestWithTimeout.mockResolvedValueOnce(feedResponse(sampleRssFeed)); + const existing = await new FeedParser().getFeed("https://example.com/original.xml"); + const renamedFeed = sampleRssFeed.replace("Test Podcast", "Renamed Podcast"); + mockRequestWithTimeout.mockResolvedValueOnce(feedResponse(renamedFeed)); + + const refreshed = await new FeedParser(existing).getFeed( + "https://redirected.example.com/feed.xml", + ); + + expect(refreshed.title).toBe("Renamed Podcast"); + expect(refreshed.url).toBe("https://redirected.example.com/feed.xml"); + expect(refreshed.feedId).toBe(existing.feedId); }); test("parses artwork URL from image element", async () => { @@ -396,6 +432,9 @@ describe("FeedParser", () => { expect(episode.title).toBe("Episode 1"); expect(episode.streamUrl).toBe("https://example.com/episode1.mp3"); expect(episode.url).toBe("https://example.com/episode1"); + expect(episode.itemLink).toBe("https://example.com/episode1"); + expect(episode.feedId).toMatch(/^pnf1_[A-Za-z0-9_-]{43}$/); + expect(episode.episodeId).toMatch(/^pne1_[A-Za-z0-9_-]{43}$/); expect(episode.description).toBe("First episode description"); expect(episode.episodeDate).toEqual(new Date("Mon, 01 Jan 2024 00:00:00 GMT")); expect(episode.itunesTitle).toBe("Episode 1 iTunes Title"); @@ -404,6 +443,122 @@ describe("FeedParser", () => { expect(episode.feedUrl).toBe("https://example.com/feed.xml"); }); + test("excludes a duplicated direct-child GUID and assigns distinct episode IDs", async () => { + const duplicateGuidFeed = ` + + + Duplicate GUID Podcast + + Episode One + duplicate-guid + + Mon, 01 Jan 2024 00:00:00 GMT + + + Episode Two + duplicate-guid + + Tue, 02 Jan 2024 00:00:00 GMT + + +`; + mockRequestWithTimeout.mockResolvedValueOnce(feedResponse(duplicateGuidFeed)); + + const episodes = await new FeedParser().getEpisodes("https://example.com/feed.xml"); + + expect(episodes.map((episode) => episode.guid)).toEqual([ + "duplicate-guid", + "duplicate-guid", + ]); + expect(episodes[0].episodeId).toMatch(/^pne1_[A-Za-z0-9_-]{43}$/); + expect(episodes[1].episodeId).toMatch(/^pne1_[A-Za-z0-9_-]{43}$/); + expect(episodes[0].episodeId).not.toBe(episodes[1].episodeId); + }); + + test("reconciles a newly appearing GUID through one-to-one media evidence", async () => { + const initialFeed = ` + + + Initial Podcast + + Initial Episode + + https://example.com/stable + Mon, 01 Jan 2024 00:00:00 GMT + + +`; + const refreshedFeed = initialFeed + .replace("Initial Podcast", "Renamed Podcast") + .replace("Initial Episode", "Renamed Episode") + .replace( + '', + 'appeared-guid', + ); + mockRequestWithTimeout.mockResolvedValueOnce(feedResponse(initialFeed)); + const previousEpisodes = await new FeedParser().getEpisodes( + "https://example.com/original.xml", + ); + const previousFeed: PodcastFeed = { + title: "Initial Podcast", + url: "https://example.com/original.xml", + artworkUrl: "", + feedId: previousEpisodes[0].feedId, + }; + mockRequestWithTimeout.mockResolvedValueOnce(feedResponse(refreshedFeed)); + + const refreshed = await new FeedParser(previousFeed, previousEpisodes).getEpisodes( + "https://redirected.example.com/feed.xml", + ); + + expect(refreshed[0].guid).toBe("appeared-guid"); + expect(refreshed[0].episodeId).toBe(previousEpisodes[0].episodeId); + expect(refreshed[0].episodeAliases).toContain(previousEpisodes[0].episodeId); + }); + + test("uses only an actual direct item link as itemLink identity evidence", async () => { + const nestedLinkFeed = ` + + + Nested Link Podcast + + Episode + + https://attacker.example/nested + Mon, 01 Jan 2024 00:00:00 GMT + + +`; + mockRequestWithTimeout.mockResolvedValueOnce(feedResponse(nestedLinkFeed)); + + const [episode] = await new FeedParser().getEpisodes("https://example.com/feed.xml"); + + expect(episode.itemLink).toBeUndefined(); + expect(episode.url).toBe("https://example.com/feed.xml"); + }); + + test("keeps an episode with a malformed publication date but omits the date", async () => { + const malformedDateFeed = ` + + + Test Podcast + + Malformed Date Episode + + definitely not a date + + +`; + mockRequestWithTimeout.mockResolvedValueOnce(feedResponse(malformedDateFeed)); + + const parser = new FeedParser(); + const episodes = await parser.getEpisodes("https://example.com/feed.xml"); + + expect(episodes).toHaveLength(1); + expect(episodes[0]).toMatchObject({ title: "Malformed Date Episode" }); + expect(episodes[0].episodeDate).toBeUndefined(); + }); + test("parses Podcasting 2.0 chapter URLs from episodes (#47)", async () => { mockRequestWithTimeout.mockResolvedValueOnce(feedResponse(rssFeedWithPodcastChapters)); @@ -623,6 +778,7 @@ describe("FeedParser", () => { expect(episodes[0].content).toBe(""); // url falls back to feed.url when episode link is missing expect(episodes[0].url).toBe("https://example.com/feed.xml"); + expect(episodes[0].itemLink).toBeUndefined(); }); test("handles CDATA content in description", async () => { diff --git a/src/parser/feedParser.ts b/src/parser/feedParser.ts index 49ec82b7..06ae7f58 100644 --- a/src/parser/feedParser.ts +++ b/src/parser/feedParser.ts @@ -1,5 +1,6 @@ import type { PodcastFeed } from "src/types/PodcastFeed"; import type { Episode } from "src/types/Episode"; +import { decodeDate } from "src/persistence/dateCodec"; import { requestWithTimeout } from "src/utility/networkRequest"; import { parseEpisodeNumber } from "src/utility/parseEpisodeNumber"; import { parseDurationToSeconds } from "src/utility/parseDuration"; @@ -7,12 +8,22 @@ import { getMediaTypeFromContentType, getUnambiguousMediaTypeFromPath, } from "src/utility/mediaType"; +import { + assignEpisodeIdentities, + createFeedId, + isCanonicalFeedId, + reconcileEpisodeIdentities, + type EpisodeIdentitySource, + type PreviousEpisodeIdentity, +} from "src/utility/identity"; export default class FeedParser { private feed: PodcastFeed | undefined; + private previousEpisodes: readonly Episode[]; - constructor(feed?: PodcastFeed) { + constructor(feed?: PodcastFeed, previousEpisodes: readonly Episode[] = []) { this.feed = feed; + this.previousEpisodes = Array.isArray(previousEpisodes) ? previousEpisodes : []; } public async getEpisodes(url: string): Promise { @@ -27,6 +38,9 @@ export default class FeedParser { // the podcast name and the per-episode artwork fallback. if (!this.feed || this.feed.url !== url) { this.feed = this.extractFeed(body, url); + } else if (!isCanonicalFeedId(this.feed.feedId)) { + const feedId = createFeedId(url); + if (feedId) this.feed = { ...this.feed, feedId }; } return this.parsePage(body); @@ -66,6 +80,10 @@ export default class FeedParser { url, artworkUrl, }; + const feedId = isCanonicalFeedId(this.feed?.feedId) ? this.feed.feedId : createFeedId(url); + if (feedId) feed.feedId = feedId; + const guid = this.findDirectChildText(channel, ["podcast:guid"]); + if (guid) feed.guid = guid; const link = this.findFeedLink(channel); if (link) feed.link = link; @@ -161,20 +179,73 @@ export default class FeedParser { return ""; } - protected parsePage(page: Document): Episode[] { - const items = page.querySelectorAll("item"); + private findDirectChild(scope: Document | Element, tagName: string): Element | undefined { + const wanted = tagName.toLowerCase(); + return Array.from(scope.children).find((child) => child.tagName.toLowerCase() === wanted); + } - function isEpisode(ep: Episode | null): ep is Episode { - return !!ep; - } + private episodeIdentitySource( + episode: Episode, + publishedAtFallback = "", + ): EpisodeIdentitySource { + const publishedAt = + episode.episodeDate instanceof Date && Number.isFinite(episode.episodeDate.getTime()) + ? episode.episodeDate.toISOString() + : publishedAtFallback; + return { + feedId: episode.feedId ?? this.feed?.feedId ?? "", + guid: episode.guid, + enclosureUrl: episode.streamUrl, + itemLink: episode.itemLink, + publishedAt, + title: episode.title, + }; + } - return Array.from(items).map(this.parseItem.bind(this)).filter(isEpisode); + protected parsePage(page: Document): Episode[] { + const items = page.querySelectorAll("item"); + const parsed = Array.from(items).flatMap((item) => { + const episode = this.parseItem(item); + if (!episode) return []; + + return [ + { + episode, + identitySource: this.episodeIdentitySource( + episode, + this.findDirectChildText(item, ["pubDate"]), + ), + }, + ]; + }); + const currentSources = parsed.map(({ identitySource }) => identitySource); + const previous: PreviousEpisodeIdentity[] = this.previousEpisodes.map((episode) => ({ + episodeId: episode.episodeId, + source: this.episodeIdentitySource(episode), + aliases: episode.episodeAliases, + })); + const identities = previous.length + ? reconcileEpisodeIdentities(previous, currentSources) + : assignEpisodeIdentities(currentSources); + + return parsed.map(({ episode }, index) => { + const identity = identities[index]; + return identity?.episodeId + ? { + ...episode, + episodeId: identity.episodeId, + ...(identity.aliases.length + ? { episodeAliases: [...identity.aliases] } + : {}), + } + : episode; + }); } protected parseItem(item: Element): Episode | null { const titleEl = item.querySelector("title"); const streamUrlEl = item.querySelector("enclosure"); - const linkEl = item.querySelector("link"); + const linkEl = this.findDirectChild(item, "link"); const descriptionEl = item.querySelector("description"); const contentEl = item.querySelector("*|encoded"); const pubDateEl = item.querySelector("pubDate"); @@ -183,6 +254,7 @@ export default class FeedParser { const itunesEpisodeEl = item.getElementsByTagName("itunes:episode")[0]; const itunesDurationEl = item.getElementsByTagName("itunes:duration")[0]; const chaptersEl = item.getElementsByTagName("podcast:chapters")[0]; + const guid = this.findDirectChildText(item, ["guid"]); if (!titleEl || !streamUrlEl || !pubDateEl) { return null; @@ -191,10 +263,10 @@ export default class FeedParser { const title = titleEl.textContent || ""; const streamUrl = streamUrlEl.getAttribute("url") || ""; const enclosureType = streamUrlEl.getAttribute("type"); - const url = linkEl?.textContent || ""; + const itemLink = linkEl?.textContent || ""; const description = descriptionEl?.textContent || ""; const content = contentEl?.textContent || ""; - const pubDate = new Date(pubDateEl.textContent as string); + const pubDate = decodeDate(pubDateEl.textContent); const artworkUrl = itunesImageEl?.getAttribute("href") || this.feed?.artworkUrl; const itunesTitle = itunesTitleEl?.textContent; const episodeNumber = parseEpisodeNumber(itunesEpisodeEl?.textContent, title); @@ -203,14 +275,17 @@ export default class FeedParser { return { title, + guid: guid || undefined, streamUrl, - url: url || this.feed?.url || "", + url: itemLink || this.feed?.url || "", + itemLink: itemLink || undefined, description, content, podcastName: this.feed?.title || "", artworkUrl, episodeDate: pubDate, feedUrl: this.feed?.url || "", + feedId: isCanonicalFeedId(this.feed?.feedId) ? this.feed.feedId : undefined, itunesTitle: itunesTitle || "", episodeNumber, duration, diff --git a/src/persistence/codecUtils.ts b/src/persistence/codecUtils.ts new file mode 100644 index 00000000..11ad9e57 --- /dev/null +++ b/src/persistence/codecUtils.ts @@ -0,0 +1,195 @@ +export type UnknownRecord = Record; + +export const DANGEROUS_KEYS = new Set(["__proto__", "constructor", "prototype"]); + +export function readString( + record: UnknownRecord, + key: string, + fallback: string, + warnings: Set, + basePath = "", +): string { + const value = record[key]; + if (value === undefined) return fallback; + if (typeof value === "string") return value; + warn(warnings, joinPath(basePath, key), "expected a string"); + return fallback; +} + +export function readNullableString( + record: UnknownRecord, + key: string, + warnings: Set, + basePath: string, +): string | null | undefined { + const value = record[key]; + if (value === undefined || value === null || typeof value === "string") return value; + warn(warnings, joinPath(basePath, key), "expected a string"); + return undefined; +} + +export function readBoolean( + record: UnknownRecord, + key: string, + fallback: boolean, + warnings: Set, + basePath = "", +): boolean { + const value = record[key]; + if (value === undefined) return fallback; + if (typeof value === "boolean") return value; + warn(warnings, joinPath(basePath, key), "expected a boolean"); + return fallback; +} + +export function readFiniteNumber( + record: UnknownRecord, + key: string, + fallback: number, + warnings: Set, + basePath = "", +): number { + const value = record[key]; + if (value === undefined) return fallback; + if (typeof value === "number" && Number.isFinite(value)) return value; + warn(warnings, joinPath(basePath, key), "expected a finite number"); + return fallback; +} + +export function readNonNegativeNumber( + record: UnknownRecord, + key: string, + fallback: number, + warnings: Set, + basePath: string, +): number { + const value = record[key]; + if (value === undefined) return fallback; + if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value; + warn(warnings, joinPath(basePath, key), "expected a non-negative number"); + return fallback; +} + +export function readPositiveNumber( + record: UnknownRecord, + key: string, + fallback: number, + warnings: Set, + basePath: string, +): number { + const value = record[key]; + if (value === undefined) return fallback; + if (typeof value === "number" && Number.isFinite(value) && value > 0) return value; + warn(warnings, joinPath(basePath, key), "expected a positive number"); + return fallback; +} + +export function readClampedNumber( + record: UnknownRecord, + key: string, + fallback: number, + minimum: number, + maximum: number, + warnings: Set, +): number { + const value = record[key]; + if (value === undefined) return fallback; + if (typeof value !== "number" || !Number.isFinite(value)) { + warn(warnings, key, "expected a finite number"); + return fallback; + } + const clamped = Math.min(maximum, Math.max(minimum, value)); + if (clamped !== value) warn(warnings, key, "value was clamped"); + return clamped; +} + +export function setOptionalString( + target: UnknownRecord, + source: UnknownRecord, + key: string, + warnings: Set, + basePath: string, +): void { + const value = source[key]; + if (value === undefined || value === null) { + delete target[key]; + } else if (typeof value === "string") { + target[key] = value; + } else { + delete target[key]; + warn(warnings, joinPath(basePath, key), "expected a string"); + } +} + +export function setOptionalFiniteNumber( + target: UnknownRecord, + source: UnknownRecord, + key: string, + warnings: Set, + basePath: string, + minimum: number, +): void { + const value = source[key]; + if (value === undefined || value === null) { + delete target[key]; + } else if (typeof value === "number" && Number.isFinite(value) && value >= minimum) { + target[key] = value; + } else { + delete target[key]; + warn(warnings, joinPath(basePath, key), `expected a finite number >= ${minimum}`); + } +} + +export function optionalRecord(value: unknown, warnings: Set, path: string): UnknownRecord { + if (value === undefined || value === null) return {}; + if (isPlainObject(value)) return value; + warn(warnings, path, "expected an object"); + return {}; +} + +export function copySafeObject(value: UnknownRecord): UnknownRecord { + const copy: UnknownRecord = {}; + for (const [key, field] of Object.entries(value)) { + if (!DANGEROUS_KEYS.has(key)) copy[key] = field; + } + return copy; +} + +export function safeEntries( + value: UnknownRecord, + warnings: Set, + path: string, +): [string, unknown][] { + const entries: [string, unknown][] = []; + for (const [key, field] of Object.entries(value)) { + if (DANGEROUS_KEYS.has(key)) { + warn(warnings, `${path}.${key}`, "unsafe key was removed"); + continue; + } + entries.push([key, field]); + } + return entries; +} + +export function mapRecord( + value: Record, + mapper: (entry: T) => R, +): Record { + const result: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (!DANGEROUS_KEYS.has(key)) result[key] = mapper(entry); + } + return result; +} + +export function isPlainObject(value: unknown): value is UnknownRecord { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function warn(warnings: Set, path: string, reason: string): void { + warnings.add(`${path}: ${reason}`); +} + +function joinPath(basePath: string, key: string): string { + return basePath ? `${basePath}.${key}` : key; +} diff --git a/src/persistence/collectionCodecs.ts b/src/persistence/collectionCodecs.ts new file mode 100644 index 00000000..9b051aa8 --- /dev/null +++ b/src/persistence/collectionCodecs.ts @@ -0,0 +1,185 @@ +import type DownloadedEpisode from "src/types/DownloadedEpisode"; +import type { Playlist } from "src/types/Playlist"; +import type { PodcastFeed } from "src/types/PodcastFeed"; +import type { PodNote } from "src/types/PodNotes"; +import type { PlayedEpisode } from "src/types/PlayedEpisode"; +import { + copySafeObject, + isPlainObject, + readBoolean, + readNonNegativeNumber, + readString, + safeEntries, + setOptionalString, + warn, +} from "./codecUtils"; +import { decodeEpisode, decodePlaylist } from "./episodeCodec"; + +export function decodeSavedFeeds( + value: unknown, + warnings: Set, +): Record { + if (!isPlainObject(value)) { + if (value !== undefined) warn(warnings, "savedFeeds", "expected an object"); + return {}; + } + + const feeds: Record = {}; + for (const [key, candidate] of safeEntries(value, warnings, "savedFeeds")) { + if (!isPlainObject(candidate)) { + warn(warnings, `savedFeeds.${key}`, "expected an object; feed was skipped"); + continue; + } + + const feed = copySafeObject(candidate); + feed.title = readString(candidate, "title", key, warnings, `savedFeeds.${key}`); + feed.url = readString(candidate, "url", "", warnings, `savedFeeds.${key}`); + feed.artworkUrl = readString(candidate, "artworkUrl", "", warnings, `savedFeeds.${key}`); + setOptionalString(feed, candidate, "description", warnings, `savedFeeds.${key}`); + setOptionalString(feed, candidate, "link", warnings, `savedFeeds.${key}`); + setOptionalString(feed, candidate, "author", warnings, `savedFeeds.${key}`); + if (typeof candidate.collectionId === "number" && Number.isFinite(candidate.collectionId)) { + feed.collectionId = String(candidate.collectionId); + warn(warnings, `savedFeeds.${key}.collectionId`, "number was converted to text"); + } else { + setOptionalString(feed, candidate, "collectionId", warnings, `savedFeeds.${key}`); + } + feeds[key] = feed as unknown as PodcastFeed; + } + return feeds; +} + +export function decodePodNotes(value: unknown, warnings: Set): Record { + if (!isPlainObject(value)) { + if (value !== undefined) warn(warnings, "podNotes", "expected an object"); + return {}; + } + + const notes: Record = {}; + for (const [key, candidate] of safeEntries(value, warnings, "podNotes")) { + if (!isPlainObject(candidate)) { + warn(warnings, `podNotes.${key}`, "expected an object; note mapping was skipped"); + continue; + } + notes[key] = { + ...copySafeObject(candidate), + episodeName: readString(candidate, "episodeName", key, warnings, `podNotes.${key}`), + filePath: readString(candidate, "filePath", "", warnings, `podNotes.${key}`), + podcastFeedKey: readString( + candidate, + "podcastFeedKey", + "", + warnings, + `podNotes.${key}`, + ), + } as PodNote; + } + return notes; +} + +export function decodePlayedEpisodes( + value: unknown, + warnings: Set, +): Record { + if (!isPlainObject(value)) { + if (value !== undefined) warn(warnings, "playedEpisodes", "expected an object"); + return {}; + } + + const played: Record = {}; + for (const [key, candidate] of safeEntries(value, warnings, "playedEpisodes")) { + if (!isPlainObject(candidate)) { + warn(warnings, `playedEpisodes.${key}`, "expected an object; progress was skipped"); + continue; + } + played[key] = { + ...copySafeObject(candidate), + title: readString(candidate, "title", key, warnings, `playedEpisodes.${key}`), + podcastName: readString( + candidate, + "podcastName", + "", + warnings, + `playedEpisodes.${key}`, + ), + time: readNonNegativeNumber(candidate, "time", 0, warnings, `playedEpisodes.${key}`), + duration: readNonNegativeNumber( + candidate, + "duration", + 0, + warnings, + `playedEpisodes.${key}`, + ), + finished: readBoolean(candidate, "finished", false, warnings, `playedEpisodes.${key}`), + } as PlayedEpisode; + } + return played; +} + +export function decodePlaylists(value: unknown, warnings: Set): Record { + if (!isPlainObject(value)) { + if (value !== undefined) warn(warnings, "playlists", "expected an object"); + return {}; + } + + const playlists: Record = {}; + for (const [key, candidate] of safeEntries(value, warnings, "playlists")) { + if (!isPlainObject(candidate)) { + warn(warnings, `playlists.${key}`, "expected an object; playlist was skipped"); + continue; + } + playlists[key] = decodePlaylist( + candidate, + { + name: key, + icon: "list", + episodes: [], + shouldEpisodeRemoveAfterPlay: false, + shouldRepeat: false, + }, + warnings, + `playlists.${key}`, + ); + } + return playlists; +} + +export function decodeDownloadedEpisodes( + value: unknown, + warnings: Set, +): Record { + if (!isPlainObject(value)) { + if (value !== undefined) warn(warnings, "downloadedEpisodes", "expected an object"); + return {}; + } + + const downloads: Record = {}; + for (const [podcastName, candidate] of safeEntries(value, warnings, "downloadedEpisodes")) { + if (!Array.isArray(candidate)) { + warn( + warnings, + `downloadedEpisodes.${podcastName}`, + "expected an array; download list was skipped", + ); + continue; + } + + downloads[podcastName] = candidate.flatMap((entry, index) => { + const path = `downloadedEpisodes.${podcastName}[${index}]`; + const decoded = decodeEpisode(entry, warnings, path); + if (!decoded || !isPlainObject(entry) || typeof entry.filePath !== "string") { + if (decoded) + warn(warnings, `${path}.filePath`, "missing path; download was skipped"); + return []; + } + + const size = + typeof entry.size === "number" && Number.isFinite(entry.size) && entry.size >= 0 + ? entry.size + : 0; + if (size !== entry.size) warn(warnings, `${path}.size`, "value was normalized"); + return [{ ...decoded, filePath: entry.filePath, size } as DownloadedEpisode]; + }); + } + return downloads; +} diff --git a/src/persistence/dateCodec.test.ts b/src/persistence/dateCodec.test.ts new file mode 100644 index 00000000..7a3e195e --- /dev/null +++ b/src/persistence/dateCodec.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { dateTimestamp, decodeDate, encodeDate } from "./dateCodec"; + +describe("dateCodec", () => { + const iso = "2024-03-01T10:05:03.000Z"; + + it("decodes Date instances and ISO text to cloned valid dates", () => { + const source = new Date(iso); + const fromDate = decodeDate(source); + const fromText = decodeDate(iso); + + expect(fromDate).toEqual(source); + expect(fromDate).not.toBe(source); + expect(fromText).toEqual(source); + }); + + it("encodes valid inputs as canonical ISO-8601 text", () => { + expect(encodeDate(new Date(iso))).toBe(iso); + expect(encodeDate("2024-03-01T11:05:03+01:00")).toBe(iso); + }); + + it.each([undefined, null, "", " ", "not-a-date", 1_709_290_000_000, {}, []])( + "rejects invalid or unsupported input %p without throwing", + (value) => { + expect(decodeDate(value)).toBeUndefined(); + expect(encodeDate(value)).toBeUndefined(); + expect(dateTimestamp(value)).toBeUndefined(); + }, + ); + + it("returns a timestamp only for valid inputs", () => { + expect(dateTimestamp(iso)).toBe(new Date(iso).getTime()); + }); +}); diff --git a/src/persistence/dateCodec.ts b/src/persistence/dateCodec.ts new file mode 100644 index 00000000..5f76943a --- /dev/null +++ b/src/persistence/dateCodec.ts @@ -0,0 +1,34 @@ +/** + * Decode a date crossing a JSON or external-data boundary. + * + * Runtime PodNotes objects use real `Date` instances. Persisted dates use ISO + * strings. Keeping the conversion here prevents the two representations from + * leaking into the rest of the application. + */ +export function decodeDate(value: unknown): Date | undefined { + let decoded: Date; + + if (value instanceof Date) { + decoded = new Date(value.getTime()); + } else if (typeof value === "string" && value.trim() !== "") { + decoded = new Date(value); + } else { + return undefined; + } + + return Number.isFinite(decoded.getTime()) ? decoded : undefined; +} + +/** Encode a runtime date as canonical ISO-8601 text without ever throwing. */ +export function encodeDate(value: unknown): string | undefined { + return decodeDate(value)?.toISOString(); +} + +/** Return a comparable epoch timestamp for a valid date-like value. */ +export function dateTimestamp(value: unknown): number | undefined { + if (value instanceof Date) { + const timestamp = value.getTime(); + return Number.isFinite(timestamp) ? timestamp : undefined; + } + return decodeDate(value)?.getTime(); +} diff --git a/src/persistence/episodeCodec.ts b/src/persistence/episodeCodec.ts new file mode 100644 index 00000000..e2eca5a6 --- /dev/null +++ b/src/persistence/episodeCodec.ts @@ -0,0 +1,175 @@ +import type { Episode } from "src/types/Episode"; +import type { Playlist } from "src/types/Playlist"; +import { + copySafeObject, + isPlainObject, + readBoolean, + readString, + setOptionalFiniteNumber, + setOptionalString, + type UnknownRecord, + warn, +} from "./codecUtils"; +import { decodeDate, encodeDate } from "./dateCodec"; + +export type PersistedEpisode = Omit & { + episodeDate?: string; +} & UnknownRecord; + +export type PersistedPlaylist = Omit & { + episodes: PersistedEpisode[]; + currentEpisode?: PersistedEpisode; +} & UnknownRecord; + +/** Decode a standalone episode at a persistence, import, or cache boundary. */ +export function decodeEpisode( + value: unknown, + warnings: Set = new Set(), + path = "episode", +): Episode | undefined { + if (!isPlainObject(value)) { + warn(warnings, path, "expected an object"); + return undefined; + } + + if (typeof value.title !== "string") { + warn(warnings, `${path}.title`, "expected a string; episode was skipped"); + return undefined; + } + + const decoded = copySafeObject(value); + decoded.title = value.title; + decoded.streamUrl = readString(value, "streamUrl", "", warnings, path); + decoded.url = readString(value, "url", "", warnings, path); + decoded.description = readString(value, "description", "", warnings, path); + decoded.content = readString(value, "content", "", warnings, path); + decoded.podcastName = readString(value, "podcastName", "", warnings, path); + + setOptionalString(decoded, value, "feedUrl", warnings, path); + setOptionalString(decoded, value, "artworkUrl", warnings, path); + setOptionalString(decoded, value, "itunesTitle", warnings, path); + setOptionalString(decoded, value, "chaptersUrl", warnings, path); + // Local and downloaded episode projections carry this structural extension + // while still flowing through Episode-typed playlists. + setOptionalString(decoded, value, "filePath", warnings, path); + + setOptionalFiniteNumber(decoded, value, "episodeNumber", warnings, path, 0); + setOptionalFiniteNumber(decoded, value, "duration", warnings, path, 0); + setOptionalFiniteNumber(decoded, value, "size", warnings, path, 0); + + if (value.mediaType === undefined) { + delete decoded.mediaType; + } else if (value.mediaType === "audio" || value.mediaType === "video") { + decoded.mediaType = value.mediaType; + } else { + delete decoded.mediaType; + warn(warnings, `${path}.mediaType`, "expected audio or video"); + } + + if (value.episodeDate === undefined || value.episodeDate === null || value.episodeDate === "") { + delete decoded.episodeDate; + } else { + const episodeDate = decodeDate(value.episodeDate); + if (episodeDate) { + decoded.episodeDate = episodeDate; + } else { + delete decoded.episodeDate; + warn(warnings, `${path}.episodeDate`, "invalid date was removed"); + } + } + + return decoded as unknown as Episode; +} + +/** Encode an episode while preserving safe structural extension fields. */ +export function encodeEpisode(value: Episode): PersistedEpisode; +export function encodeEpisode(value: undefined): undefined; +export function encodeEpisode(value: Episode | undefined): PersistedEpisode | undefined { + if (!value) return undefined; + + const encoded = copySafeObject(value as unknown as UnknownRecord); + const episodeDate = encodeDate(value.episodeDate); + if (episodeDate) encoded.episodeDate = episodeDate; + else delete encoded.episodeDate; + return encoded as PersistedEpisode; +} + +/** Decode a playlist and every episode snapshot it owns. */ +export function decodePlaylist( + value: unknown, + fallback: Playlist, + warnings: Set = new Set(), + path = "playlist", +): Playlist { + if (!isPlainObject(value)) { + if (value !== undefined) warn(warnings, path, "expected an object"); + return clonePlaylist(fallback); + } + + const decoded = copySafeObject(value); + decoded.icon = readString(value, "icon", fallback.icon, warnings, path); + decoded.name = readString(value, "name", fallback.name, warnings, path); + decoded.shouldEpisodeRemoveAfterPlay = readBoolean( + value, + "shouldEpisodeRemoveAfterPlay", + fallback.shouldEpisodeRemoveAfterPlay, + warnings, + path, + ); + decoded.shouldRepeat = readBoolean( + value, + "shouldRepeat", + fallback.shouldRepeat, + warnings, + path, + ); + + if (Array.isArray(value.episodes)) { + decoded.episodes = value.episodes.flatMap((episode, index) => { + const result = decodeEpisode(episode, warnings, `${path}.episodes[${index}]`); + return result ? [result] : []; + }); + } else { + if (value.episodes !== undefined) warn(warnings, `${path}.episodes`, "expected an array"); + decoded.episodes = fallback.episodes + .map((episode, index) => + decodeEpisode(episode, warnings, `${path}.fallbackEpisodes[${index}]`), + ) + .filter((episode): episode is Episode => Boolean(episode)); + } + + const currentEpisode = + value.currentEpisode === undefined || value.currentEpisode === null + ? undefined + : decodeEpisode(value.currentEpisode, warnings, `${path}.currentEpisode`); + if (currentEpisode) decoded.currentEpisode = currentEpisode; + else delete decoded.currentEpisode; + + if (value.isVirtual === undefined) { + delete decoded.isVirtual; + } else if (typeof value.isVirtual === "boolean") { + decoded.isVirtual = value.isVirtual; + } else { + delete decoded.isVirtual; + warn(warnings, `${path}.isVirtual`, "expected a boolean"); + } + + return decoded as unknown as Playlist; +} + +export function encodePlaylist(value: Playlist): PersistedPlaylist { + const encoded = copySafeObject(value as unknown as UnknownRecord); + encoded.episodes = value.episodes.map((episode) => encodeEpisode(episode)); + const currentEpisode = value.currentEpisode ? encodeEpisode(value.currentEpisode) : undefined; + if (currentEpisode) encoded.currentEpisode = currentEpisode; + else delete encoded.currentEpisode; + return encoded as PersistedPlaylist; +} + +function clonePlaylist(value: Playlist): Playlist { + return { + ...value, + episodes: value.episodes.map((episode) => ({ ...episode })), + currentEpisode: value.currentEpisode ? { ...value.currentEpisode } : undefined, + }; +} diff --git a/src/persistence/podNotesData.test.ts b/src/persistence/podNotesData.test.ts new file mode 100644 index 00000000..f6d1b8d0 --- /dev/null +++ b/src/persistence/podNotesData.test.ts @@ -0,0 +1,306 @@ +import { describe, expect, it } from "vitest"; +import { DEFAULT_SETTINGS } from "src/constants"; +import type DownloadedEpisode from "src/types/DownloadedEpisode"; +import type { Episode } from "src/types/Episode"; +import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; +import type { Playlist } from "src/types/Playlist"; +import { + decodePodNotesData, + encodePodNotesData, + PODNOTES_DATA_SCHEMA_VERSION, + PodNotesDataError, +} from "./podNotesData"; + +const episodeDate = new Date("2024-03-01T10:05:03.000Z"); + +function episode(title: string): Episode { + return { + title, + streamUrl: `https://example.com/${title}.mp3`, + url: `https://example.com/${title}`, + description: `${title} description`, + content: `${title} content`, + podcastName: "Test Podcast", + episodeDate, + }; +} + +function playlist(name: string, item: Episode): Playlist { + return { + name, + icon: "list", + episodes: [item], + currentEpisode: item, + shouldEpisodeRemoveAfterPlay: false, + shouldRepeat: false, + }; +} + +function settingsWithEveryEpisodeContainer(): IPodNotesSettings { + const current = episode("Current"); + const queued = episode("Queued"); + const favorite = episode("Favorite"); + const local = { ...episode("Local"), filePath: "Podcasts/local.mp3" }; + const custom = episode("Custom"); + const downloaded = { + ...episode("Downloaded"), + filePath: "Podcasts/downloaded.mp3", + size: 42, + } satisfies DownloadedEpisode; + + return { + ...structuredClone(DEFAULT_SETTINGS), + currentEpisode: current, + queue: playlist("Queue", queued), + favorites: playlist("Favorites", favorite), + localFiles: playlist("Local Files", local), + playlists: { Custom: playlist("Custom", custom) }, + downloadedEpisodes: { "Test Podcast": [downloaded] }, + }; +} + +describe("PodNotes data schema", () => { + it("loads fresh data as independent deep-cloned defaults", () => { + const first = decodePodNotesData(undefined); + const second = decodePodNotesData(null); + + expect(first.sourceVersion).toBe(0); + expect(first.warnings).toEqual([]); + expect(first.settings).toEqual(DEFAULT_SETTINGS); + first.settings.queue.episodes.push(episode("Mutation")); + first.settings.savedFeeds.Modified = { + title: "Modified", + url: "https://example.com/feed.xml", + artworkUrl: "", + }; + + expect(DEFAULT_SETTINGS.queue.episodes).toEqual([]); + expect(DEFAULT_SETTINGS.savedFeeds).toEqual({}); + expect(second.settings.queue.episodes).toEqual([]); + }); + + it("round-trips every persisted episode container with canonical dates", () => { + const settings = settingsWithEveryEpisodeContainer(); + const persisted = encodePodNotesData(settings); + const json = JSON.parse(JSON.stringify(persisted)) as Record; + const decoded = decodePodNotesData(json); + + expect(persisted.schemaVersion).toBe(PODNOTES_DATA_SCHEMA_VERSION); + expect((persisted.currentEpisode as Record).episodeDate).toBe( + episodeDate.toISOString(), + ); + expect(decoded.sourceVersion).toBe(2); + expect(decoded.changed).toBe(false); + expect(decoded.settings.currentEpisode?.episodeDate).toEqual(episodeDate); + expect(decoded.settings.queue.episodes[0].episodeDate).toEqual(episodeDate); + expect(decoded.settings.queue.currentEpisode?.episodeDate).toEqual(episodeDate); + expect(decoded.settings.favorites.episodes[0].episodeDate).toEqual(episodeDate); + expect(decoded.settings.localFiles.episodes[0].episodeDate).toEqual(episodeDate); + expect(decoded.settings.playlists.Custom.episodes[0].episodeDate).toEqual(episodeDate); + expect(decoded.settings.playlists.Custom.currentEpisode?.episodeDate).toEqual(episodeDate); + expect(decoded.settings.downloadedEpisodes["Test Podcast"][0].episodeDate).toEqual( + episodeDate, + ); + expect(decoded.settings.localFiles.episodes[0]).toMatchObject({ + filePath: "Podcasts/local.mp3", + }); + }); + + it("migrates valid legacy values through the existing repair rules", () => { + const decoded = decodePodNotesData({ + defaultPlaybackRate: 99, + episodeListLimit: 0, + skipBackwardLength: null, + download: { path: "" }, + note: { path: "", template: "" }, + transcript: { path: "custom.md", template: "{{transcript}}" }, + feedNote: { path: "custom-feed.md" }, + }); + + expect(decoded.sourceVersion).toBe(0); + expect(decoded.settings.defaultPlaybackRate).toBe(4); + expect(decoded.settings.episodeListLimit).toBe(DEFAULT_SETTINGS.episodeListLimit); + expect(decoded.settings.skipBackwardLength).toBe(DEFAULT_SETTINGS.skipBackwardLength); + expect(decoded.settings.download.path).toBe(DEFAULT_SETTINGS.download.path); + expect(decoded.settings.note).toEqual(DEFAULT_SETTINGS.note); + expect(decoded.settings.transcript).toMatchObject({ + path: "custom.md", + template: "{{transcript}}", + diarization: DEFAULT_SETTINGS.transcript.diarization, + }); + expect(decoded.settings.feedNote).toEqual({ + path: "custom-feed.md", + template: DEFAULT_SETTINGS.feedNote.template, + }); + }); + + it("preserves an intentionally disabled note feature from schema v1 and on v2 save", () => { + const decoded = decodePodNotesData({ + schemaVersion: 1, + note: { path: "", template: "" }, + }); + const persisted = encodePodNotesData(decoded.settings); + + expect(decoded.settings.note).toEqual({ path: "", template: "" }); + expect(persisted.note).toEqual({ path: "", template: "" }); + }); + + it("extracts legacy plaintext credentials without putting them in runtime settings", () => { + const decoded = decodePodNotesData({ + schemaVersion: 1, + openAIApiKey: " sk-legacy ", + diarizationApiKey: "dg-legacy", + }); + + expect(decoded.legacySecrets).toEqual({ openAI: "sk-legacy", deepgram: "dg-legacy" }); + expect(decoded.settings).not.toHaveProperty("openAIApiKey"); + expect(decoded.settings).not.toHaveProperty("diarizationApiKey"); + expect(decoded.unknownFields).not.toHaveProperty("openAIApiKey"); + expect(decoded.unknownFields).not.toHaveProperty("diarizationApiKey"); + }); + + it("retires plaintext fields even if they appear in v2 data", () => { + const decoded = decodePodNotesData({ + schemaVersion: 2, + openAIApiKey: "must-not-survive", + diarizationApiKey: "must-not-survive", + }); + const persisted = encodePodNotesData(decoded.settings, decoded.unknownFields); + + expect(decoded.legacySecrets).toEqual({}); + expect(decoded.retiredPlaintextPresent).toBe(true); + expect(decoded.changed).toBe(true); + expect(decoded.unknownFields).not.toHaveProperty("openAIApiKey"); + expect(decoded.unknownFields).not.toHaveProperty("diarizationApiKey"); + expect(JSON.stringify(persisted)).not.toContain("must-not-survive"); + expect(persisted).not.toHaveProperty("openAIApiKey"); + expect(persisted).not.toHaveProperty("diarizationApiKey"); + }); + + it("preserves valid SecretStorage IDs and removes invalid references", () => { + const valid = decodePodNotesData({ + schemaVersion: 2, + openAISecretId: "podnotes-openai-api-key-2", + }); + const invalid = decodePodNotesData({ + schemaVersion: 2, + openAISecretId: "Not Valid!", + }); + + expect(valid.settings.openAISecretId).toBe("podnotes-openai-api-key-2"); + expect(invalid.settings.openAISecretId).toBe(""); + expect(invalid.warnings).toContain( + "openAISecretId: invalid SecretStorage ID; reference was removed", + ); + }); + + it("salvages valid fields and only drops individually invalid collection entries", () => { + const validEpisode = { + ...episode("Valid"), + episodeDate: episodeDate.toISOString(), + }; + const decoded = decodePodNotesData({ + schemaVersion: 1, + defaultVolume: 3, + queue: { + name: "Queue", + icon: "list-ordered", + shouldEpisodeRemoveAfterPlay: true, + shouldRepeat: false, + episodes: [validEpisode, { title: 42 }], + }, + downloadedEpisodes: { + Podcast: [ + { ...validEpisode, filePath: "episode.mp3" }, + { ...validEpisode, title: "No path" }, + ], + }, + playlists: { Valid: playlist("Valid", episode("Playlist")), Broken: null }, + }); + + expect(decoded.settings.defaultVolume).toBe(1); + expect(decoded.settings.queue.episodes.map((item) => item.title)).toEqual(["Valid"]); + expect(decoded.settings.downloadedEpisodes.Podcast).toHaveLength(1); + expect(decoded.settings.downloadedEpisodes.Podcast[0]).toMatchObject({ + filePath: "episode.mp3", + size: 0, + }); + expect(Object.keys(decoded.settings.playlists)).toEqual(["Valid"]); + expect(decoded.warnings).toContain("defaultVolume: value was clamped"); + expect(decoded.warnings).toContain( + "queue.episodes[1].title: expected a string; episode was skipped", + ); + }); + + it("removes an invalid date while preserving the episode", () => { + const decoded = decodePodNotesData({ + currentEpisode: { ...episode("Restored"), episodeDate: "not-a-date" }, + }); + + expect(decoded.settings.currentEpisode).toMatchObject({ title: "Restored" }); + expect(decoded.settings.currentEpisode?.episodeDate).toBeUndefined(); + expect(decoded.warnings).toContain("currentEpisode.episodeDate: invalid date was removed"); + }); + + it("backfills fields missing from early episode and download snapshots", () => { + const early = { + title: "Early", + streamUrl: "early.mp3", + url: "", + description: "", + podcastName: "Old Podcast", + filePath: "early.mp3", + }; + const decoded = decodePodNotesData({ + currentEpisode: early, + downloadedEpisodes: { "Old Podcast": [early] }, + }); + + expect(decoded.settings.currentEpisode?.content).toBe(""); + expect(decoded.settings.downloadedEpisodes["Old Podcast"][0].size).toBe(0); + }); + + it("preserves safe unknown root and nested fields across a schema upgrade", () => { + const decoded = decodePodNotesData({ + schemaVersion: 1, + futureRootField: { enabled: true }, + queue: { + ...DEFAULT_SETTINGS.queue, + futurePlaylistField: "kept", + }, + }); + const persisted = encodePodNotesData(decoded.settings, decoded.unknownFields); + + expect(persisted.futureRootField).toEqual({ enabled: true }); + expect((persisted.queue as Record).futurePlaylistField).toBe("kept"); + }); + + it("removes prototype-pollution keys", () => { + const raw = JSON.parse( + '{"schemaVersion":1,"__proto__":{"polluted":true},"playlists":{"constructor":{"episodes":[]}}}', + ) as Record; + const decoded = decodePodNotesData(raw); + const persisted = encodePodNotesData(decoded.settings, decoded.unknownFields); + + expect(decoded.unknownFields).not.toHaveProperty("__proto__"); + expect(decoded.settings.playlists).toEqual({}); + expect(persisted).not.toHaveProperty("__proto__"); + expect(({} as { polluted?: boolean }).polluted).toBeUndefined(); + }); + + it.each([[], "data", 42, true])("fails closed for malformed root %p", (value) => { + expect(() => decodePodNotesData(value)).toThrowError(PodNotesDataError); + }); + + it.each([0, -1, 1.5, "1", null])("fails closed for invalid schema version %p", (version) => { + expect(() => decodePodNotesData({ schemaVersion: version })).toThrowError( + /invalid schemaVersion/, + ); + }); + + it("fails closed for a future schema version", () => { + expect(() => decodePodNotesData({ schemaVersion: 3 })).toThrowError( + /schema v3 requires a newer version/, + ); + }); +}); diff --git a/src/persistence/podNotesData.ts b/src/persistence/podNotesData.ts new file mode 100644 index 00000000..ad3a7fbc --- /dev/null +++ b/src/persistence/podNotesData.ts @@ -0,0 +1,197 @@ +import { DEFAULT_SETTINGS } from "src/constants"; +import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; +import { DANGEROUS_KEYS, isPlainObject, mapRecord, type UnknownRecord } from "./codecUtils"; +import { + encodeEpisode, + encodePlaylist, + type PersistedEpisode, + type PersistedPlaylist, +} from "./episodeCodec"; +import { decodeSettings } from "./settingsCodec"; +import type { CredentialValues } from "src/types/Credentials"; + +export { decodeEpisode, decodePlaylist, encodeEpisode, encodePlaylist } from "./episodeCodec"; + +export const PODNOTES_DATA_SCHEMA_VERSION = 2; + +/** Plaintext fields accepted only as migration input and never persisted again. */ +export const RETIRED_PLAINTEXT_SECRET_KEYS = new Set(["openAIApiKey", "diarizationApiKey"]); + +const KNOWN_TOP_LEVEL_KEYS = new Set([ + ...Object.keys(DEFAULT_SETTINGS), + ...RETIRED_PLAINTEXT_SECRET_KEYS, + "schemaVersion", +]); + +export type PodNotesDataSchemaVersion = 0 | 1 | typeof PODNOTES_DATA_SCHEMA_VERSION; + +export class PodNotesDataError extends Error { + constructor( + message: string, + public readonly code: "invalid-data" | "unsupported-version", + ) { + super(message); + this.name = "PodNotesDataError"; + } +} + +export interface DecodedPodNotesData { + settings: IPodNotesSettings; + sourceVersion: PodNotesDataSchemaVersion; + changed: boolean; + warnings: string[]; + /** Plaintext v0/v1 values that must move into SecretStorage before a v2 save. */ + legacySecrets: CredentialValues; + /** Retired fields found in v2 must be scrubbed without importing their values. */ + retiredPlaintextPresent: boolean; + /** Unknown supported-schema root fields retained so a normal save does not erase them. */ + unknownFields: UnknownRecord; +} + +export type PersistedPodNotesSettings = Omit< + IPodNotesSettings, + "currentEpisode" | "favorites" | "queue" | "localFiles" | "playlists" | "downloadedEpisodes" +> & { + currentEpisode?: PersistedEpisode; + favorites: PersistedPlaylist; + queue: PersistedPlaylist; + localFiles: PersistedPlaylist; + playlists: Record; + downloadedEpisodes: Record; +}; + +export type PersistedPodNotesDataV2 = PersistedPodNotesSettings & + UnknownRecord & { + schemaVersion: typeof PODNOTES_DATA_SCHEMA_VERSION; + }; + +/** + * Decode and validate plugin data before any store sees it. + * + * Missing `schemaVersion` is the legacy v0 shape. Persisted data remains flat + * so existing data files, rollback paths, and E2E tooling stay compatible. A + * newer version fails closed so an older plugin can never overwrite data it + * does not understand. + */ +export function decodePodNotesData(value: unknown): DecodedPodNotesData { + if (value === null || value === undefined) value = {}; + + if (!isPlainObject(value)) { + throw new PodNotesDataError( + "PodNotes data.json does not contain an object. The file was not modified.", + "invalid-data", + ); + } + + const sourceVersion = readSchemaVersion(value); + const warnings = new Set(); + const settings = decodeSettings(value, warnings, sourceVersion); + const legacySecrets = decodeLegacySecrets(value, sourceVersion, warnings); + const retiredPlaintextPresent = [...RETIRED_PLAINTEXT_SECRET_KEYS].some((key) => + Object.prototype.hasOwnProperty.call(value, key), + ); + const unknownFields = copyUnknownRootFields(value); + + return { + settings, + sourceVersion, + changed: + sourceVersion !== PODNOTES_DATA_SCHEMA_VERSION || + retiredPlaintextPresent || + warnings.size > 0, + warnings: [...warnings], + legacySecrets, + retiredPlaintextPresent, + unknownFields, + }; +} + +/** Serialize a validated runtime snapshot with canonical dates and schema. */ +export function encodePodNotesData( + settings: IPodNotesSettings, + unknownFields: UnknownRecord = {}, +): PersistedPodNotesDataV2 { + const validated = decodeSettings( + settings as unknown as UnknownRecord, + new Set(), + PODNOTES_DATA_SCHEMA_VERSION, + ); + + return { + ...copyUnknownRootFields(unknownFields), + ...validated, + schemaVersion: PODNOTES_DATA_SCHEMA_VERSION, + currentEpisode: validated.currentEpisode + ? encodeEpisode(validated.currentEpisode) + : undefined, + favorites: encodePlaylist(validated.favorites), + queue: encodePlaylist(validated.queue), + localFiles: encodePlaylist(validated.localFiles), + playlists: mapRecord(validated.playlists, (playlist) => encodePlaylist(playlist)), + downloadedEpisodes: mapRecord(validated.downloadedEpisodes, (episodes) => + episodes.map((episode) => encodeEpisode(episode)), + ), + } as PersistedPodNotesDataV2; +} + +function readSchemaVersion(root: UnknownRecord): PodNotesDataSchemaVersion { + if (!Object.prototype.hasOwnProperty.call(root, "schemaVersion")) return 0; + + const version = root.schemaVersion; + if (version === 1 || version === PODNOTES_DATA_SCHEMA_VERSION) return version; + if ( + typeof version === "number" && + Number.isInteger(version) && + version > PODNOTES_DATA_SCHEMA_VERSION + ) { + throw new PodNotesDataError( + `PodNotes data schema v${version} requires a newer version of PodNotes. The file was not modified.`, + "unsupported-version", + ); + } + + throw new PodNotesDataError( + "PodNotes data.json has an invalid schemaVersion. The file was not modified.", + "invalid-data", + ); +} + +function decodeLegacySecrets( + root: UnknownRecord, + sourceVersion: PodNotesDataSchemaVersion, + warnings: Set, +): CredentialValues { + if (sourceVersion >= PODNOTES_DATA_SCHEMA_VERSION) return {}; + + const values: CredentialValues = {}; + readLegacySecret(root, "openAIApiKey", "openAI", values, warnings); + readLegacySecret(root, "diarizationApiKey", "deepgram", values, warnings); + return values; +} + +function readLegacySecret( + root: UnknownRecord, + legacyKey: "openAIApiKey" | "diarizationApiKey", + credentialKey: keyof CredentialValues, + values: CredentialValues, + warnings: Set, +): void { + const value = root[legacyKey]; + if (value === undefined || value === null || value === "") return; + if (typeof value !== "string") { + warnings.add(`${legacyKey}: expected a string; value was ignored`); + return; + } + + const normalized = value.trim(); + if (normalized) values[credentialKey] = normalized; +} + +function copyUnknownRootFields(root: UnknownRecord): UnknownRecord { + const copy: UnknownRecord = {}; + for (const [key, value] of Object.entries(root)) { + if (DANGEROUS_KEYS.has(key) || KNOWN_TOP_LEVEL_KEYS.has(key)) continue; + copy[key] = value; + } + return copy; +} diff --git a/src/persistence/settingsCodec.ts b/src/persistence/settingsCodec.ts new file mode 100644 index 00000000..6fbeca46 --- /dev/null +++ b/src/persistence/settingsCodec.ts @@ -0,0 +1,262 @@ +import { DEFAULT_SETTINGS } from "src/constants"; +import { + migrateDownloadPath, + migrateFeedNoteSettings, + migrateNoteSettings, + migrateSkipLength, + migrateTranscriptSettings, +} from "src/settingsMigrations"; +import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; +import { isValidSecretId } from "src/types/Credentials"; +import { sanitizeEpisodeListLimit } from "src/utility/episodeListLimit"; +import { normalizePlaybackRate } from "src/utility/playbackRate"; +import { + decodeDownloadedEpisodes, + decodePlayedEpisodes, + decodePlaylists, + decodePodNotes, + decodeSavedFeeds, +} from "./collectionCodecs"; +import { + copySafeObject, + optionalRecord, + readBoolean, + readClampedNumber, + readFiniteNumber, + readNullableString, + readPositiveNumber, + readString, + type UnknownRecord, + warn, +} from "./codecUtils"; +import { decodeEpisode, decodePlaylist } from "./episodeCodec"; +import type { PodNotesDataSchemaVersion } from "./podNotesData"; + +export function decodeSettings( + root: UnknownRecord, + warnings: Set, + sourceVersion: PodNotesDataSchemaVersion = 2, +): IPodNotesSettings { + const defaultPlaybackRate = normalizePlaybackRate(root.defaultPlaybackRate); + if ( + root.defaultPlaybackRate !== undefined && + (typeof root.defaultPlaybackRate !== "number" || + root.defaultPlaybackRate !== defaultPlaybackRate) + ) { + warn(warnings, "defaultPlaybackRate", "value was normalized"); + } + + const defaultVolume = readClampedNumber( + root, + "defaultVolume", + DEFAULT_SETTINGS.defaultVolume, + 0, + 1, + warnings, + ); + const episodeListLimit = sanitizeEpisodeListLimit(root.episodeListLimit); + if ( + root.episodeListLimit !== undefined && + (typeof root.episodeListLimit !== "number" || root.episodeListLimit !== episodeListLimit) + ) { + warn(warnings, "episodeListLimit", "value was normalized"); + } + + return { + savedFeeds: decodeSavedFeeds(root.savedFeeds, warnings), + podNotes: decodePodNotes(root.podNotes, warnings), + defaultPlaybackRate, + defaultVolume, + hidePlayedEpisodes: readBoolean( + root, + "hidePlayedEpisodes", + DEFAULT_SETTINGS.hidePlayedEpisodes, + warnings, + ), + episodeListLimit, + playedEpisodes: decodePlayedEpisodes(root.playedEpisodes, warnings), + skipBackwardLength: decodeSkipLength( + root.skipBackwardLength, + DEFAULT_SETTINGS.skipBackwardLength, + "skipBackwardLength", + warnings, + ), + skipForwardLength: decodeSkipLength( + root.skipForwardLength, + DEFAULT_SETTINGS.skipForwardLength, + "skipForwardLength", + warnings, + ), + playlists: decodePlaylists(root.playlists, warnings), + queue: decodePlaylist(root.queue, DEFAULT_SETTINGS.queue, warnings, "queue"), + autoQueue: readBoolean(root, "autoQueue", DEFAULT_SETTINGS.autoQueue, warnings), + favorites: decodePlaylist( + root.favorites, + DEFAULT_SETTINGS.favorites, + warnings, + "favorites", + ), + localFiles: decodePlaylist( + root.localFiles, + DEFAULT_SETTINGS.localFiles, + warnings, + "localFiles", + ), + currentEpisode: + root.currentEpisode === undefined || root.currentEpisode === null + ? undefined + : decodeEpisode(root.currentEpisode, warnings, "currentEpisode"), + timestamp: decodeTimestamp(root.timestamp, warnings), + note: decodeNote(root.note, warnings, sourceVersion), + feedNote: decodeFeedNote(root.feedNote, warnings), + download: decodeDownload(root.download, warnings), + downloadedEpisodes: decodeDownloadedEpisodes(root.downloadedEpisodes, warnings), + openAISecretId: decodeSecretId(root, "openAISecretId", warnings), + deepgramSecretId: decodeSecretId(root, "deepgramSecretId", warnings), + transcript: decodeTranscript(root.transcript, warnings), + feedCache: decodeFeedCache(root.feedCache, warnings), + }; +} + +function decodeSecretId( + root: UnknownRecord, + key: "openAISecretId" | "deepgramSecretId", + warnings: Set, +): string { + const value = readString(root, key, DEFAULT_SETTINGS[key], warnings); + if (!value || isValidSecretId(value)) return value; + + warn(warnings, key, "invalid SecretStorage ID; reference was removed"); + return ""; +} + +function decodeTimestamp(value: unknown, warnings: Set): IPodNotesSettings["timestamp"] { + const record = optionalRecord(value, warnings, "timestamp"); + return { + ...copySafeObject(record), + template: readString( + record, + "template", + DEFAULT_SETTINGS.timestamp.template, + warnings, + "timestamp", + ), + offset: readFiniteNumber( + record, + "offset", + DEFAULT_SETTINGS.timestamp.offset, + warnings, + "timestamp", + ), + }; +} + +function decodeNote( + value: unknown, + warnings: Set, + sourceVersion: PodNotesDataSchemaVersion, +): IPodNotesSettings["note"] { + const record = optionalRecord(value, warnings, "note"); + const path = readNullableString(record, "path", warnings, "note"); + const template = readNullableString(record, "template", warnings, "note"); + const validated = + sourceVersion === 0 + ? migrateNoteSettings({ path, template }) + : { + path: typeof path === "string" ? path : DEFAULT_SETTINGS.note.path, + template: + typeof template === "string" ? template : DEFAULT_SETTINGS.note.template, + }; + return { ...copySafeObject(record), ...validated }; +} + +function decodeFeedNote(value: unknown, warnings: Set): IPodNotesSettings["feedNote"] { + const record = optionalRecord(value, warnings, "feedNote"); + const migrated = migrateFeedNoteSettings({ + path: readNullableString(record, "path", warnings, "feedNote"), + template: readNullableString(record, "template", warnings, "feedNote"), + }); + return { ...copySafeObject(record), ...migrated }; +} + +function decodeDownload(value: unknown, warnings: Set): IPodNotesSettings["download"] { + const record = optionalRecord(value, warnings, "download"); + return { + ...copySafeObject(record), + path: migrateDownloadPath(readNullableString(record, "path", warnings, "download")), + }; +} + +function decodeTranscript(value: unknown, warnings: Set): IPodNotesSettings["transcript"] { + const record = optionalRecord(value, warnings, "transcript"); + const diarization = optionalRecord(record.diarization, warnings, "transcript.diarization"); + const migrated = migrateTranscriptSettings({ + path: record.path as string | null | undefined, + template: record.template as string | null | undefined, + diarization, + }); + + if (record.path !== undefined && typeof record.path !== "string") { + warn(warnings, "transcript.path", "expected a string"); + } + if (record.template !== undefined && typeof record.template !== "string") { + warn(warnings, "transcript.template", "expected a string"); + } + if (diarization.enabled !== undefined && typeof diarization.enabled !== "boolean") { + warn(warnings, "transcript.diarization.enabled", "expected a boolean"); + } + if ( + diarization.provider !== undefined && + diarization.provider !== "openai" && + diarization.provider !== "deepgram" + ) { + warn(warnings, "transcript.diarization.provider", "unknown provider"); + } + if ( + diarization.speakerTemplate !== undefined && + typeof diarization.speakerTemplate !== "string" + ) { + warn(warnings, "transcript.diarization.speakerTemplate", "expected a string"); + } + + return { + ...copySafeObject(record), + ...migrated, + diarization: { + ...copySafeObject(diarization), + ...migrated.diarization, + }, + }; +} + +function decodeFeedCache(value: unknown, warnings: Set): IPodNotesSettings["feedCache"] { + const record = optionalRecord(value, warnings, "feedCache"); + return { + ...copySafeObject(record), + enabled: readBoolean( + record, + "enabled", + DEFAULT_SETTINGS.feedCache.enabled, + warnings, + "feedCache", + ), + ttlHours: readPositiveNumber( + record, + "ttlHours", + DEFAULT_SETTINGS.feedCache.ttlHours, + warnings, + "feedCache", + ), + }; +} + +function decodeSkipLength( + value: unknown, + fallback: number, + path: string, + warnings: Set, +): number { + const decoded = migrateSkipLength(value, fallback); + if (value !== undefined && value !== decoded) warn(warnings, path, "value was normalized"); + return decoded; +} diff --git a/src/services/CredentialRepository.test.ts b/src/services/CredentialRepository.test.ts new file mode 100644 index 00000000..e20944b7 --- /dev/null +++ b/src/services/CredentialRepository.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, vi } from "vitest"; +import type { SecretStorage } from "obsidian"; +import { DEFAULT_SETTINGS } from "src/constants"; +import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; +import { CredentialRepository } from "./CredentialRepository"; + +function storage(initial: Record = {}) { + const values = new Map(Object.entries(initial)); + return { + values, + api: { + getSecret: vi.fn((id: string) => values.get(id) ?? null), + setSecret: vi.fn((id: string, value: string) => { + values.set(id, value); + }), + listSecrets: vi.fn(() => [...values.keys()]), + } as unknown as SecretStorage, + }; +} + +function settings(overrides: Partial = {}): IPodNotesSettings { + return { ...structuredClone(DEFAULT_SETTINGS), ...overrides }; +} + +describe("CredentialRepository", () => { + it("stores credentials under fixed PodNotes IDs and verifies them", () => { + const { api, values } = storage(); + const repository = new CredentialRepository(api); + + expect(repository.storeValues({ openAI: " sk-openai ", deepgram: "dg-key" })).toEqual({ + openAISecretId: "podnotes-openai-api-key", + deepgramSecretId: "podnotes-deepgram-api-key", + }); + expect(values.get("podnotes-openai-api-key")).toBe("sk-openai"); + expect(values.get("podnotes-deepgram-api-key")).toBe("dg-key"); + expect(api.getSecret).toHaveBeenCalledWith("podnotes-openai-api-key"); + }); + + it("reuses an exact value so a migration retry is idempotent", () => { + const { api } = storage({ "podnotes-openai-api-key": "sk-existing" }); + const repository = new CredentialRepository(api); + + expect(repository.storeValues({ openAI: "sk-existing" })).toEqual({ + openAISecretId: "podnotes-openai-api-key", + }); + expect(api.setSecret).not.toHaveBeenCalled(); + }); + + it("uses a collision-safe suffix and never overwrites another value", () => { + const { api, values } = storage({ + "podnotes-openai-api-key": "someone-else", + "podnotes-openai-api-key-2": "also-taken", + }); + const repository = new CredentialRepository(api); + + expect(repository.storeValues({ openAI: "sk-new" })).toEqual({ + openAISecretId: "podnotes-openai-api-key-3", + }); + expect(values.get("podnotes-openai-api-key")).toBe("someone-else"); + expect(values.get("podnotes-openai-api-key-3")).toBe("sk-new"); + }); + + it("fails when SecretStorage cannot read back the written value", () => { + const api = { + getSecret: vi.fn().mockReturnValueOnce(null).mockReturnValueOnce(null), + setSecret: vi.fn(), + listSecrets: vi.fn(() => []), + } as unknown as SecretStorage; + const repository = new CredentialRepository(api); + + expect(() => repository.storeValues({ openAI: "sk" })).toThrow( + /SecretStorage did not retain/, + ); + }); + + it("runtime reads fail closed without retaining values in the repository", () => { + const { api } = storage({ token: " sk-runtime " }); + const repository = new CredentialRepository(api); + const configured = settings({ openAISecretId: "token" }); + + expect(repository.get(configured, "openai")).toBe("sk-runtime"); + expect(repository.has(configured, "openai")).toBe(true); + expect(repository.get(settings(), "openai")).toBeNull(); + + (api.getSecret as ReturnType).mockImplementation(() => { + throw new Error("storage unavailable"); + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + expect(repository.get(configured, "openai")).toBeNull(); + }); + + it("exports only values whose referenced secrets still exist", () => { + const { api } = storage({ openai: "sk", deepgram: " " }); + const repository = new CredentialRepository(api); + + expect( + repository.exportValues( + settings({ openAISecretId: "openai", deepgramSecretId: "deepgram" }), + ), + ).toEqual({ openAI: "sk" }); + }); + + it("distinguishes unconfigured and synced-but-missing references", () => { + const { api } = storage(); + const repository = new CredentialRepository(api); + + expect(repository.status(settings(), "openai")).toBe("unconfigured"); + expect(repository.status(settings({ openAISecretId: "missing" }), "openai")).toBe( + "missing", + ); + }); + + it("refuses an explicit plaintext export when a configured value is missing locally", () => { + const { api } = storage(); + const repository = new CredentialRepository(api); + + expect(() => + repository.exportValues(settings({ openAISecretId: "synced-reference" }), { + requireConfigured: true, + }), + ).toThrow("not available in SecretStorage on this device"); + }); +}); diff --git a/src/services/CredentialRepository.ts b/src/services/CredentialRepository.ts new file mode 100644 index 00000000..87077599 --- /dev/null +++ b/src/services/CredentialRepository.ts @@ -0,0 +1,117 @@ +import type { SecretStorage } from "obsidian"; +import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; +import type { CredentialKind, CredentialReferences, CredentialValues } from "src/types/Credentials"; + +export type { CredentialKind, CredentialReferences, CredentialValues } from "src/types/Credentials"; + +export type CredentialStatus = "unconfigured" | "available" | "missing"; + +const CREDENTIALS: Record< + CredentialKind, + { + baseId: string; + settingsKey: keyof Pick; + } +> = { + openai: { + baseId: "podnotes-openai-api-key", + settingsKey: "openAISecretId", + }, + deepgram: { + baseId: "podnotes-deepgram-api-key", + settingsKey: "deepgramSecretId", + }, +}; + +/** + * Narrow boundary around Obsidian's vault-local SecretStorage. + * + * Runtime reads fail closed and never cache secret values. Migration/import + * writes are strict: an ID is returned only after the exact value reads back. + */ +export class CredentialRepository { + constructor(private readonly storage: SecretStorage) {} + + get(settings: IPodNotesSettings, kind: CredentialKind): string | null { + const id = settings[CREDENTIALS[kind].settingsKey].trim(); + if (!id) return null; + + try { + const value = this.storage.getSecret(id); + return value?.trim() ? value.trim() : null; + } catch (error) { + console.error(`PodNotes: failed to read the ${kind} credential`, error); + return null; + } + } + + has(settings: IPodNotesSettings, kind: CredentialKind): boolean { + return this.get(settings, kind) !== null; + } + + status(settings: IPodNotesSettings, kind: CredentialKind): CredentialStatus { + if (!settings[CREDENTIALS[kind].settingsKey].trim()) return "unconfigured"; + return this.has(settings, kind) ? "available" : "missing"; + } + + exportValues( + settings: IPodNotesSettings, + options: { requireConfigured?: boolean } = {}, + ): CredentialValues { + const openAI = this.get(settings, "openai"); + const deepgram = this.get(settings, "deepgram"); + if (options.requireConfigured) { + if (settings.openAISecretId.trim() && !openAI) { + throw new Error( + "The selected OpenAI API key is not available in SecretStorage on this device.", + ); + } + if (settings.deepgramSecretId.trim() && !deepgram) { + throw new Error( + "The selected Deepgram API key is not available in SecretStorage on this device.", + ); + } + } + + return { + ...(openAI ? { openAI } : {}), + ...(deepgram ? { deepgram } : {}), + }; + } + + /** + * Store values under PodNotes-owned IDs. Existing matching values are reused, + * making retries idempotent after a later data.json save failure. A conflicting + * value is never overwritten: the first free numeric suffix is used instead. + */ + storeValues(values: CredentialValues): CredentialReferences { + const references: CredentialReferences = {}; + const openAI = values.openAI?.trim(); + const deepgram = values.deepgram?.trim(); + + if (openAI) references.openAISecretId = this.store("openai", openAI); + if (deepgram) references.deepgramSecretId = this.store("deepgram", deepgram); + + return references; + } + + private store(kind: CredentialKind, secret: string): string { + const { baseId } = CREDENTIALS[kind]; + + for (let suffix = 1; suffix <= 10_000; suffix++) { + const id = suffix === 1 ? baseId : `${baseId}-${suffix}`; + const existing = this.storage.getSecret(id); + + if (existing === secret) return id; + if (existing !== null) continue; + + this.storage.setSecret(id, secret); + if (this.storage.getSecret(id) !== secret) { + throw new Error(`SecretStorage did not retain the ${kind} credential.`); + } + return id; + } + + throw new Error(`Could not allocate a SecretStorage ID for the ${kind} credential.`); + } +} diff --git a/src/services/FeedCacheService.test.ts b/src/services/FeedCacheService.test.ts index 8ce87e81..52275ed9 100644 --- a/src/services/FeedCacheService.test.ts +++ b/src/services/FeedCacheService.test.ts @@ -191,4 +191,78 @@ describe("App-backed (vault-scoped) storage", () => { plugin.set(undefined as unknown as PodNotes); }); + + test("salvages a cached episode with an invalid date without creating Invalid Date", async () => { + const backing = new Map(); + backing.set( + "podnotes:feed-cache:v5", + JSON.stringify({ + [testFeed.url]: { + updatedAt: Date.now(), + episodes: [{ ...createEpisode(1), episodeDate: "not-a-date" }], + }, + }), + ); + const app = { + loadLocalStorage: (key: string) => backing.get(key) ?? null, + saveLocalStorage: (key: string, value: string | null) => { + if (value == null) backing.delete(key); + else backing.set(key, value); + }, + }; + + vi.resetModules(); + const freshStore = await import("../store"); + const freshSvc = await import("./FeedCacheService"); + (freshStore.plugin as typeof plugin).set({ app } as unknown as PodNotes); + + const cached = freshSvc.getCachedEpisodes(testFeed); + expect(cached).toHaveLength(1); + expect(cached?.[0]).toMatchObject({ title: "Episode 1" }); + expect(cached?.[0].episodeDate).toBeUndefined(); + }); + + test.each([ + JSON.stringify([]), + JSON.stringify({ + [testFeed.url]: { updatedAt: "yesterday", episodes: [createEpisode(1)] }, + }), + ])("ignores a structurally invalid cache without throwing", async (stored) => { + const backing = new Map([["podnotes:feed-cache:v5", stored]]); + const app = { + loadLocalStorage: (key: string) => backing.get(key) ?? null, + saveLocalStorage: (key: string, value: string | null) => { + if (value == null) backing.delete(key); + else backing.set(key, value); + }, + }; + + vi.resetModules(); + const freshStore = await import("../store"); + const freshSvc = await import("./FeedCacheService"); + (freshStore.plugin as typeof plugin).set({ app } as unknown as PodNotes); + + expect(() => freshSvc.getCachedEpisodes(testFeed)).not.toThrow(); + expect(freshSvc.getCachedEpisodes(testFeed)).toBeNull(); + }); + + test("drops prototype-pollution cache keys", async () => { + const stored = `{"__proto__":{"updatedAt":1,"episodes":[]},"${testFeed.url}":{"updatedAt":${Date.now()},"episodes":[]}}`; + const backing = new Map([["podnotes:feed-cache:v5", stored]]); + const app = { + loadLocalStorage: (key: string) => backing.get(key) ?? null, + saveLocalStorage: (key: string, value: string | null) => { + if (value == null) backing.delete(key); + else backing.set(key, value); + }, + }; + + vi.resetModules(); + const freshStore = await import("../store"); + const freshSvc = await import("./FeedCacheService"); + (freshStore.plugin as typeof plugin).set({ app } as unknown as PodNotes); + + expect(freshSvc.getCachedEpisodes(testFeed)).toEqual([]); + expect(({} as { updatedAt?: number }).updatedAt).toBeUndefined(); + }); }); diff --git a/src/services/FeedCacheService.ts b/src/services/FeedCacheService.ts index 6c1130e4..9c8fb63e 100644 --- a/src/services/FeedCacheService.ts +++ b/src/services/FeedCacheService.ts @@ -2,6 +2,9 @@ import { get } from "svelte/store"; import { plugin } from "../store"; import type { Episode } from "src/types/Episode"; import type { PodcastFeed } from "src/types/PodcastFeed"; +import { dateTimestamp, decodeDate, encodeDate } from "src/persistence/dateCodec"; +import { decodeEpisode } from "src/persistence/episodeCodec"; +import { DANGEROUS_KEYS } from "src/persistence/codecUtils"; type SerializableEpisode = Omit & { episodeDate?: string; @@ -135,8 +138,7 @@ function loadCache(): FeedCache { return cache; } - const parsed = JSON.parse(raw) as FeedCache; - cache = parsed; + cache = decodeCache(JSON.parse(raw)); return cache; } catch (error) { console.error("Failed to parse feed cache:", error); @@ -145,6 +147,39 @@ function loadCache(): FeedCache { } } +function decodeCache(value: unknown): FeedCache { + if (!isPlainObject(value)) { + throw new TypeError("Feed cache root must be an object"); + } + + const decoded: FeedCache = {}; + for (const [key, candidate] of Object.entries(value)) { + if (DANGEROUS_KEYS.has(key)) continue; + if ( + !isPlainObject(candidate) || + !Array.isArray(candidate.episodes) || + typeof candidate.updatedAt !== "number" || + !Number.isFinite(candidate.updatedAt) || + candidate.updatedAt < 0 + ) { + continue; + } + + decoded[key] = { + updatedAt: candidate.updatedAt, + episodes: candidate.episodes.flatMap((episode) => { + const runtimeEpisode = decodeEpisode(episode); + return runtimeEpisode ? [serializeEpisode(runtimeEpisode)] : []; + }), + }; + } + return decoded; +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + function evictOldestEntries(cacheData: FeedCache, targetSizeBytes: number): FeedCache { const entries = Object.entries(cacheData); @@ -211,14 +246,12 @@ function persistCache(): void { function serializeEpisode(episode: Episode): SerializableEpisode { return { ...episode, - episodeDate: episode.episodeDate?.toISOString(), + episodeDate: encodeDate(episode.episodeDate), }; } function episodeTimestamp(episode: Episode): number { - if (!episode.episodeDate) return 0; - const time = new Date(episode.episodeDate).getTime(); - return Number.isNaN(time) ? 0 : time; + return dateTimestamp(episode.episodeDate) ?? 0; } /** @@ -245,7 +278,7 @@ function selectNewestEpisodes(episodes: Episode[], limit: number): Episode[] { function deserializeEpisode(episode: SerializableEpisode): Episode { return { ...episode, - episodeDate: episode.episodeDate ? new Date(episode.episodeDate) : undefined, + episodeDate: decodeDate(episode.episodeDate), }; } diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index dd6cddc7..2f9dda99 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -34,20 +34,21 @@ const mockEpisode: Episode = { function createMockPlugin( overrides: { - openAIApiKey?: string; + openAIKey?: string; podcast?: Episode | null; existingTranscriptPath?: string | null; } = {}, ): PodNotes { const { - openAIApiKey = "test-api-key", + openAIKey = "test-api-key", podcast = mockEpisode, existingTranscriptPath = null, } = overrides; return { settings: { - openAIApiKey, + openAISecretId: openAIKey ? "openai-secret" : "", + deepgramSecretId: "", transcript: { path: "Transcripts/{{podcast}}/{{title}}.md", template: "# {{title}}\n\n{{transcript}}", @@ -56,6 +57,17 @@ function createMockPlugin( path: "Downloads", }, }, + credentials: { + get: vi.fn((_settings, kind: "openai" | "deepgram") => + kind === "openai" ? openAIKey || null : null, + ), + has: vi.fn((_settings, kind: "openai" | "deepgram") => + kind === "openai" ? Boolean(openAIKey) : false, + ), + status: vi.fn((_settings, kind: "openai" | "deepgram") => + kind === "openai" && openAIKey ? "available" : "unconfigured", + ), + }, api: { podcast, }, @@ -133,7 +145,7 @@ describe("TranscriptionService", () => { describe("transcribeCurrentEpisode validation", () => { test("shows notice when no API key is configured", async () => { - const mockPlugin = createMockPlugin({ openAIApiKey: "" }); + const mockPlugin = createMockPlugin({ openAIKey: "" }); const service = new TranscriptionService(mockPlugin); await service.transcribeCurrentEpisode(); @@ -147,6 +159,52 @@ describe("TranscriptionService", () => { }); }); + describe("dispose", () => { + test("prevents queued work and credential reads after unload", async () => { + const plugin = createMockPlugin(); + const service = new TranscriptionService(plugin); + (service as unknown as { pendingEpisodes: Episode[] }).pendingEpisodes = [mockEpisode]; + service.dispose(); + + (service as unknown as { drainQueue: () => void }).drainQueue(); + await expect( + (service as unknown as { getClient: () => Promise }).getClient(), + ).rejects.toThrow("unloaded"); + + expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); + expect(plugin.credentials.get).not.toHaveBeenCalled(); + }); + + test("cannot recreate a client when unload happens during the dynamic import", async () => { + const plugin = createMockPlugin(); + let resolveModule!: (module: Pick) => void; + const loader = vi.fn( + () => + new Promise>((resolve) => { + resolveModule = resolve; + }), + ); + const constructor = vi.fn(() => ({ audio: { transcriptions: { create: vi.fn() } } })); + const service = new TranscriptionService(plugin, loader); + const pending = ( + service as unknown as { getClient: () => Promise } + ).getClient(); + await vi.waitFor(() => expect(loader).toHaveBeenCalledOnce()); + + service.dispose(); + resolveModule({ OpenAI: constructor as unknown as typeof import("openai").OpenAI }); + + await expect(pending).rejects.toThrow("unloaded"); + expect(constructor).not.toHaveBeenCalled(); + expect( + (service as unknown as { client: unknown; cachedApiKey: unknown }).client, + ).toBeNull(); + expect( + (service as unknown as { client: unknown; cachedApiKey: unknown }).cachedApiKey, + ).toBeNull(); + }); + }); + describe("empty Whisper transcript (TR-01)", () => { // transcribeEpisode is private; drive it directly so the empty-body throw is // observed as "no file written" (the throw is caught and surfaced as a diff --git a/src/services/TranscriptionService.ts b/src/services/TranscriptionService.ts index b6efd2cd..15100533 100644 --- a/src/services/TranscriptionService.ts +++ b/src/services/TranscriptionService.ts @@ -69,24 +69,39 @@ export class TranscriptionService { private plugin: PodNotes; private client: OpenAI | null = null; private cachedApiKey: string | null = null; + private disposed = false; private MAX_RETRIES = 3; private readonly MAX_CONCURRENT_TRANSCRIPTIONS = 2; private readonly MAX_CONCURRENT_CHUNK_TRANSCRIPTIONS = 3; private pendingEpisodes: Episode[] = []; private activeTranscriptions = new Set(); - constructor(plugin: PodNotes) { + constructor( + plugin: PodNotes, + private readonly loadOpenAI: () => Promise> = () => + import("openai"), + ) { this.plugin = plugin; } async transcribeCurrentEpisode(): Promise { - if (!requiredTranscriptionKeyPresent(this.plugin.settings)) { + if (this.disposed) return; + if ( + !requiredTranscriptionKeyPresent(this.plugin.settings, (kind) => + this.plugin.credentials.has(this.plugin.settings, kind), + ) + ) { const diarization = this.plugin.settings.transcript.diarization; const needsDeepgram = diarization?.enabled && diarization.provider === "deepgram"; + const kind = needsDeepgram ? "deepgram" : "openai"; + const unavailableOnDevice = + this.plugin.credentials.status(this.plugin.settings, kind) === "missing"; new Notice( - needsDeepgram - ? "Please add your Deepgram API key in the transcript settings to use Deepgram diarization." - : "Please add your OpenAI API key in the transcript settings first.", + unavailableOnDevice + ? `The selected ${needsDeepgram ? "Deepgram" : "OpenAI"} API key is not available on this device. Select or create it in the transcript settings.` + : needsDeepgram + ? "Select or create a Deepgram API key in the transcript settings on this device." + : "Select or create an OpenAI API key in the transcript settings on this device.", ); return; } @@ -122,6 +137,10 @@ export class TranscriptionService { } private drainQueue(): void { + if (this.disposed) { + this.pendingEpisodes = []; + return; + } while ( this.activeTranscriptions.size < this.MAX_CONCURRENT_TRANSCRIPTIONS && this.pendingEpisodes.length > 0 @@ -136,7 +155,7 @@ export class TranscriptionService { void this.transcribeEpisode(nextEpisode).finally(() => { this.activeTranscriptions.delete(episodeKey); - this.drainQueue(); + if (!this.disposed) this.drainQueue(); }); } } @@ -259,10 +278,11 @@ export class TranscriptionService { provider: DiarizationProviderId, updateNotice: (message: string) => void, ): Promise { + this.assertActive(); if (provider === "deepgram") { - const apiKey = this.plugin.settings.diarizationApiKey?.trim(); + const apiKey = this.plugin.credentials.get(this.plugin.settings, "deepgram"); if (!apiKey) { - throw new Error("Missing Deepgram API key for diarization."); + throw new Error("Missing Deepgram API key on this device."); } // Deepgram ingests the whole file in one request, so it needs no // chunking — which is exactly why its speaker labels stay consistent @@ -397,16 +417,18 @@ export class TranscriptionService { } private async getClient(): Promise { - const apiKey = this.plugin.settings.openAIApiKey?.trim(); + this.assertActive(); + const apiKey = this.plugin.credentials.get(this.plugin.settings, "openai"); if (!apiKey) { - throw new Error("Missing OpenAI API key"); + throw new Error("Missing OpenAI API key on this device"); } if (this.client && this.cachedApiKey === apiKey) { return this.client; } - const { OpenAI } = await import("openai"); + const { OpenAI } = await this.loadOpenAI(); + this.assertActive(); this.client = new OpenAI({ apiKey, dangerouslyAllowBrowser: true, @@ -415,4 +437,20 @@ export class TranscriptionService { return this.client; } + + /** Drop the client and its credential material when the plugin unloads. */ + dispose(): void { + this.disposed = true; + this.pendingEpisodes = []; + this.clearCredentialCache(); + } + + clearCredentialCache(): void { + this.client = null; + this.cachedApiKey = null; + } + + private assertActive(): void { + if (this.disposed) throw new Error("PodNotes was unloaded before transcription started."); + } } diff --git a/src/services/diarization/index.ts b/src/services/diarization/index.ts index 80f62afb..a09453d0 100644 --- a/src/services/diarization/index.ts +++ b/src/services/diarization/index.ts @@ -1,4 +1,5 @@ import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; +import type { CredentialKind } from "src/types/Credentials"; export * from "./types"; export * from "./segments"; @@ -8,16 +9,19 @@ export { diarizeWithDeepgram, type RequestUrlFn } from "./deepgramProvider"; /** * Whether the credentials the active transcription mode needs are present. * - * Diarization via Deepgram needs the dedicated `diarizationApiKey`; every other - * mode (plain Whisper, or OpenAI diarization) reuses `openAIApiKey`. Used to gate + * Diarization via Deepgram needs the dedicated Deepgram credential; every other + * mode (plain Whisper, or OpenAI diarization) reuses the OpenAI credential. Used to gate * the transcribe command and guard the service so a user can't kick off a run * that is certain to fail for a missing key. Pure so it is unit-testable and * usable from both the command callback and the service. */ -export function requiredTranscriptionKeyPresent(settings: IPodNotesSettings): boolean { +export function requiredTranscriptionKeyPresent( + settings: IPodNotesSettings, + hasCredential: (kind: CredentialKind) => boolean, +): boolean { const diarization = settings.transcript?.diarization; if (diarization?.enabled && diarization.provider === "deepgram") { - return Boolean(settings.diarizationApiKey?.trim()); + return hasCredential("deepgram"); } - return Boolean(settings.openAIApiKey?.trim()); + return hasCredential("openai"); } diff --git a/src/services/diarization/requiredTranscriptionKeyPresent.test.ts b/src/services/diarization/requiredTranscriptionKeyPresent.test.ts index 40eea65e..b1dd5f7e 100644 --- a/src/services/diarization/requiredTranscriptionKeyPresent.test.ts +++ b/src/services/diarization/requiredTranscriptionKeyPresent.test.ts @@ -7,71 +7,54 @@ function settings(overrides: Partial = {}): IPodNotesSettings return structuredClone({ ...DEFAULT_SETTINGS, ...overrides }); } -function withDiarization( - provider: "openai" | "deepgram", - enabled: boolean, - keys: { openAIApiKey?: string; diarizationApiKey?: string } = {}, -): IPodNotesSettings { - const s = settings(keys); +function withDiarization(provider: "openai" | "deepgram", enabled: boolean): IPodNotesSettings { + const s = settings(); s.transcript.diarization.enabled = enabled; s.transcript.diarization.provider = provider; return s; } +const has = + (...kinds: Array<"openai" | "deepgram">) => + (kind: "openai" | "deepgram") => + kinds.includes(kind); + describe("requiredTranscriptionKeyPresent (#168)", () => { it("requires the OpenAI key when diarization is off", () => { - expect(requiredTranscriptionKeyPresent(settings({ openAIApiKey: "sk" }))).toBe(true); - expect(requiredTranscriptionKeyPresent(settings({ openAIApiKey: "" }))).toBe(false); + expect(requiredTranscriptionKeyPresent(settings(), has("openai"))).toBe(true); + expect(requiredTranscriptionKeyPresent(settings(), has())).toBe(false); }); it("requires the OpenAI key for the OpenAI diarization provider", () => { expect( - requiredTranscriptionKeyPresent( - withDiarization("openai", true, { openAIApiKey: "sk" }), - ), + requiredTranscriptionKeyPresent(withDiarization("openai", true), has("openai")), ).toBe(true); expect( - requiredTranscriptionKeyPresent( - withDiarization("openai", true, { - openAIApiKey: "", - diarizationApiKey: "dg", - }), - ), + requiredTranscriptionKeyPresent(withDiarization("openai", true), has("deepgram")), ).toBe(false); }); it("requires the Deepgram key for the Deepgram diarization provider", () => { expect( - requiredTranscriptionKeyPresent( - withDiarization("deepgram", true, { diarizationApiKey: "dg" }), - ), + requiredTranscriptionKeyPresent(withDiarization("deepgram", true), has("deepgram")), ).toBe(true); // An OpenAI key does not satisfy the Deepgram provider. expect( - requiredTranscriptionKeyPresent( - withDiarization("deepgram", true, { - openAIApiKey: "sk", - diarizationApiKey: "", - }), - ), + requiredTranscriptionKeyPresent(withDiarization("deepgram", true), has("openai")), ).toBe(false); }); it("falls back to the OpenAI key when Deepgram is selected but diarization is off", () => { // provider=deepgram only matters when diarization is enabled. expect( - requiredTranscriptionKeyPresent( - withDiarization("deepgram", false, { openAIApiKey: "sk" }), - ), + requiredTranscriptionKeyPresent(withDiarization("deepgram", false), has("openai")), ).toBe(true); }); - it("treats whitespace-only keys as absent", () => { - expect(requiredTranscriptionKeyPresent(settings({ openAIApiKey: " " }))).toBe(false); - expect( - requiredTranscriptionKeyPresent( - withDiarization("deepgram", true, { diarizationApiKey: " " }), - ), - ).toBe(false); + it("treats dangling SecretStorage references as absent", () => { + expect(requiredTranscriptionKeyPresent(settings(), has())).toBe(false); + expect(requiredTranscriptionKeyPresent(withDiarization("deepgram", true), has())).toBe( + false, + ); }); }); diff --git a/src/settingsTransfer.test.ts b/src/settingsTransfer.test.ts index 1a2b425c..9a39ae6d 100644 --- a/src/settingsTransfer.test.ts +++ b/src/settingsTransfer.test.ts @@ -19,7 +19,7 @@ const NOW = "2026-06-14T00:00:00.000Z"; describe("serializeSettings", () => { it("wraps settings in a versioned envelope", () => { - const envelope = serializeSettings(makeSettings(), { includeSecret: false }, "2.16.0", NOW); + const envelope = serializeSettings(makeSettings(), {}, "2.16.0", NOW); expect(envelope.type).toBe(SETTINGS_EXPORT_TYPE); expect(envelope.version).toBe(SETTINGS_EXPORT_VERSION); @@ -37,52 +37,55 @@ describe("serializeSettings", () => { currentEpisode: { title: "ep" } as never, }); - const { settings: exported } = serializeSettings( - settings, - { includeSecret: false }, - "2.16.0", - NOW, - ); + const { settings: exported } = serializeSettings(settings, {}, "2.16.0", NOW); for (const key of EXCLUDED_KEYS) { expect(exported).not.toHaveProperty(key as string); } }); - it("omits the API key unless includeSecret is set", () => { - const settings = makeSettings({ openAIApiKey: "sk-secret" }); + it("keeps SecretStorage references out and puts opted-in values in a separate payload", () => { + const settings = makeSettings({ openAISecretId: "podnotes-openai-api-key" }); - const without = serializeSettings(settings, { includeSecret: false }, "2.16.0", NOW); - expect(without.settings).not.toHaveProperty("openAIApiKey"); + const ordinary = serializeSettings(settings, {}, "2.16.0", NOW); + expect(ordinary.settings).not.toHaveProperty("openAISecretId"); + expect(ordinary).not.toHaveProperty("secrets"); - const withKey = serializeSettings(settings, { includeSecret: true }, "2.16.0", NOW); - expect(withKey.settings.openAIApiKey).toBe("sk-secret"); + const withKey = serializeSettings( + settings, + { secrets: { openAI: "sk-secret" } }, + "2.16.0", + NOW, + ); + expect(withKey.settings).not.toHaveProperty("openAISecretId"); + expect(withKey.secrets).toEqual({ openAI: "sk-secret" }); }); - it("redacts the diarization (Deepgram) key alongside the OpenAI key (#168)", () => { + it("keeps both provider values in the explicit top-level payload (#168)", () => { const settings = makeSettings({ - openAIApiKey: "sk-secret", - diarizationApiKey: "dg-secret", + openAISecretId: "podnotes-openai-api-key", + deepgramSecretId: "podnotes-deepgram-api-key", }); - const without = serializeSettings(settings, { includeSecret: false }, "2.16.0", NOW); - expect(without.settings).not.toHaveProperty("openAIApiKey"); - expect(without.settings).not.toHaveProperty("diarizationApiKey"); + const without = serializeSettings(settings, {}, "2.16.0", NOW); + expect(without.settings).not.toHaveProperty("openAISecretId"); + expect(without.settings).not.toHaveProperty("deepgramSecretId"); - const withKey = serializeSettings(settings, { includeSecret: true }, "2.16.0", NOW); - expect(withKey.settings.diarizationApiKey).toBe("dg-secret"); + const withKeys = serializeSettings( + settings, + { secrets: { openAI: "sk-secret", deepgram: "dg-secret" } }, + "2.16.0", + NOW, + ); + expect(withKeys.secrets).toEqual({ openAI: "sk-secret", deepgram: "dg-secret" }); + expect(JSON.stringify(withKeys.settings)).not.toContain("secret"); }); it("never lets the Deepgram key ride along inside the transcript object (#168)", () => { // The key lives top-level precisely so it can be redacted; the nested // transcript object (copied wholesale) must never carry a secret. - const settings = makeSettings({ diarizationApiKey: "dg-secret" }); - const { settings: exported } = serializeSettings( - settings, - { includeSecret: false }, - "2.16.0", - NOW, - ); + const settings = makeSettings({ deepgramSecretId: "podnotes-deepgram-api-key" }); + const { settings: exported } = serializeSettings(settings, {}, "2.16.0", NOW); expect(JSON.stringify(exported.transcript ?? {})).not.toContain("dg-secret"); }); @@ -94,17 +97,35 @@ describe("serializeSettings", () => { }, }); - const { settings: exported } = serializeSettings( - settings, - { includeSecret: false }, - "2.16.0", - NOW, - ); + const { settings: exported } = serializeSettings(settings, {}, "2.16.0", NOW); expect(exported.note).toEqual(settings.note); expect(exported.savedFeeds).toEqual(settings.savedFeeds); expect(exported.timestamp).toEqual(settings.timestamp); }); + + it("encodes dates inside exported playlists as canonical text", () => { + const date = new Date("2024-03-01T10:05:03.000Z"); + const settings = makeSettings({ + queue: { + ...DEFAULT_SETTINGS.queue, + episodes: [ + { + title: "Queued", + streamUrl: "queued.mp3", + url: "", + description: "", + content: "", + podcastName: "Podcast", + episodeDate: date, + }, + ], + }, + }); + + const exported = serializeSettings(settings, {}, "2.17.3", NOW); + expect(exported.settings.queue?.episodes[0].episodeDate).toBe(date.toISOString()); + }); }); describe("parseImport", () => { @@ -149,7 +170,7 @@ describe("parseImport", () => { defaultPlaybackRate: 1.5, note: { path: "p", template: "t" }, }); - const envelope = serializeSettings(settings, { includeSecret: false }, "2.16.0", NOW); + const envelope = serializeSettings(settings, {}, "2.16.0", NOW); const result = parseImport(JSON.stringify(envelope)); expect(result.ok).toBe(true); @@ -191,6 +212,13 @@ describe("parseImport", () => { expect(result.ok).toBe(false); }); + it("rejects a raw data file from a newer persistence schema", () => { + const result = parseImport(JSON.stringify({ schemaVersion: 3, defaultVolume: 0.5 })); + expect(result).toEqual( + expect.objectContaining({ ok: false, error: expect.stringContaining("schema v3") }), + ); + }); + it("drops a wrong-typed top-level value", () => { const result = parseImport( JSON.stringify({ defaultVolume: "evil", note: { path: "p", template: "t" } }), @@ -268,11 +296,63 @@ describe("parseImport", () => { const result = parseImport(JSON.stringify({ openAIApiKey: "sk-real" })); expect(result.ok).toBe(true); if (result.ok) { - expect(result.settings.openAIApiKey).toBe("sk-real"); + expect(result.settings).not.toHaveProperty("openAIApiKey"); + expect(result.secrets.openAI).toBe("sk-real"); expect(result.meta.includesSecret).toBe(true); } }); + it("parses the v2 secrets payload separately from settings", () => { + const result = parseImport( + JSON.stringify({ + type: SETTINGS_EXPORT_TYPE, + version: 2, + settings: { defaultVolume: 0.5, openAISecretId: "not-transferable" }, + secrets: { openAI: " sk ", deepgram: "dg" }, + }), + ); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.settings).not.toHaveProperty("openAISecretId"); + expect(result.secrets).toEqual({ openAI: "sk", deepgram: "dg" }); + } + }); + + it("rejects a malformed v2 secrets payload", () => { + const result = parseImport( + JSON.stringify({ + type: SETTINGS_EXPORT_TYPE, + version: 2, + settings: { defaultVolume: 0.5 }, + secrets: { openAI: 42 }, + }), + ); + expect(result).toEqual(expect.objectContaining({ ok: false })); + }); + + it("extracts legacy values from v1 envelopes but not from v2 data", () => { + const legacy = parseImport( + JSON.stringify({ + type: SETTINGS_EXPORT_TYPE, + version: 1, + settings: { openAIApiKey: "sk-legacy" }, + }), + ); + expect(legacy).toEqual( + expect.objectContaining({ ok: true, secrets: { openAI: "sk-legacy" } }), + ); + + const retiredInV2 = parseImport( + JSON.stringify({ + schemaVersion: 2, + defaultVolume: 0.5, + openAIApiKey: "must-not-return", + }), + ); + expect(retiredInV2).toEqual(expect.objectContaining({ ok: true, secrets: {} })); + }); + it("drops a built-in playlist whose episodes is not an array (PL-10)", () => { const result = parseImport( JSON.stringify({ @@ -358,6 +438,29 @@ describe("mergeImportedSettings", () => { expect(merged.timestamp.template).toBe(DEFAULT_SETTINGS.timestamp.template); }); + it("revives imported playlist dates before hydrating live stores", () => { + const date = "2024-03-01T10:05:03.000Z"; + const current = makeSettings(); + const merged = mergeImportedSettings(current, { + queue: { + ...DEFAULT_SETTINGS.queue, + episodes: [ + { + title: "Queued", + streamUrl: "queued.mp3", + url: "", + description: "", + content: "", + podcastName: "Podcast", + episodeDate: date as unknown as Date, + }, + ], + }, + }); + + expect(merged.queue.episodes[0].episodeDate).toEqual(new Date(date)); + }); + it("replaces collection settings wholesale rather than merging them", () => { const current = makeSettings({ savedFeeds: { @@ -376,16 +479,16 @@ describe("mergeImportedSettings", () => { it("preserves a saved secret when the import omits it (IE-05)", () => { const current = makeSettings({ - openAIApiKey: "sk-saved", - diarizationApiKey: "dg-saved", + openAISecretId: "saved-openai", + deepgramSecretId: "saved-deepgram", }); // parseImport strips empty secrets, so a raw data.json with blank keys // arrives here without them; the merge must keep the configured keys. const merged = mergeImportedSettings(current, { defaultVolume: 0.5 }); - expect(merged.openAIApiKey).toBe("sk-saved"); - expect(merged.diarizationApiKey).toBe("dg-saved"); + expect(merged.openAISecretId).toBe("saved-openai"); + expect(merged.deepgramSecretId).toBe("saved-deepgram"); }); it("keeps the current nested value when the import omits the field", () => { @@ -434,17 +537,16 @@ describe("mergeImportedSettings", () => { describe("describeSecrets (#168)", () => { it("names only the secrets that actually hold a value", () => { - expect(describeSecrets(makeSettings())).toEqual([]); - expect(describeSecrets(makeSettings({ openAIApiKey: "sk" }))).toEqual(["OpenAI API key"]); - expect(describeSecrets(makeSettings({ diarizationApiKey: "dg" }))).toEqual([ + expect(describeSecrets({})).toEqual([]); + expect(describeSecrets({ openAI: "sk" })).toEqual(["OpenAI API key"]); + expect(describeSecrets({ deepgram: "dg" })).toEqual(["Deepgram API key"]); + expect(describeSecrets({ openAI: "sk", deepgram: "dg" })).toEqual([ + "OpenAI API key", "Deepgram API key", ]); - expect( - describeSecrets(makeSettings({ openAIApiKey: "sk", diarizationApiKey: "dg" })), - ).toEqual(["OpenAI API key", "Deepgram API key"]); }); it("ignores whitespace-only secrets", () => { - expect(describeSecrets(makeSettings({ openAIApiKey: " " }))).toEqual([]); + expect(describeSecrets({ openAI: " " })).toEqual([]); }); }); diff --git a/src/settingsTransfer.ts b/src/settingsTransfer.ts index 2955a604..f9675527 100644 --- a/src/settingsTransfer.ts +++ b/src/settingsTransfer.ts @@ -1,5 +1,12 @@ import { DEFAULT_SETTINGS } from "./constants"; -import { migrateTranscriptSettings } from "./settingsMigrations"; +import { + decodePodNotesData, + encodePodNotesData, + PODNOTES_DATA_SCHEMA_VERSION, + type PersistedPodNotesSettings, + PodNotesDataError, +} from "./persistence/podNotesData"; +import type { CredentialValues } from "./types/Credentials"; import type { IPodNotesSettings } from "./types/IPodNotesSettings"; /** @@ -13,7 +20,7 @@ import type { IPodNotesSettings } from "./types/IPodNotesSettings"; */ export const SETTINGS_EXPORT_TYPE = "podnotes-settings"; -export const SETTINGS_EXPORT_VERSION = 1; +export const SETTINGS_EXPORT_VERSION = 2; /** * Runtime / vault-specific state that must never travel between vaults: playback @@ -28,22 +35,22 @@ export const EXCLUDED_KEYS: readonly (keyof IPodNotesSettings)[] = [ "currentEpisode", ]; -/** - * API keys are only exported when the user explicitly opts in. Both the OpenAI - * key and the dedicated diarization (Deepgram) key are top-level so they can be - * redacted by name here; the diarization key is deliberately NOT nested inside - * `transcript` (a wholesale-copied nested key) so it can never leak (#168). - */ -export const SECRET_KEYS: ReadonlySet = new Set([ - "openAIApiKey", - "diarizationApiKey", +/** SecretStorage references are device-local implementation details, not settings transfer data. */ +export const SECRET_REFERENCE_KEYS: ReadonlySet = new Set([ + "openAISecretId", + "deepgramSecretId", ]); +const LEGACY_SECRET_KEYS = { + openAIApiKey: "openAI", + diarizationApiKey: "deepgram", +} as const; + /** Human-facing names for each secret, so export/import copy can name exactly * which keys leave or enter the vault instead of hard-coding "OpenAI". */ -const SECRET_KEY_LABELS: Record = { - openAIApiKey: "OpenAI API key", - diarizationApiKey: "Deepgram API key", +const SECRET_KEY_LABELS: Record = { + openAI: "OpenAI API key", + deepgram: "Deepgram API key", }; /** @@ -52,10 +59,10 @@ const SECRET_KEY_LABELS: Record = { * import confirmation honest about which keys are involved — so a Deepgram-only * user is never told only "OpenAI API key" (and vice versa). See issue #168. */ -export function describeSecrets(settings: Partial): string[] { - return [...SECRET_KEYS] - .filter((key) => Boolean((settings[key] as string | undefined)?.trim())) - .map((key) => SECRET_KEY_LABELS[key as string]); +export function describeSecrets(secrets: CredentialValues): string[] { + return (Object.keys(SECRET_KEY_LABELS) as (keyof CredentialValues)[]) + .filter((key) => Boolean(secrets[key]?.trim())) + .map((key) => SECRET_KEY_LABELS[key]); } /** Keys that, if copied into the settings object, could pollute Object.prototype. */ @@ -64,7 +71,9 @@ const DANGEROUS_KEYS = new Set(["__proto__", "constructor", "prototype"]); /** Known top-level setting keys that are safe to import (excludes runtime state). */ const IMPORTABLE_KEYS = new Set( Object.keys(DEFAULT_SETTINGS).filter( - (key) => !EXCLUDED_KEYS.includes(key as keyof IPodNotesSettings), + (key) => + !EXCLUDED_KEYS.includes(key as keyof IPodNotesSettings) && + !SECRET_REFERENCE_KEYS.has(key as keyof IPodNotesSettings), ), ); @@ -83,18 +92,21 @@ export interface SettingsEnvelope { version: number; pluginVersion: string; exportedAt: string; - settings: Partial; + settings: Partial; + /** Present only after the user explicitly opts into a plaintext credential backup. */ + secrets?: CredentialValues; } export interface ExportOptions { - /** Include the (plaintext) OpenAI API key in the export. */ - includeSecret: boolean; + /** Explicit values resolved by the UI after the user opts in. Never inferred here. */ + secrets?: CredentialValues; } export type ParseResult = | { ok: true; settings: Partial; + secrets: CredentialValues; meta: { fromEnvelope: boolean; version: number | null; @@ -106,8 +118,9 @@ export type ParseResult = /** * Build a versioned export envelope from the live settings, copying only - * allow-listed keys by name (never spreading the whole object). Runtime state is - * always excluded; the API key is excluded unless `opts.includeSecret` is set. + * allow-listed keys by name (never spreading the whole object). Runtime state and + * SecretStorage references are always excluded. Plaintext values appear only in + * the separate payload explicitly passed by the caller. */ export function serializeSettings( settings: IPodNotesSettings, @@ -116,20 +129,22 @@ export function serializeSettings( nowISO: string, ): SettingsEnvelope { const out: Record = {}; + const persisted = encodePodNotesData(settings); for (const key of Object.keys(settings)) { if (DANGEROUS_KEYS.has(key)) continue; if (!IMPORTABLE_KEYS.has(key)) continue; - if (SECRET_KEYS.has(key as keyof IPodNotesSettings) && !opts.includeSecret) continue; - out[key] = settings[key as keyof IPodNotesSettings]; + out[key] = persisted[key]; } + const secrets = normalizeSecrets(opts.secrets ?? {}); return { type: SETTINGS_EXPORT_TYPE, version: SETTINGS_EXPORT_VERSION, pluginVersion, exportedAt: nowISO, - settings: out as Partial, + settings: out as Partial, + ...(Object.keys(secrets).length > 0 ? { secrets } : {}), }; } @@ -154,6 +169,7 @@ export function parseImport(jsonText: string): ParseResult { let fromEnvelope = false; let version: number | null = null; let pluginVersion: string | null = null; + let secrets: CredentialValues = {}; if (raw.type === SETTINGS_EXPORT_TYPE) { fromEnvelope = true; @@ -181,11 +197,34 @@ export function parseImport(jsonText: string): ParseResult { } source = raw.settings; + if (version >= 2) { + const parsedSecrets = parseSecretsPayload(raw.secrets); + if ("error" in parsedSecrets) return { ok: false, error: parsedSecrets.error }; + secrets = parsedSecrets.values; + } else { + secrets = extractLegacySecrets(source); + } + } else if (Object.prototype.hasOwnProperty.call(raw, "schemaVersion")) { + try { + // Validate raw data.json imports against the same schema gate as plugin + // startup. The field sanitizer below still keeps import partial. + const decoded = decodePodNotesData(raw); + if (decoded.sourceVersion < PODNOTES_DATA_SCHEMA_VERSION) { + secrets = extractLegacySecrets(source); + } + } catch (error) { + if (error instanceof PodNotesDataError) { + return { ok: false, error: error.message }; + } + throw error; + } + } else { + secrets = extractLegacySecrets(source); } const settings = sanitizeImportedSettings(source); - if (Object.keys(settings).length === 0) { + if (Object.keys(settings).length === 0 && Object.keys(secrets).length === 0) { return { ok: false, error: "No recognizable PodNotes settings were found in the file.", @@ -195,11 +234,12 @@ export function parseImport(jsonText: string): ParseResult { return { ok: true, settings, + secrets, meta: { fromEnvelope, version, pluginVersion, - includesSecret: [...SECRET_KEYS].some((key) => key in settings), + includesSecret: Object.keys(secrets).length > 0, }, }; } @@ -224,14 +264,10 @@ export function mergeImportedSettings( } as never; } - // The per-key spread above is only one level deep, so an imported - // `transcript.diarization` overrides the whole nested object — which could - // carry an unknown provider or drop `speakerTemplate`. Run the same migration - // the load path uses so the import path converges on a clamped, fully-formed - // transcript instead of relying on the next reload to repair it (#168). - merged.transcript = migrateTranscriptSettings(merged.transcript); - - return merged; + // Converge import and startup on the same deep validators and date revival. + // Mark the in-memory merge as current so it cannot be mistaken for a legacy + // data migration while preserving the import envelope's independent version. + return decodePodNotesData({ ...merged, schemaVersion: PODNOTES_DATA_SCHEMA_VERSION }).settings; } function sanitizeImportedSettings(source: Record): Partial { @@ -248,14 +284,6 @@ function sanitizeImportedSettings(source: Record): Partial, @@ -281,6 +309,43 @@ function sanitizeImportedSettings(source: Record): Partial; } +function extractLegacySecrets(source: Record): CredentialValues { + const values: CredentialValues = {}; + + for (const [legacyKey, credentialKey] of Object.entries(LEGACY_SECRET_KEYS) as Array< + [keyof typeof LEGACY_SECRET_KEYS, keyof CredentialValues] + >) { + const value = source[legacyKey]; + if (typeof value === "string" && value.trim()) values[credentialKey] = value.trim(); + } + + return values; +} + +function parseSecretsPayload(value: unknown): { values: CredentialValues } | { error: string } { + if (value === undefined) return { values: {} }; + if (!isPlainObject(value)) { + return { error: "Export file has an invalid secrets payload." }; + } + + for (const key of ["openAI", "deepgram"] as const) { + if (value[key] !== undefined && typeof value[key] !== "string") { + return { error: `Export file has an invalid ${SECRET_KEY_LABELS[key]}.` }; + } + } + + return { values: normalizeSecrets(value as CredentialValues) }; +} + +function normalizeSecrets(values: CredentialValues): CredentialValues { + const openAI = values.openAI?.trim(); + const deepgram = values.deepgram?.trim(); + return { + ...(openAI ? { openAI } : {}), + ...(deepgram ? { deepgram } : {}), + }; +} + /** Drop any playlist entry that isn't a plain object with an `episodes` array. */ function sanitizePlaylistMap(value: Record): Record { const out: Record = {}; diff --git a/src/store/feeds.ts b/src/store/feeds.ts index 887487ce..22fef496 100644 --- a/src/store/feeds.ts +++ b/src/store/feeds.ts @@ -1,7 +1,11 @@ import { get, readable, writable } from "svelte/store"; import type { Episode } from "src/types/Episode"; import type { PodcastFeed } from "src/types/PodcastFeed"; -import { DEFAULT_EPISODE_LIST_LIMIT, MAX_EPISODE_LIST_LIMIT } from "src/constants"; +import { DEFAULT_EPISODE_LIST_LIMIT } from "src/constants"; +import { dateTimestamp } from "src/persistence/dateCodec"; +import { sanitizeEpisodeListLimit } from "src/utility/episodeListLimit"; + +export { sanitizeEpisodeListLimit } from "src/utility/episodeListLimit"; /** * Saved-feed metadata, the per-feed episode cache, and the aggregated "Latest @@ -23,31 +27,11 @@ export const episodeCache = writable<{ [podcastName: string]: Episode[] }>({}); */ export const episodeListLimit = writable(DEFAULT_EPISODE_LIST_LIMIT); -/** - * Coerce a stored/raw limit into a usable positive integer, falling back to the - * default for missing/NaN/zero/negative values and clamping the upper bound so a - * stray huge number can't materialise an unbounded list. - */ -export function sanitizeEpisodeListLimit(value: unknown): number { - const numeric = typeof value === "number" ? value : Number(value); - if (!Number.isFinite(numeric) || numeric < 1) { - return DEFAULT_EPISODE_LIST_LIMIT; - } - - return Math.min(Math.floor(numeric), MAX_EPISODE_LIST_LIMIT); -} - type LatestEpisodesByFeed = Map; type FeedEpisodeSources = Map; function getEpisodeTimestamp(episode?: Episode): number { - if (!episode?.episodeDate) return 0; - - // An Invalid Date coerces to NaN, which makes every comparison false and - // produces an unstable/incorrect sort order. Collapse it to 0 (sorts as - // oldest), matching FeedCacheService.episodeTimestamp (FP-12). - const timestamp = Number(episode.episodeDate); - return Number.isFinite(timestamp) ? timestamp : 0; + return dateTimestamp(episode?.episodeDate) ?? 0; } function getLatestEpisodesForFeed(episodes: Episode[], perFeedLimit: number): Episode[] { diff --git a/src/types/Credentials.ts b/src/types/Credentials.ts new file mode 100644 index 00000000..e6fcdb2e --- /dev/null +++ b/src/types/Credentials.ts @@ -0,0 +1,16 @@ +export type CredentialKind = "openai" | "deepgram"; + +export interface CredentialValues { + openAI?: string; + deepgram?: string; +} + +export interface CredentialReferences { + openAISecretId?: string; + deepgramSecretId?: string; +} + +/** Obsidian SecretStorage IDs are lowercase alphanumeric strings with dashes. */ +export function isValidSecretId(value: string): boolean { + return /^(?=.*[a-z0-9])[a-z0-9-]+$/.test(value); +} diff --git a/src/types/Episode.ts b/src/types/Episode.ts index 92053ae6..a0e2b7e2 100644 --- a/src/types/Episode.ts +++ b/src/types/Episode.ts @@ -1,6 +1,16 @@ export type EpisodeMediaType = "audio" | "video"; export interface Episode { + /** PodNotes' canonical identity, stabilized across observations by reconciliation. */ + episodeId?: string; + /** Current strong locators plus bounded, trusted reconciliation history. */ + episodeAliases?: string[]; + /** Canonical identity of the parent feed. */ + feedId?: string; + /** Direct-child RSS item GUID, retained as later reconciliation evidence. */ + guid?: string; + /** The item's own RSS link, excluding the compatibility fallback in `url`. */ + itemLink?: string; title: string; streamUrl: string; url: string; diff --git a/src/types/IPodNotesSettings.ts b/src/types/IPodNotesSettings.ts index 535514e8..884d507d 100644 --- a/src/types/IPodNotesSettings.ts +++ b/src/types/IPodNotesSettings.ts @@ -53,13 +53,10 @@ export interface IPodNotesSettings { path: string; }; downloadedEpisodes: { [podcastName: string]: DownloadedEpisode[] }; - openAIApiKey: string; - /** - * API key for the dedicated diarization provider (Deepgram). Kept separate - * from `openAIApiKey` and top-level so the settings export can redact it as a - * secret; OpenAI diarization reuses `openAIApiKey` instead. See issue #168. - */ - diarizationApiKey: string; + /** SecretStorage ID for the OpenAI API key. Never contains the key itself. */ + openAISecretId: string; + /** SecretStorage ID for the Deepgram API key. Never contains the key itself. */ + deepgramSecretId: string; transcript: { path: string; template: string; diff --git a/src/types/PodcastFeed.ts b/src/types/PodcastFeed.ts index ab7155a8..4a65147b 100644 --- a/src/types/PodcastFeed.ts +++ b/src/types/PodcastFeed.ts @@ -1,4 +1,8 @@ export interface PodcastFeed { + /** PodNotes' canonical stable identity. */ + feedId?: string; + /** Direct-child Podcasting 2.0 channel GUID. Never used as feed identity alone. */ + guid?: string; title: string; url: string; artworkUrl: string; diff --git a/src/ui/settings/PodNotesSettingsTab.import.test.ts b/src/ui/settings/PodNotesSettingsTab.import.test.ts index f1f9f51d..ddeda4f3 100644 --- a/src/ui/settings/PodNotesSettingsTab.import.test.ts +++ b/src/ui/settings/PodNotesSettingsTab.import.test.ts @@ -1,6 +1,7 @@ import { get } from "svelte/store"; import { describe, expect, it, vi } from "vitest"; import type { App } from "obsidian"; +import * as obsidian from "obsidian"; import { DEFAULT_SETTINGS } from "src/constants"; import type PodNotes from "src/main"; @@ -12,7 +13,9 @@ describe("PodNotesSettingsTab settings import", () => { playbackRate.set(1); const plugin = { settings: structuredClone(DEFAULT_SETTINGS), + credentials: { storeValues: vi.fn(() => ({})) }, saveSettings: vi.fn().mockResolvedValue(undefined), + saveSettingsStrict: vi.fn().mockResolvedValue(undefined), } as unknown as PodNotes; const tab = new PodNotesSettingsTab({} as App, plugin); vi.spyOn(tab, "display").mockImplementation(() => {}); @@ -27,6 +30,206 @@ describe("PodNotesSettingsTab settings import", () => { expect(plugin.settings.defaultPlaybackRate).toBe(2.3); expect(get(playbackRate)).toBe(2.3); - expect(plugin.saveSettings).toHaveBeenCalledTimes(1); + expect(plugin.saveSettingsStrict).toHaveBeenCalledTimes(2); + }); + + it("restores the previous settings and reports a strict save failure", async () => { + playbackRate.set(1); + const previous = structuredClone(DEFAULT_SETTINGS); + const failure = new Error("disk full"); + const plugin = { + settings: previous, + credentials: { storeValues: vi.fn(() => ({})) }, + saveSettings: vi.fn().mockResolvedValue(undefined), + saveSettingsStrict: vi + .fn() + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce(undefined), + } as unknown as PodNotes; + const tab = new PodNotesSettingsTab({} as App, plugin); + const display = vi.spyOn(tab, "display").mockImplementation(() => {}); + const notice = vi.spyOn(obsidian, "Notice"); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await ( + tab as unknown as { + applyImportedSettings: ( + imported: Partial, + ) => Promise; + } + ).applyImportedSettings({ defaultPlaybackRate: 2.3 }); + + expect(plugin.settings).toBe(previous); + expect(get(playbackRate)).toBe(1); + expect(display).not.toHaveBeenCalled(); + expect(notice).toHaveBeenCalledWith( + "Could not import PodNotes settings. Previous settings were kept.", + 10000, + ); + expect(plugin.saveSettingsStrict).toHaveBeenCalledTimes(2); + }); + + it("stores imported secret values first and persists only their references", async () => { + const plugin = { + settings: structuredClone(DEFAULT_SETTINGS), + credentials: { + storeValues: vi.fn(() => ({ + openAISecretId: "podnotes-openai-api-key", + deepgramSecretId: "podnotes-deepgram-api-key", + })), + }, + invalidateTranscriptionCredentialCache: vi.fn(), + saveSettings: vi.fn().mockResolvedValue(undefined), + saveSettingsStrict: vi.fn().mockResolvedValue(undefined), + } as unknown as PodNotes; + const tab = new PodNotesSettingsTab({} as App, plugin); + vi.spyOn(tab, "display").mockImplementation(() => {}); + + await ( + tab as unknown as { + applyImportedSettings: ( + imported: Partial, + secrets: { openAI?: string; deepgram?: string }, + ) => Promise; + } + ).applyImportedSettings({ defaultVolume: 0.4 }, { openAI: "sk", deepgram: "dg" }); + + expect(plugin.credentials.storeValues).toHaveBeenCalledWith({ + openAI: "sk", + deepgram: "dg", + }); + expect(plugin.settings.openAISecretId).toBe("podnotes-openai-api-key"); + expect(plugin.settings.deepgramSecretId).toBe("podnotes-deepgram-api-key"); + expect(plugin.invalidateTranscriptionCredentialCache).toHaveBeenCalledOnce(); + expect(plugin.settings).not.toHaveProperty("openAIApiKey"); + expect(plugin.settings).not.toHaveProperty("diarizationApiKey"); + }); + + it("does not mutate settings when SecretStorage import fails partway", async () => { + const previous = structuredClone(DEFAULT_SETTINGS); + const plugin = { + settings: previous, + credentials: { + storeValues: vi.fn(() => { + throw new Error("second credential failed"); + }), + }, + saveSettings: vi.fn().mockResolvedValue(undefined), + saveSettingsStrict: vi.fn().mockResolvedValue(undefined), + } as unknown as PodNotes; + const tab = new PodNotesSettingsTab({} as App, plugin); + const notice = vi.spyOn(obsidian, "Notice"); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await ( + tab as unknown as { + applyImportedSettings: ( + imported: Partial, + secrets: { openAI?: string; deepgram?: string }, + ) => Promise; + } + ).applyImportedSettings({ defaultVolume: 0.4 }, { openAI: "sk", deepgram: "dg" }); + + expect(plugin.settings).toBe(previous); + expect(plugin.saveSettingsStrict).not.toHaveBeenCalled(); + expect(notice).toHaveBeenCalledWith( + expect.stringContaining("retrying will safely reuse"), + 10000, + ); + }); + + it("restores previous secret references when the settings save fails", async () => { + const previous = { + ...structuredClone(DEFAULT_SETTINGS), + openAISecretId: "previous-openai", + }; + const plugin = { + settings: previous, + credentials: { + storeValues: vi.fn(() => ({ openAISecretId: "new-openai" })), + }, + invalidateTranscriptionCredentialCache: vi.fn(), + saveSettings: vi.fn().mockResolvedValue(undefined), + saveSettingsStrict: vi + .fn() + .mockRejectedValueOnce(new Error("disk full")) + .mockResolvedValueOnce(undefined), + } as unknown as PodNotes; + const tab = new PodNotesSettingsTab({} as App, plugin); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + await ( + tab as unknown as { + applyImportedSettings: ( + imported: Partial, + secrets: { openAI?: string }, + ) => Promise; + } + ).applyImportedSettings({ defaultVolume: 0.4 }, { openAI: "sk-new" }); + + expect(plugin.settings).toBe(previous); + expect(plugin.settings.openAISecretId).toBe("previous-openai"); + expect(plugin.invalidateTranscriptionCredentialCache).toHaveBeenCalledTimes(2); + expect(plugin.saveSettingsStrict).toHaveBeenCalledTimes(2); + }); + + it("rolls back a SecretComponent reference when strict persistence fails", async () => { + const plugin = { + settings: { + ...structuredClone(DEFAULT_SETTINGS), + openAISecretId: "previous-secret", + }, + saveSettingsStrict: vi + .fn() + .mockRejectedValueOnce(new Error("disk full")) + .mockResolvedValueOnce(undefined), + invalidateTranscriptionCredentialCache: vi.fn(), + } as unknown as PodNotes; + const tab = new PodNotesSettingsTab({} as App, plugin); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + const saved = await ( + tab as unknown as { + saveSecretReference: (key: "openAISecretId", value: string) => Promise; + } + ).saveSecretReference("openAISecretId", "new-secret"); + + expect(saved).toBe(false); + expect(plugin.settings.openAISecretId).toBe("previous-secret"); + expect(plugin.invalidateTranscriptionCredentialCache).toHaveBeenCalledTimes(2); + expect(plugin.saveSettingsStrict).toHaveBeenCalledTimes(2); + }); + + it("ordinary settings export never resolves or serializes SecretStorage values", async () => { + const create = vi.fn().mockResolvedValue(undefined); + const exportValues = vi.fn(() => ({ openAI: "must-not-be-read" })); + const app = { + vault: { + getAbstractFileByPath: vi.fn(() => null), + create, + }, + } as unknown as App; + const plugin = { + settings: { + ...structuredClone(DEFAULT_SETTINGS), + openAISecretId: "podnotes-openai-api-key", + }, + credentials: { exportValues }, + manifest: { version: "2.17.3" }, + } as unknown as PodNotes; + const tab = new PodNotesSettingsTab(app, plugin); + + await ( + tab as unknown as { + handleSettingsExport: (fileName: string, includeSecret: boolean) => Promise; + } + ).handleSettingsExport("settings.json", false); + + expect(exportValues).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledTimes(1); + const serialized = create.mock.calls[0][1] as string; + expect(serialized).not.toContain("must-not-be-read"); + expect(serialized).not.toContain("openAISecretId"); + expect(JSON.parse(serialized)).not.toHaveProperty("secrets"); }); }); diff --git a/src/ui/settings/PodNotesSettingsTab.ts b/src/ui/settings/PodNotesSettingsTab.ts index 0519664d..2017ccf1 100644 --- a/src/ui/settings/PodNotesSettingsTab.ts +++ b/src/ui/settings/PodNotesSettingsTab.ts @@ -5,6 +5,7 @@ import { Modal, Notice, PluginSettingTab, + SecretComponent, Setting, } from "obsidian"; import type PodNotes from "../../main"; @@ -46,6 +47,7 @@ import { } from "src/settingsTransfer"; import { normalizePlaybackRate } from "src/utility/playbackRate"; import { DEFAULT_SPEAKER_TEMPLATE, type DiarizationProviderId } from "src/services/diarization"; +import { isValidSecretId, type CredentialValues } from "src/types/Credentials"; /** * Stack a Setting's control beneath its name, full width — the layout the @@ -590,7 +592,7 @@ export class PodNotesSettingsTab extends PluginSettingTab { new Setting(containerEl) .setName("Include API keys") .setDesc( - "When enabled, the export file contains your OpenAI and Deepgram API keys in plaintext (whichever you have set). The file is stored in your vault, so it may sync to other devices and be read by other plugins.", + "When enabled, a separate plaintext secrets payload is added for the OpenAI and Deepgram keys available on this device. The file is stored in your vault, so it may sync to other devices and be read by other plugins.", ) .addToggle((toggle) => toggle.setValue(includeSecret).onChange((value) => { @@ -673,9 +675,14 @@ export class PodNotesSettingsTab extends PluginSettingTab { private async handleSettingsExport(fileName: string, includeSecret: boolean): Promise { try { + const secrets = includeSecret + ? this.plugin.credentials.exportValues(this.plugin.settings, { + requireConfigured: true, + }) + : {}; const envelope = serializeSettings( this.plugin.settings, - { includeSecret }, + { secrets: includeSecret ? secrets : undefined }, this.plugin.manifest.version, new Date().toISOString(), ); @@ -690,7 +697,7 @@ export class PodNotesSettingsTab extends PluginSettingTab { } await this.app.vault.create(fileName, contents); - const exportedSecrets = includeSecret ? describeSecrets(this.plugin.settings) : []; + const exportedSecrets = describeSecrets(secrets); new Notice( exportedSecrets.length ? `Exported PodNotes settings to "${fileName}" (includes your ${exportedSecrets.join(" and ")}).` @@ -734,23 +741,74 @@ export class PodNotesSettingsTab extends PluginSettingTab { if (willReplace("playlists", this.plugin.settings.playlists)) { sections.push("playlists"); } - sections.push(...describeSecrets(result.settings)); + sections.push(...describeSecrets(result.secrets)); const detail = sections.length ? ` This also replaces your ${sections.join(", ")}.` : ""; + const secretDetail = + Object.keys(result.secrets).length > 0 + ? " Existing Obsidian secrets are never overwritten; conflicts are saved under a new PodNotes name." + : ""; new ConfirmModal( this.app, "Import PodNotes settings?", - `This overwrites your current PodNotes preferences and templates with the imported values.${detail} Your playback progress and downloads are kept.`, + `This overwrites your current PodNotes preferences and templates with the imported values.${detail}${secretDetail} Your playback progress and downloads are kept.`, "Import", () => { - void this.applyImportedSettings(result.settings); + void this.applyImportedSettings(result.settings, result.secrets); }, ).open(); } - private async applyImportedSettings(imported: Partial): Promise { - const merged = mergeImportedSettings(this.plugin.settings, imported); + private async applyImportedSettings( + imported: Partial, + secrets: CredentialValues = {}, + ): Promise { + const previous = this.plugin.settings; + let secretReferences: Partial = {}; + try { + secretReferences = this.plugin.credentials.storeValues(secrets); + } catch (error) { + new Notice( + "Could not finish importing API keys into Obsidian SecretStorage. Existing settings were kept, and retrying will safely reuse any PodNotes secrets already created.", + 10000, + ); + console.error("PodNotes: failed to import credentials into SecretStorage", error); + return; + } + + const merged = mergeImportedSettings(previous, { ...imported, ...secretReferences }); + const openAIReferenceChanged = merged.openAISecretId !== previous.openAISecretId; + if (openAIReferenceChanged) this.plugin.invalidateTranscriptionCredentialCache(); this.plugin.settings = merged; + try { + // Persist before mutating live stores so a disk failure can restore the + // previous in-memory settings without leaving the UI half-imported. + await this.plugin.saveSettingsStrict(); + } catch (error) { + this.plugin.settings = previous; + if (openAIReferenceChanged) this.plugin.invalidateTranscriptionCredentialCache(); + try { + // A store event may have queued a newer merged snapshot while the first + // write was pending. Queue the rollback after it and wait for durability + // before claiming that the previous settings were kept. + await this.plugin.saveSettingsStrict(); + new Notice( + "Could not import PodNotes settings. Previous settings were kept.", + 10000, + ); + } catch (rollbackError) { + new Notice( + "Could not import PodNotes settings or persist the rollback. Previous settings remain active for this session.", + 10000, + ); + console.error( + "PodNotes: failed to persist settings-import rollback", + rollbackError, + ); + } + console.error("PodNotes: failed to persist imported settings", error); + return; + } // Re-hydrate the live stores so the running UI and the persistence // bindings reflect the import. Keys without a store (templates, paths, @@ -768,7 +826,19 @@ export class PodNotesSettingsTab extends PluginSettingTab { volume.set(Math.min(1, Math.max(0, importedVolume))); playbackRate.set(normalizePlaybackRate(merged.defaultPlaybackRate)); - await this.plugin.saveSettings(); + try { + // Store setters can canonicalize or deduplicate their slices. Await one + // final strict snapshot so the success notice means that live state is + // durable too. + await this.plugin.saveSettingsStrict(); + } catch (error) { + new Notice( + "Imported PodNotes settings, but could not finish saving normalized live state. Change any setting to retry.", + 10000, + ); + console.error("PodNotes: failed to persist normalized imported settings", error); + return; + } // Re-emit the plugin store so an open player/grid recomputes Queue tile/list // visibility (and any other $plugin-derived UI) after an import, mirroring the // autoQueue toggle. Today the queue.set above already triggers that recompute; @@ -779,23 +849,77 @@ export class PodNotesSettingsTab extends PluginSettingTab { new Notice("Imported PodNotes settings."); } + private async saveSecretReference( + key: "openAISecretId" | "deepgramSecretId", + value: string, + ): Promise { + if (value && !isValidSecretId(value)) { + new Notice( + "Obsidian returned an invalid SecretStorage ID. The selection was not saved.", + 0, + ); + return false; + } + + const previous = this.plugin.settings[key]; + if (value === previous) return true; + if (key === "openAISecretId") this.plugin.invalidateTranscriptionCredentialCache(); + this.plugin.settings[key] = value; + + try { + await this.plugin.saveSettingsStrict(); + return true; + } catch (error) { + this.plugin.settings[key] = previous; + if (key === "openAISecretId") this.plugin.invalidateTranscriptionCredentialCache(); + try { + await this.plugin.saveSettingsStrict(); + new Notice( + "Could not save the API key selection. The previous selection was kept.", + 0, + ); + } catch (rollbackError) { + new Notice( + "Could not save the API key selection or its rollback. The previous selection remains active for this session.", + 0, + ); + console.error( + "PodNotes: failed to persist credential-reference rollback", + rollbackError, + ); + } + console.error("PodNotes: failed to persist credential reference", error); + return false; + } + } + private addTranscriptSettings(container: HTMLDivElement) { new Setting(container).setName("Transcript settings").setHeading(); const randomEpisode = getRandomEpisode(); - new Setting(container) - .setName("OpenAI API Key") - .setDesc("Enter your OpenAI API key for transcription functionality.") - .addText((text) => { - text.setPlaceholder("Enter your OpenAI API key") - .setValue(this.plugin.settings.openAIApiKey) - .onChange(async (value) => { - this.plugin.settings.openAIApiKey = value; - await this.plugin.saveSettings(); - }); - text.inputEl.type = "password"; + const openAISetting = new Setting(container).setName("OpenAI API key"); + const updateOpenAIDescription = () => { + openAISetting.setDesc( + this.plugin.credentials.status(this.plugin.settings, "openai") === "missing" + ? "The selected secret is not available on this device. Select an existing secret or create one." + : "Select an existing Obsidian secret or create one for transcription.", + ); + }; + updateOpenAIDescription(); + openAISetting.addComponent((element) => { + const secret = new SecretComponent(this.app, element).setValue( + this.plugin.settings.openAISecretId, + ); + secret.onChange(async (value) => { + const previous = this.plugin.settings.openAISecretId; + if (!(await this.saveSecretReference("openAISecretId", value))) { + secret.setValue(previous); + } + updateOpenAIDescription(); }); + return secret; + }); new Setting(container) .setName("Transcript file path") @@ -841,84 +965,89 @@ export class PodNotesSettingsTab extends PluginSettingTab { this.addDiarizationSettings(container); } - /** - * Opt-in speaker diarization controls (issue #168). Rendered into its own - * container so the provider-dependent fields (Deepgram key) can be shown or - * hidden in place when the toggle/provider changes, without re-rendering the - * whole settings tab. - */ + /** Opt-in speaker diarization controls (issue #168). */ private addDiarizationSettings(container: HTMLElement): void { const diarizationContainer = container.createDiv(); + const diarization = this.plugin.settings.transcript.diarization; + let updateVisibility = () => {}; - const renderDiarizationSettings = () => { - diarizationContainer.empty(); - const diarization = this.plugin.settings.transcript.diarization; + new Setting(diarizationContainer) + .setName("Speaker diarization") + .setDesc( + "Label transcript segments by speaker. Routes the episode audio to a diarization-capable provider instead of plain Whisper.", + ) + .addToggle((toggle) => + toggle.setValue(diarization.enabled).onChange(async (value) => { + this.plugin.settings.transcript.diarization.enabled = value; + await this.plugin.saveSettings(); + updateVisibility(); + }), + ); - new Setting(diarizationContainer) - .setName("Speaker diarization") - .setDesc( - "Label transcript segments by speaker. Routes the episode audio to a diarization-capable provider instead of plain Whisper.", - ) - .addToggle((toggle) => - toggle.setValue(diarization.enabled).onChange(async (value) => { - this.plugin.settings.transcript.diarization.enabled = value; + const providerSetting = new Setting(diarizationContainer) + .setName("Diarization provider") + .setDesc( + "OpenAI reuses your OpenAI API key above (long episodes are chunked, so speaker labels can reset across chunks). Deepgram needs its own key and keeps speaker labels consistent across the whole episode.", + ) + .addDropdown((dropdown) => + dropdown + .addOption("openai", "OpenAI (gpt-4o-transcribe-diarize)") + .addOption("deepgram", "Deepgram") + .setValue(diarization.provider) + .onChange(async (value) => { + this.plugin.settings.transcript.diarization.provider = + value as DiarizationProviderId; await this.plugin.saveSettings(); - renderDiarizationSettings(); + updateVisibility(); }), - ); - - if (!diarization.enabled) return; + ); - new Setting(diarizationContainer) - .setName("Diarization provider") - .setDesc( - "OpenAI reuses your OpenAI API key above (long episodes are chunked, so speaker labels can reset across chunks). Deepgram needs its own key and keeps speaker labels consistent across the whole episode.", - ) - .addDropdown((dropdown) => - dropdown - .addOption("openai", "OpenAI (gpt-4o-transcribe-diarize)") - .addOption("deepgram", "Deepgram") - .setValue(diarization.provider) - .onChange(async (value) => { - this.plugin.settings.transcript.diarization.provider = - value as DiarizationProviderId; - await this.plugin.saveSettings(); - renderDiarizationSettings(); - }), - ); + const deepgramSetting = new Setting(diarizationContainer).setName("Deepgram API key"); + const updateDeepgramDescription = () => { + deepgramSetting.setDesc( + this.plugin.credentials.status(this.plugin.settings, "deepgram") === "missing" + ? "The selected secret is not available on this device. Select an existing secret or create one." + : "Select an Obsidian secret for Deepgram diarization, or create one at deepgram.com.", + ); + }; + updateDeepgramDescription(); + deepgramSetting.addComponent((element) => { + const secret = new SecretComponent(this.app, element).setValue( + this.plugin.settings.deepgramSecretId, + ); + secret.onChange(async (value) => { + const previous = this.plugin.settings.deepgramSecretId; + if (!(await this.saveSecretReference("deepgramSecretId", value))) { + secret.setValue(previous); + } + updateDeepgramDescription(); + }); + return secret; + }); - if (diarization.provider === "deepgram") { - new Setting(diarizationContainer) - .setName("Deepgram API key") - .setDesc("Used only for Deepgram diarization. Create one at deepgram.com.") - .addText((text) => { - text.setPlaceholder("Enter your Deepgram API key") - .setValue(this.plugin.settings.diarizationApiKey) - .onChange(async (value) => { - this.plugin.settings.diarizationApiKey = value; - await this.plugin.saveSettings(); - }); - text.inputEl.type = "password"; - }); - } + const speakerSetting = new Setting(diarizationContainer) + .setName("Speaker label format") + .setDesc( + "Prefix added before each speaker's turn. Use {{speaker}} for the speaker label (OpenAI labels speakers A, B, …; Deepgram labels them 1, 2, …).", + ) + .addText((text) => + text + .setPlaceholder(DEFAULT_SPEAKER_TEMPLATE) + .setValue(diarization.speakerTemplate) + .onChange(async (value) => { + this.plugin.settings.transcript.diarization.speakerTemplate = value; + await this.plugin.saveSettings(); + }), + ); - new Setting(diarizationContainer) - .setName("Speaker label format") - .setDesc( - "Prefix added before each speaker's turn. Use {{speaker}} for the speaker label (OpenAI labels speakers A, B, …; Deepgram labels them 1, 2, …).", - ) - .addText((text) => - text - .setPlaceholder(DEFAULT_SPEAKER_TEMPLATE) - .setValue(diarization.speakerTemplate) - .onChange(async (value) => { - this.plugin.settings.transcript.diarization.speakerTemplate = value; - await this.plugin.saveSettings(); - }), - ); + updateVisibility = () => { + providerSetting.settingEl.toggle(diarization.enabled); + deepgramSetting.settingEl.toggle( + diarization.enabled && diarization.provider === "deepgram", + ); + speakerSetting.settingEl.toggle(diarization.enabled); }; - - renderDiarizationSettings(); + updateVisibility(); } } diff --git a/src/utility/episodeListLimit.ts b/src/utility/episodeListLimit.ts new file mode 100644 index 00000000..b3ac522b --- /dev/null +++ b/src/utility/episodeListLimit.ts @@ -0,0 +1,11 @@ +import { DEFAULT_EPISODE_LIST_LIMIT, MAX_EPISODE_LIST_LIMIT } from "src/constants"; + +/** Coerce a persisted per-feed episode limit into its supported integer range. */ +export function sanitizeEpisodeListLimit(value: unknown): number { + const numeric = typeof value === "number" ? value : Number(value); + if (!Number.isFinite(numeric) || numeric < 1) { + return DEFAULT_EPISODE_LIST_LIMIT; + } + + return Math.min(Math.floor(numeric), MAX_EPISODE_LIST_LIMIT); +} diff --git a/src/utility/formatDate.test.ts b/src/utility/formatDate.test.ts index bacd6e7d..bd6ccbba 100644 --- a/src/utility/formatDate.test.ts +++ b/src/utility/formatDate.test.ts @@ -14,6 +14,19 @@ describe("formatDate", () => { // Noon edge case const noonDate = new Date("2024-01-01T12:00:00"); + it("formats a date restored from JSON text", () => { + expect(formatDate("2024-03-01T10:05:03", "YYYY-MM-DD HH:mm:ss")).toBe( + "2024-03-01 10:05:03", + ); + }); + + it.each([undefined, null, "", "not-a-date", new Date("invalid")])( + "returns an empty value for invalid input %p", + (value) => { + expect(formatDate(value, "YYYY-MM-DD")).toBe(""); + }, + ); + describe("year tokens", () => { it("formats YYYY as 4-digit year", () => { expect(formatDate(testDate, "YYYY")).toBe("2024"); diff --git a/src/utility/formatDate.ts b/src/utility/formatDate.ts index 5ec3afc1..3679797a 100644 --- a/src/utility/formatDate.ts +++ b/src/utility/formatDate.ts @@ -1,3 +1,5 @@ +import { decodeDate } from "src/persistence/dateCodec"; + /** * Formats a date using Moment.js-style format tokens for backward compatibility. * Common tokens supported: @@ -11,7 +13,10 @@ * - A: AM/PM, a: am/pm * - [text]: literal text (escaped, not parsed as tokens) */ -export function formatDate(date: Date, format: string): string { +export function formatDate(value: unknown, format: string): string { + const date = decodeDate(value); + if (!date) return ""; + const year = date.getFullYear(); const month = date.getMonth(); const day = date.getDate(); diff --git a/src/utility/identity.test.ts b/src/utility/identity.test.ts new file mode 100644 index 00000000..1d744723 --- /dev/null +++ b/src/utility/identity.test.ts @@ -0,0 +1,613 @@ +import { describe, expect, test } from "vitest"; +import { + MAX_EPISODE_IDENTITY_ALIASES, + MAX_IDENTITY_COMPONENT_BYTES, + assignEpisodeIdentities, + createFeedId, + createLocalEpisodeId, + getEpisodeIdentityCandidates, + isCanonicalEpisodeId, + isCanonicalFeedId, + normalizeStrongIdentityUrl, + reconcileEpisodeIdentities, + type EpisodeIdentitySource, +} from "./identity"; + +const feedId = createFeedId("https://example.com/feed.xml")!; + +function source(overrides: Partial = {}): EpisodeIdentitySource { + return { + feedId, + guid: "episode-guid", + enclosureUrl: "https://cdn.example.com/episode.mp3?token=secret&part=1", + itemLink: "https://example.com/episodes/1", + publishedAt: "2024-01-01T00:00:00.000Z", + title: "Episode 1", + ...overrides, + }; +} + +function previousEntries(sources: EpisodeIdentitySource[]) { + const assigned = assignEpisodeIdentities(sources); + return sources.map((episodeSource, index) => ({ + episodeId: assigned[index]?.episodeId, + source: episodeSource, + })); +} + +function persistedEntries(sources: EpisodeIdentitySource[]) { + const assigned = assignEpisodeIdentities(sources); + return sources.map((episodeSource, index) => ({ + episodeId: assigned[index]?.episodeId, + aliases: assigned[index]?.aliases, + source: episodeSource, + })); +} + +describe("canonical identity IDs", () => { + test("uses a stable, fixed-length SHA-256 base64url grammar", () => { + expect(feedId).toBe("pnf1_1FkzYRu7QhyXP_UmFwiNli7Pz-Fvl0Mun3-mvl4k2_8"); + expect(feedId).toMatch(/^pnf1_[A-Za-z0-9_-]{43}$/); + expect(isCanonicalFeedId(feedId)).toBe(true); + expect(getEpisodeIdentityCandidates(source())[0]?.id).toMatch(/^pne1_[A-Za-z0-9_-]{43}$/); + }); + + test.each([ + undefined, + null, + {}, + "pnf1_", + `pnf1_${"A".repeat(42)}`, + `pnf1_${"A".repeat(44)}`, + `pnf1_${"A".repeat(42)}=`, + `pnf1_${"A".repeat(42)}\n`, + `__proto__`, + ])("rejects malformed feed IDs without throwing: %j", (candidate) => { + expect(() => isCanonicalFeedId(candidate)).not.toThrow(); + expect(isCanonicalFeedId(candidate)).toBe(false); + }); + + test.each([ + undefined, + null, + {}, + "pne1_", + `pne1_${"A".repeat(42)}`, + `pne1_${"A".repeat(44)}`, + `pne1_${"A".repeat(42)}=`, + `pne1_${"A".repeat(42)}\u0000`, + ])("rejects malformed episode IDs without throwing: %j", (candidate) => { + expect(() => isCanonicalEpisodeId(candidate)).not.toThrow(); + expect(isCanonicalEpisodeId(candidate)).toBe(false); + }); + + test("keeps feed and episode domains disjoint", () => { + const episodeId = getEpisodeIdentityCandidates(source())[0]?.id; + expect(isCanonicalEpisodeId(episodeId)).toBe(true); + expect(isCanonicalFeedId(episodeId)).toBe(false); + expect(isCanonicalEpisodeId(feedId)).toBe(false); + }); + + test("bounds untrusted inputs and handles lone surrogates deterministically", () => { + const oversized = `https://example.com/${"a".repeat(MAX_IDENTITY_COMPONENT_BYTES)}`; + expect(createFeedId(oversized)).toBeUndefined(); + + const loneSurrogate = createFeedId("https://example.com/\ud800"); + const replacementCharacter = createFeedId("https://example.com/�"); + expect(loneSurrogate).toBeUndefined(); + expect(replacementCharacter).toBeDefined(); + expect(() => createFeedId("https://example.com/\ud800")).not.toThrow(); + expect(createFeedId("https://example.com/feed\n.xml")).toBeUndefined(); + }); + + test("rejects invalid feed URLs and ignores fragments in feed identity", () => { + expect(createFeedId("not a URL")).toBeUndefined(); + expect(createFeedId("https://example.com/feed.xml#first")).toBe( + createFeedId("https://example.com/feed.xml#second"), + ); + }); + + test.each([{}, [], 1, Symbol("url"), 1n])( + "rejects non-string URL sources without throwing", + (candidate) => { + expect(() => createFeedId(candidate)).not.toThrow(); + expect(createFeedId(candidate)).toBeUndefined(); + expect(normalizeStrongIdentityUrl(candidate)).toBeUndefined(); + }, + ); +}); + +describe("strong URL and path identity", () => { + test("removes only the URL fragment", () => { + expect( + normalizeStrongIdentityUrl( + "HTTPS://Example.COM:443/a/../episode.mp3?b=2&a=1&token=secret#chapter", + ), + ).toBe("https://example.com/episode.mp3?b=2&a=1&token=secret"); + }); + + test("preserves query order and authentication parameters", () => { + const first = createFeedId("https://example.com/feed?b=2&a=1&token=first"); + const reordered = createFeedId("https://example.com/feed?a=1&b=2&token=first"); + const rotated = createFeedId("https://example.com/feed?b=2&a=1&token=second"); + expect(first).not.toBe(reordered); + expect(first).not.toBe(rotated); + }); + + test("does not trim or reduce caller-normalized local vault paths", () => { + const nested = createLocalEpisodeId(feedId, "shows/a/episode.mp3"); + const otherFolder = createLocalEpisodeId(feedId, "shows/b/episode.mp3"); + const leadingSpace = createLocalEpisodeId(feedId, " shows/a/episode.mp3"); + expect(nested).not.toBe(otherFolder); + expect(nested).not.toBe(leadingSpace); + }); + + test("rejects invalid feed IDs and impossible local paths at runtime", () => { + expect(createLocalEpisodeId("not-a-feed-id", "shows/episode.mp3")).toBeUndefined(); + expect(createLocalEpisodeId(feedId, "")).toBeUndefined(); + expect(createLocalEpisodeId(feedId, "shows/\u0000episode.mp3")).toBeUndefined(); + expect(createLocalEpisodeId(feedId, { path: "shows/episode.mp3" })).toBeUndefined(); + expect(createLocalEpisodeId(Symbol("feed"), "shows/episode.mp3")).toBeUndefined(); + }); +}); + +describe("whole-feed episode identity assignment", () => { + test("scopes the same item GUID to its feed", () => { + const otherFeedId = createFeedId("https://other.example/feed.xml")!; + const first = assignEpisodeIdentities([source()])[0]?.episodeId; + const second = assignEpisodeIdentities([source({ feedId: otherFeedId })])[0]?.episodeId; + expect(first).not.toBe(second); + }); + + test("keeps a stable GUID identity across title and enclosure changes", () => { + const before = assignEpisodeIdentities([source()])[0]?.episodeId; + const after = assignEpisodeIdentities([ + source({ + title: "Renamed episode", + enclosureUrl: "https://cdn.example.com/replaced.mp3", + itemLink: "https://example.com/episodes/renamed", + }), + ])[0]?.episodeId; + expect(before).toBe(after); + }); + + test("does not equate same-titled episodes when every strong locator changes", () => { + const before = assignEpisodeIdentities([source()])[0]?.episodeId; + const after = assignEpisodeIdentities([ + source({ + guid: "other-guid", + enclosureUrl: "https://cdn.example.com/other.mp3", + itemLink: "https://example.com/episodes/other", + }), + ])[0]?.episodeId; + expect(before).not.toBe(after); + }); + + test("handles control characters and lone surrogates in opaque GUIDs safely", () => { + const control = source({ guid: "guid\u0000value" }); + const escaped = source({ guid: "guid\\u0000value" }); + const surrogate = source({ guid: "guid\ud800value" }); + expect(() => assignEpisodeIdentities([control, escaped, surrogate])).not.toThrow(); + const ids = assignEpisodeIdentities([control, escaped, surrogate]).map( + (identity) => identity.episodeId, + ); + expect(new Set(ids).size).toBe(3); + }); + + test("returns no identity without a valid canonical feed ID", () => { + expect(assignEpisodeIdentities([source({ feedId: "" })])[0]?.episodeId).toBeUndefined(); + expect( + assignEpisodeIdentities([source({ feedId: "pnf1_forged" })])[0]?.episodeId, + ).toBeUndefined(); + }); + + test("rejects non-record sources and invalid component types without throwing", () => { + const throwingGetter = Object.defineProperty({}, "feedId", { + get() { + throw new Error("untrusted getter"); + }, + }); + const invalid = [ + null, + [], + 1, + Symbol("source"), + throwingGetter, + { ...source(), title: 1 }, + { ...source(), enclosureUrl: {} }, + { ...source(), guid: [] }, + { ...source(), itemLink: 1n }, + ]; + for (const candidate of invalid) { + expect(() => getEpisodeIdentityCandidates(candidate)).not.toThrow(); + expect(getEpisodeIdentityCandidates(candidate)).toEqual([]); + } + expect(assignEpisodeIdentities("not an array")).toEqual([]); + expect(assignEpisodeIdentities(invalid)).toHaveLength(invalid.length); + }); + + test("excludes a duplicated GUID and falls through to unique enclosures", () => { + const sources = [ + source({ enclosureUrl: "https://cdn.example.com/one.mp3", itemLink: undefined }), + source({ + title: "Episode 2", + enclosureUrl: "https://cdn.example.com/two.mp3", + itemLink: undefined, + publishedAt: "2024-01-02T00:00:00.000Z", + }), + ]; + const assigned = assignEpisodeIdentities(sources); + const candidates = sources.map(getEpisodeIdentityCandidates); + + expect(assigned[0]?.episodeId).toBe( + candidates[0].find((candidate) => candidate.kind === "media")?.id, + ); + expect(assigned[1]?.episodeId).toBe( + candidates[1].find((candidate) => candidate.kind === "media")?.id, + ); + const duplicatedGuid = candidates[0].find((candidate) => candidate.kind === "guid")?.id; + expect(assigned[0]?.aliases).not.toContain(duplicatedGuid); + expect(assigned[1]?.aliases).not.toContain(duplicatedGuid); + }); + + test("excludes duplicated GUID and enclosure candidates before using item links", () => { + const shared = { + guid: "duplicate-guid", + enclosureUrl: "https://cdn.example.com/shared.mp3", + }; + const sources = [ + source({ ...shared, itemLink: "https://example.com/one" }), + source({ + ...shared, + title: "Episode 2", + itemLink: "https://example.com/two", + publishedAt: "2024-01-02T00:00:00.000Z", + }), + ]; + const assigned = assignEpisodeIdentities(sources); + const candidates = sources.map(getEpisodeIdentityCandidates); + expect(assigned[0]?.episodeId).toBe( + candidates[0].find((candidate) => candidate.kind === "itemLink")?.id, + ); + expect(assigned[1]?.episodeId).toBe( + candidates[1].find((candidate) => candidate.kind === "itemLink")?.id, + ); + }); + + test("uses distinct source tuples when every strong locator is duplicated", () => { + const shared = { + guid: "duplicate-guid", + enclosureUrl: "https://cdn.example.com/shared.mp3", + itemLink: "https://example.com/shared", + }; + const sources = [ + source({ ...shared }), + source({ + ...shared, + title: "Episode 2", + publishedAt: "2024-01-02T00:00:00.000Z", + }), + ]; + const assigned = assignEpisodeIdentities(sources); + expect(assigned[0]?.episodeId).toBe( + getEpisodeIdentityCandidates(sources[0]).find( + (candidate) => candidate.kind === "source", + )?.id, + ); + expect(assigned[1]?.episodeId).toBe( + getEpisodeIdentityCandidates(sources[1]).find( + (candidate) => candidate.kind === "source", + )?.id, + ); + expect(assigned[0]?.episodeId).not.toBe(assigned[1]?.episodeId); + }); + + test("treats exact repeated source tuples as one logical identity", () => { + const episode = source(); + const assigned = assignEpisodeIdentities([episode, { ...episode }]); + expect(assigned[0]?.episodeId).toBe(assigned[1]?.episodeId); + }); + + test("never creates an item-link alias from the feed URL fallback", () => { + const candidates = getEpisodeIdentityCandidates(source({ itemLink: undefined })); + expect(candidates.some((candidate) => candidate.kind === "itemLink")).toBe(false); + }); + + test("fails closed when an oversized tuple shares a strong alias", () => { + const oversizedTitle = "x".repeat(MAX_IDENTITY_COMPONENT_BYTES); + const assigned = assignEpisodeIdentities([ + source({ title: oversizedTitle, enclosureUrl: "not a URL", itemLink: undefined }), + source({ title: `${oversizedTitle}y`, enclosureUrl: "not a URL", itemLink: undefined }), + ]); + expect(assigned[0]?.episodeId).toBeUndefined(); + expect(assigned[1]?.episodeId).toBeUndefined(); + }); +}); + +describe("whole-feed reconciliation", () => { + test("retains an ID when a GUID appears and later changes through stable media evidence", () => { + const initialSource = source({ guid: undefined }); + const [initial] = previousEntries([initialSource]); + const withGuid = source({ guid: "new-guid" }); + const [firstRefresh] = reconcileEpisodeIdentities([initial], [withGuid]); + expect(firstRefresh.episodeId).toBe(initial.episodeId); + expect(firstRefresh.aliases).toContain(initial.episodeId); + + const changedGuid = source({ guid: "changed-guid" }); + const [secondRefresh] = reconcileEpisodeIdentities( + [{ episodeId: firstRefresh.episodeId, source: withGuid }], + [changedGuid], + ); + expect(secondRefresh.episodeId).toBe(initial.episodeId); + }); + + test("rejects a valid substituted prior ID when aliases do not contain it", () => { + const episodeSource = source(); + const [previous] = persistedEntries([episodeSource]); + const forgedId = assignEpisodeIdentities([source({ guid: "forged-id-source" })])[0] + ?.episodeId; + const [current] = reconcileEpisodeIdentities( + [{ ...previous, episodeId: forgedId }], + [episodeSource], + ); + + expect(forgedId).toBeDefined(); + expect(previous.aliases).not.toContain(forgedId); + expect(current.episodeId).toBe(previous.episodeId); + expect(current.episodeId).not.toBe(forgedId); + }); + + test("does not resurrect a duplicate GUID from a truncated prior observation", () => { + const previousSources = [ + source({ + guid: "duplicated-guid", + enclosureUrl: "https://cdn.example.com/one.mp3", + itemLink: undefined, + }), + source({ + guid: "duplicated-guid", + title: "Episode 2", + enclosureUrl: "https://cdn.example.com/two.mp3", + itemLink: undefined, + publishedAt: "2024-01-02T00:00:00.000Z", + }), + ]; + const [firstPersisted] = persistedEntries(previousSources); + const duplicateGuidId = getEpisodeIdentityCandidates(previousSources[0]).find( + (candidate) => candidate.kind === "guid", + )?.id; + const changedSource = { + ...previousSources[0], + enclosureUrl: "https://cdn.example.com/replaced.mp3", + }; + const [current] = reconcileEpisodeIdentities([firstPersisted], [changedSource]); + + expect(firstPersisted.aliases).not.toContain(duplicateGuidId); + expect(current.episodeId).not.toBe(firstPersisted.episodeId); + }); + + test("retains the original ID across an A to B bridge and A locator reversion", () => { + const sourceA = source({ + guid: "guid-a", + enclosureUrl: "https://cdn.example.com/bridge.mp3", + itemLink: "https://example.com/a", + }); + const [persistedA] = persistedEntries([sourceA]); + const sourceB = source({ + guid: "guid-b", + enclosureUrl: "https://cdn.example.com/bridge.mp3", + itemLink: "https://example.com/b", + }); + const [observedB] = reconcileEpisodeIdentities([persistedA], [sourceB]); + expect(observedB.episodeId).toBe(persistedA.episodeId); + + const sourceAReverted = source({ + guid: "guid-a", + enclosureUrl: "https://cdn.example.com/reverted.mp3", + itemLink: "https://example.com/a-reverted", + }); + const [reverted] = reconcileEpisodeIdentities( + [{ episodeId: observedB.episodeId, aliases: observedB.aliases, source: sourceB }], + [sourceAReverted], + ); + expect(reverted.episodeId).toBe(persistedA.episodeId); + expect(reverted.aliases.length).toBeLessThanOrEqual(MAX_EPISODE_IDENTITY_ALIASES); + }); + + test("retains IDs when a formerly unique GUID becomes ambiguous", () => { + const previousSources = [ + source({ guid: "guid-one", enclosureUrl: "https://cdn.example.com/one.mp3" }), + source({ + guid: "guid-two", + title: "Episode 2", + enclosureUrl: "https://cdn.example.com/two.mp3", + itemLink: "https://example.com/episodes/2", + publishedAt: "2024-01-02T00:00:00.000Z", + }), + ]; + const previous = previousEntries(previousSources); + const currentSources = previousSources.map((episodeSource) => ({ + ...episodeSource, + guid: "now-duplicated", + })); + const current = reconcileEpisodeIdentities(previous, currentSources); + expect(current.map((identity) => identity.episodeId)).toEqual( + previous.map((identity) => identity.episodeId), + ); + }); + + test("retains IDs when a previously ambiguous GUID becomes unique", () => { + const previousSources = [ + source({ guid: "duplicated", enclosureUrl: "https://cdn.example.com/one.mp3" }), + source({ + guid: "duplicated", + title: "Episode 2", + enclosureUrl: "https://cdn.example.com/two.mp3", + itemLink: "https://example.com/episodes/2", + publishedAt: "2024-01-02T00:00:00.000Z", + }), + ]; + const previous = previousEntries(previousSources); + const currentSources = previousSources.map((episodeSource, index) => ({ + ...episodeSource, + guid: `now-unique-${index}`, + })); + const current = reconcileEpisodeIdentities(previous, currentSources); + expect(current.map((identity) => identity.episodeId)).toEqual( + previous.map((identity) => identity.episodeId), + ); + }); + + test("refuses to retain an ID when prior evidence splits across current items", () => { + const previousSource = source(); + const assignedPrevious = assignEpisodeIdentities([previousSource])[0]; + const historicalId = assignEpisodeIdentities([source({ guid: "historical-id-source" })])[0] + ?.episodeId; + if (!historicalId) throw new Error("Expected a historical canonical ID"); + const previous = { + episodeId: historicalId, + aliases: [historicalId, ...assignedPrevious.aliases], + source: previousSource, + }; + const currentSources = [ + { + ...previousSource, + title: "GUID branch", + enclosureUrl: "https://cdn.example.com/guid-branch.mp3", + itemLink: "https://example.com/guid-branch", + publishedAt: "2024-01-02T00:00:00.000Z", + }, + { + ...previousSource, + guid: "media-branch-guid", + title: "Media branch", + publishedAt: "2024-01-03T00:00:00.000Z", + }, + ]; + const current = reconcileEpisodeIdentities([previous], currentSources); + expect(current.every((identity) => identity.episodeId !== historicalId)).toBe(true); + }); + + test("refuses to retain either ID when current evidence merges two prior items", () => { + const previousSources = [ + source({ + guid: "guid-one", + enclosureUrl: "https://cdn.example.com/one.mp3", + itemLink: "https://example.com/one", + }), + source({ + guid: "guid-two", + title: "Episode 2", + enclosureUrl: "https://cdn.example.com/two.mp3", + itemLink: "https://example.com/two", + publishedAt: "2024-01-02T00:00:00.000Z", + }), + ]; + const previous = previousEntries(previousSources); + const merged = source({ + guid: "brand-new-guid", + enclosureUrl: previousSources[0].enclosureUrl, + itemLink: previousSources[1].itemLink, + }); + const [current] = reconcileEpisodeIdentities(previous, [merged]); + expect(previous.map((identity) => identity.episodeId)).not.toContain(current.episodeId); + }); + + test("never bridges observations through a source-tuple fallback", () => { + const previousSource = source({ + guid: undefined, + enclosureUrl: "invalid enclosure one", + itemLink: undefined, + }); + const [previous] = previousEntries([previousSource]); + const currentSource = { + ...previousSource, + enclosureUrl: "invalid enclosure two", + }; + const [current] = reconcileEpisodeIdentities([previous], [currentSource]); + + expect(previous.episodeId).toBeDefined(); + expect(current.aliases).toEqual([]); + expect(current.episodeId).not.toBe(previous.episodeId); + }); + + test("ignores malformed or duplicated prior canonical IDs", () => { + const episodeSource = source(); + const currentOnlyId = assignEpisodeIdentities([episodeSource])[0]?.episodeId; + expect( + reconcileEpisodeIdentities( + [{ episodeId: "forged", source: episodeSource }], + [episodeSource], + )[0]?.episodeId, + ).toBe(currentOnlyId); + + const priorSource = source({ guid: "prior-guid" }); + const validId = assignEpisodeIdentities([priorSource])[0]?.episodeId; + if (!validId) throw new Error("Expected a canonical prior episode ID"); + const duplicatePrevious = [ + { episodeId: validId, source: priorSource }, + { + episodeId: validId, + source: source({ + guid: "unrelated-guid", + enclosureUrl: "https://unrelated.example/episode.mp3", + itemLink: "https://unrelated.example/episode", + }), + }, + ]; + const changedGuid = source({ guid: "changed-guid" }); + expect(reconcileEpisodeIdentities(duplicatePrevious, [changedGuid])[0]?.episodeId).not.toBe( + validId, + ); + + const oversizedAliases = Array.from( + { length: MAX_EPISODE_IDENTITY_ALIASES + 1 }, + (_, index) => + assignEpisodeIdentities([source({ guid: `oversized-alias-${index}` })])[0] + ?.episodeId, + ); + const oversizedPrior = { + episodeId: oversizedAliases[0], + aliases: oversizedAliases, + source: priorSource, + }; + expect(reconcileEpisodeIdentities([oversizedPrior], [changedGuid])[0]?.episodeId).not.toBe( + oversizedPrior.episodeId, + ); + + const sparseSource = source({ guid: "sparse-old-guid" }); + const [sparsePersisted] = persistedEntries([sparseSource]); + const mediaAlias = getEpisodeIdentityCandidates(sparseSource).find( + (candidate) => candidate.kind === "media", + )?.id; + const sparseAliases: unknown[] = []; + sparseAliases.length = 3; + sparseAliases[0] = sparsePersisted.episodeId; + sparseAliases[2] = mediaAlias; + const sparseCurrent = source({ + guid: "sparse-new-guid", + itemLink: "https://example.com/sparse-new-link", + }); + expect( + reconcileEpisodeIdentities( + [{ ...sparsePersisted, aliases: sparseAliases }], + [sparseCurrent], + )[0]?.episodeId, + ).not.toBe(sparsePersisted.episodeId); + + const hostileAliases = [sparsePersisted.episodeId, mediaAlias]; + Object.defineProperty(hostileAliases, 1, { + get() { + throw new Error("untrusted alias getter"); + }, + }); + let hostileResult = reconcileEpisodeIdentities([], []); + expect(() => { + hostileResult = reconcileEpisodeIdentities( + [{ ...sparsePersisted, aliases: hostileAliases }], + [sparseCurrent], + ); + }).not.toThrow(); + expect(hostileResult[0]?.episodeId).not.toBe(sparsePersisted.episodeId); + }); +}); diff --git a/src/utility/identity.ts b/src/utility/identity.ts new file mode 100644 index 00000000..dbe4dad6 --- /dev/null +++ b/src/utility/identity.ts @@ -0,0 +1,450 @@ +import { sha256 } from "@noble/hashes/sha2.js"; +import { utf8ToBytes } from "@noble/hashes/utils.js"; + +const FEED_ID_PREFIX = "pnf1_"; +const EPISODE_ID_PREFIX = "pne1_"; +const SHA256_BASE64URL_LENGTH = 43; +const CANONICAL_ID_LENGTH = FEED_ID_PREFIX.length + SHA256_BASE64URL_LENGTH; +const CANONICAL_ID_BODY = /^[A-Za-z0-9_-]{43}$/; + +/** + * Identity inputs come from untrusted feeds and imports. The digest is fixed + * length, but bounding its preimage also prevents oversized input from causing + * avoidable allocation or hashing work. + */ +export const MAX_IDENTITY_COMPONENT_BYTES = 16 * 1024; +export const MAX_IDENTITY_PREIMAGE_BYTES = 64 * 1024; +export const MAX_EPISODE_IDENTITY_ALIASES = 32; + +const IDENTITY_NAMESPACE = "com.chhoumann.podnotes.identity"; +const IDENTITY_VERSION = 1; +const BASE64URL_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +export type CanonicalFeedId = string & { readonly __feedId: unique symbol }; +export type CanonicalEpisodeId = string & { readonly __episodeId: unique symbol }; + +type IdentityDomain = + | "feed-url" + | "episode-guid" + | "episode-media" + | "episode-item-link" + | "episode-source-tuple" + | "local-vault-path"; + +export type EpisodeIdentityCandidateKind = "guid" | "media" | "itemLink" | "source"; + +export interface EpisodeIdentitySource { + feedId: string; + guid?: string; + enclosureUrl: string; + itemLink?: string; + publishedAt?: string; + title: string; +} + +export interface EpisodeIdentityCandidate { + kind: EpisodeIdentityCandidateKind; + id: CanonicalEpisodeId; +} + +export interface AssignedEpisodeIdentity { + episodeId?: CanonicalEpisodeId; + /** Current strong locators plus bounded, trusted history after reconciliation. */ + aliases: readonly CanonicalEpisodeId[]; +} + +export interface PreviousEpisodeIdentity { + episodeId: unknown; + source: unknown; + aliases?: unknown; +} + +function encodeBase64Url(bytes: Uint8Array): string { + let encoded = ""; + for (let index = 0; index < bytes.length; index += 3) { + const first = bytes[index] ?? 0; + const second = bytes[index + 1]; + const third = bytes[index + 2]; + const block = (first << 16) | ((second ?? 0) << 8) | (third ?? 0); + + encoded += BASE64URL_ALPHABET[(block >>> 18) & 63]; + encoded += BASE64URL_ALPHABET[(block >>> 12) & 63]; + if (second !== undefined) encoded += BASE64URL_ALPHABET[(block >>> 6) & 63]; + if (third !== undefined) encoded += BASE64URL_ALPHABET[block & 63]; + } + return encoded; +} + +function componentIsBounded(value: unknown): value is string { + if (typeof value !== "string") return false; + if (value.length > MAX_IDENTITY_COMPONENT_BYTES) return false; + return utf8ToBytes(JSON.stringify(value)).length <= MAX_IDENTITY_COMPONENT_BYTES; +} + +function hasLoneSurrogate(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +} + +function hasAsciiControl(value: string): boolean { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code <= 0x1f || code === 0x7f) return true; + } + return false; +} + +function createCanonicalId( + prefix: typeof FEED_ID_PREFIX | typeof EPISODE_ID_PREFIX, + domain: IdentityDomain, + parts: readonly string[], +): string | undefined { + if (!parts.every(componentIsBounded)) return undefined; + + // Arrays give a canonical, length-delimited representation. JSON.stringify + // also escapes control characters and lone surrogates before UTF-8 encoding. + const preimage = utf8ToBytes( + JSON.stringify([IDENTITY_NAMESPACE, IDENTITY_VERSION, domain, ...parts]), + ); + if (preimage.length > MAX_IDENTITY_PREIMAGE_BYTES) return undefined; + + return `${prefix}${encodeBase64Url(sha256(preimage))}`; +} + +/** + * Apply WHATWG URL canonicalization and remove only the fragment. Query order, + * credentials, and every query parameter remain part of the strong identity. + */ +export function normalizeStrongIdentityUrl(value: unknown): string | undefined { + // WHATWG URL parsing replaces lone surrogates and discards raw ASCII control + // characters. Reject them first so distinct hostile inputs cannot normalize to + // the same otherwise-strong locator. + if ( + typeof value !== "string" || + !componentIsBounded(value) || + hasLoneSurrogate(value) || + hasAsciiControl(value) + ) { + return undefined; + } + try { + const url = new URL(value); + url.hash = ""; + const normalized = url.href; + return componentIsBounded(normalized) ? normalized : undefined; + } catch { + return undefined; + } +} + +export function isCanonicalFeedId(value: unknown): value is CanonicalFeedId { + return ( + typeof value === "string" && + value.length === CANONICAL_ID_LENGTH && + value.startsWith(FEED_ID_PREFIX) && + CANONICAL_ID_BODY.test(value.slice(FEED_ID_PREFIX.length)) + ); +} + +export function isCanonicalEpisodeId(value: unknown): value is CanonicalEpisodeId { + return ( + typeof value === "string" && + value.length === CANONICAL_ID_LENGTH && + value.startsWith(EPISODE_ID_PREFIX) && + CANONICAL_ID_BODY.test(value.slice(EPISODE_ID_PREFIX.length)) + ); +} + +/** Derive the immutable first-observation feed ID from its subscription URL. */ +export function createFeedId(subscriptionUrl: unknown): CanonicalFeedId | undefined { + const normalizedUrl = normalizeStrongIdentityUrl(subscriptionUrl); + if (!normalizedUrl) return undefined; + return createCanonicalId(FEED_ID_PREFIX, "feed-url", [normalizedUrl]) as + | CanonicalFeedId + | undefined; +} + +/** + * Deterministic migration helper for local files. The caller must provide an + * Obsidian-normalized vault path. This function deliberately does not trim or + * reduce the path to a basename. + */ +export function createLocalEpisodeId( + feedId: unknown, + normalizedVaultPath: unknown, +): CanonicalEpisodeId | undefined { + if ( + !isCanonicalFeedId(feedId) || + typeof normalizedVaultPath !== "string" || + !normalizedVaultPath || + normalizedVaultPath.includes("\u0000") + ) { + return undefined; + } + return createCanonicalId(EPISODE_ID_PREFIX, "local-vault-path", [ + feedId, + normalizedVaultPath, + ]) as CanonicalEpisodeId | undefined; +} + +function normalizeOpaqueGuid(value: unknown): string | undefined { + const guid = typeof value === "string" ? value : undefined; + return guid && componentIsBounded(guid) ? guid : undefined; +} + +function createEpisodeCandidate( + feedId: CanonicalFeedId, + kind: Exclude, + value: string, +): EpisodeIdentityCandidate | undefined { + const domain: IdentityDomain = + kind === "guid" ? "episode-guid" : kind === "media" ? "episode-media" : "episode-item-link"; + const id = createCanonicalId(EPISODE_ID_PREFIX, domain, [feedId, value]); + return id ? { kind, id: id as CanonicalEpisodeId } : undefined; +} + +interface NormalizedEpisodeIdentitySource { + feedId?: CanonicalFeedId; + guid?: string; + media?: string; + itemLink?: string; + publishedAt: string; + title: string; + rawMedia: string; + rawItemLink: string; +} + +function createSourceTupleCandidate( + feedId: CanonicalFeedId, + source: NormalizedEpisodeIdentitySource, +): EpisodeIdentityCandidate | undefined { + const id = createCanonicalId(EPISODE_ID_PREFIX, "episode-source-tuple", [ + feedId, + source.guid ?? "", + source.media ?? source.rawMedia, + source.itemLink ?? source.rawItemLink, + source.publishedAt, + source.title, + ]); + return id ? { kind: "source", id: id as CanonicalEpisodeId } : undefined; +} + +function normalizedIdentitySource(source: unknown): NormalizedEpisodeIdentitySource | undefined { + if (!source || typeof source !== "object" || Array.isArray(source)) return undefined; + + try { + const candidate = source as Record; + const feedId = candidate.feedId; + const enclosureUrl = candidate.enclosureUrl; + const title = candidate.title; + const guid = candidate.guid; + const itemLink = candidate.itemLink; + const publishedAt = candidate.publishedAt; + if ( + typeof feedId !== "string" || + typeof enclosureUrl !== "string" || + typeof title !== "string" || + (guid !== undefined && typeof guid !== "string") || + (itemLink !== undefined && typeof itemLink !== "string") || + (publishedAt !== undefined && typeof publishedAt !== "string") + ) { + return undefined; + } + + return { + feedId: isCanonicalFeedId(feedId) ? feedId : undefined, + guid: normalizeOpaqueGuid(guid), + media: normalizeStrongIdentityUrl(enclosureUrl), + itemLink: itemLink ? normalizeStrongIdentityUrl(itemLink) : undefined, + publishedAt: publishedAt ?? "", + title, + rawMedia: enclosureUrl, + rawItemLink: itemLink ?? "", + }; + } catch { + return undefined; + } +} + +export function getEpisodeIdentityCandidates(source: unknown): readonly EpisodeIdentityCandidate[] { + const normalized = normalizedIdentitySource(source); + if (!normalized?.feedId) return []; + + const candidates: EpisodeIdentityCandidate[] = []; + if (normalized.guid) { + const candidate = createEpisodeCandidate(normalized.feedId, "guid", normalized.guid); + if (candidate) candidates.push(candidate); + } + if (normalized.media) { + const candidate = createEpisodeCandidate(normalized.feedId, "media", normalized.media); + if (candidate) candidates.push(candidate); + } + if (normalized.itemLink) { + const candidate = createEpisodeCandidate( + normalized.feedId, + "itemLink", + normalized.itemLink, + ); + if (candidate) candidates.push(candidate); + } + + const fallback = createSourceTupleCandidate(normalized.feedId, normalized); + if (fallback) candidates.push(fallback); + + return candidates; +} + +/** + * Choose identities only after examining the complete feed. GUID, enclosure, + * and item-link candidates shared by distinct source tuples are excluded. An + * exact repeated tuple is treated as the same logical source and may share an + * ID; a distinct tuple falls through to its own source-derived ID. + */ +export function assignEpisodeIdentities(sources: unknown): readonly AssignedEpisodeIdentity[] { + if (!Array.isArray(sources)) return []; + const candidatesBySource = Array.from(sources, getEpisodeIdentityCandidates); + const sourceTupleIds = candidatesBySource.map( + (candidates) => candidates.find((candidate) => candidate.kind === "source")?.id, + ); + const distinctSourcesByCandidate = new Map>(); + + for (let index = 0; index < candidatesBySource.length; index++) { + // Oversized/malformed tuples fail closed: they never make a candidate look + // unique merely because two unidentifiable records happen to share an alias. + const sourceKey = sourceTupleIds[index] ?? `unidentified:${index}`; + for (const candidate of candidatesBySource[index]) { + const sourceKeys = distinctSourcesByCandidate.get(candidate.id) ?? new Set(); + sourceKeys.add(sourceKey); + distinctSourcesByCandidate.set(candidate.id, sourceKeys); + } + } + + return candidatesBySource.map((candidates) => { + const uniqueCandidates = candidates.filter( + (candidate) => distinctSourcesByCandidate.get(candidate.id)?.size === 1, + ); + return { + episodeId: uniqueCandidates[0]?.id, + aliases: uniqueCandidates + .filter((candidate) => candidate.kind !== "source") + .map((candidate) => candidate.id), + }; + }); +} + +function validateExplicitPriorAliases( + value: unknown, + previousId: CanonicalEpisodeId, +): Set { + try { + if (!Array.isArray(value) || value.length > MAX_EPISODE_IDENTITY_ALIASES) { + return new Set(); + } + const validated: CanonicalEpisodeId[] = []; + for (let index = 0; index < value.length; index++) { + const alias = value[index]; + if (!isCanonicalEpisodeId(alias)) return new Set(); + validated.push(alias); + } + const aliases = new Set(validated); + if (aliases.size !== validated.length || !aliases.has(previousId)) { + return new Set(); + } + return aliases; + } catch { + return new Set(); + } +} + +/** + * Reconcile two complete feed observations without display-title fallback. + * Prior IDs survive only across a one-to-one edge between independently + * ambiguity-vetted candidate sets. Splits, merges, malformed prior IDs, and + * duplicate prior IDs all fail closed to the current source-derived identity. + */ +export function reconcileEpisodeIdentities( + previous: unknown, + currentSources: unknown, +): readonly AssignedEpisodeIdentity[] { + const current = assignEpisodeIdentities(currentSources); + if (!Array.isArray(previous) || !Array.isArray(currentSources)) return current; + + const previousRecords = previous.map((entry) => { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) return undefined; + try { + const record = entry as Record; + return { + episodeId: record.episodeId, + source: record.source, + aliases: Object.prototype.hasOwnProperty.call(record, "aliases") + ? record.aliases + : undefined, + }; + } catch { + return undefined; + } + }); + const previousSources = previousRecords.map((record) => record?.source); + const previousAssigned = assignEpisodeIdentities(previousSources); + const previousIds = previousRecords.map((record) => + isCanonicalEpisodeId(record?.episodeId) ? record.episodeId : undefined, + ); + const priorIdCounts = new Map(); + for (const id of previousIds) { + if (id) priorIdCounts.set(id, (priorIdCounts.get(id) ?? 0) + 1); + } + const trustedPreviousAliases = previousRecords.map((record, index) => { + const previousId = previousIds[index]; + if (!record || !previousId) return new Set(); + + if (record.aliases !== undefined) { + return validateExplicitPriorAliases(record.aliases, previousId); + } + + const recomputed = previousAssigned[index]; + if (recomputed?.episodeId !== previousId && !recomputed?.aliases.includes(previousId)) { + return new Set(); + } + return new Set(recomputed.aliases); + }); + + const currentEdges = current.map(() => new Set()); + const previousEdges = previousAssigned.map(() => new Set()); + for (let previousIndex = 0; previousIndex < previousAssigned.length; previousIndex++) { + const previousAliases = trustedPreviousAliases[previousIndex]; + if (previousAliases.size === 0) continue; + for (let currentIndex = 0; currentIndex < current.length; currentIndex++) { + if (current[currentIndex]?.aliases.some((alias) => previousAliases.has(alias))) { + previousEdges[previousIndex]?.add(currentIndex); + currentEdges[currentIndex]?.add(previousIndex); + } + } + } + + return current.map((identity, currentIndex) => { + const priorIndexes = currentEdges[currentIndex]; + if (priorIndexes?.size !== 1) return identity; + const [previousIndex] = priorIndexes; + if (previousEdges[previousIndex]?.size !== 1) return identity; + const previousId = previousIds[previousIndex]; + if (!previousId || priorIdCounts.get(previousId) !== 1) return identity; + const aliases = new Set(); + aliases.add(previousId); + for (const alias of identity.aliases) aliases.add(alias); + for (const alias of trustedPreviousAliases[previousIndex]) aliases.add(alias); + + return { + episodeId: previousId, + aliases: [...aliases].slice(0, MAX_EPISODE_IDENTITY_ALIASES), + }; + }); +} diff --git a/tests/e2e/podnotes-runtime.test.ts b/tests/e2e/podnotes-runtime.test.ts index a9eb0b72..8f697e3b 100644 --- a/tests/e2e/podnotes-runtime.test.ts +++ b/tests/e2e/podnotes-runtime.test.ts @@ -16,7 +16,12 @@ import { waitForPodNotesReady, } from "./harness"; -type PodNotesData = Partial; +type PodNotesData = Partial & { + schemaVersion?: number; + openAIApiKey?: string; + diarizationApiKey?: string; + legacyExtension?: { enabled: boolean }; +}; type PlaybackState = { currentTime: number | null; @@ -416,6 +421,153 @@ describe("PodNotes runtime", () => { expect(data.defaultVolume).toBe(1); expect(runtimeVolume).toBe(1); }); + + test("migrates legacy dates and credentials before runtime hydration", async () => { + const { obsidian, plugin, sandbox } = getContext(); + const audioPath = await seedAudio(sandbox, "legacy-date-episode.mp3"); + const isoDate = "2024-03-01T10:05:03.000Z"; + const episode = { + ...createLocalEpisode("E2E Legacy Date Episode", audioPath), + episodeDate: isoDate as unknown as Date, + }; + await evalJsonAsync( + obsidian, + `(() => { + app.secretStorage.setSecret("podnotes-openai-api-key", "unrelated-openai-sentinel"); + app.secretStorage.setSecret("podnotes-deepgram-api-key", "unrelated-deepgram-sentinel"); + return true; + })()`, + ); + + await seedRuntimeData(plugin, sandbox, episode, { + currentEpisode: episode, + legacyData: true, + legacySecrets: { openAI: "e2e-openai-secret", deepgram: "e2e-deepgram-secret" }, + note: { + path: sandbox.path("legacy-date-note.md"), + template: "date: {{date:YYYY-MM-DD}}\n", + }, + }); + await waitForPodNotesReady(obsidian); + + const beforeSave = await plugin.data().read(); + expect(beforeSave.schemaVersion).toBe(2); + expect(beforeSave.openAIApiKey).toBeUndefined(); + expect(beforeSave.diarizationApiKey).toBeUndefined(); + expect(beforeSave.openAISecretId).toMatch(/^podnotes-openai-api-key(?:-\d+)?$/); + expect(beforeSave.deepgramSecretId).toMatch(/^podnotes-deepgram-api-key(?:-\d+)?$/); + expect(beforeSave.openAISecretId).not.toBe("podnotes-openai-api-key"); + expect(beforeSave.deepgramSecretId).not.toBe("podnotes-deepgram-api-key"); + expect(beforeSave.legacyExtension).toEqual({ enabled: true }); + const storedSecrets = await evalJsonAsync<{ + openAI: boolean; + deepgram: boolean; + openAISentinel: string | null; + deepgramSentinel: string | null; + }>( + obsidian, + `(() => { + const settings = app.plugins.plugins.${PLUGIN_ID}.settings; + return { + openAI: app.secretStorage.getSecret(settings.openAISecretId) === "e2e-openai-secret", + deepgram: app.secretStorage.getSecret(settings.deepgramSecretId) === "e2e-deepgram-secret", + openAISentinel: app.secretStorage.getSecret("podnotes-openai-api-key"), + deepgramSentinel: app.secretStorage.getSecret("podnotes-deepgram-api-key"), + }; + })()`, + ); + expect(storedSecrets).toEqual({ + openAI: true, + deepgram: true, + openAISentinel: "unrelated-openai-sentinel", + deepgramSentinel: "unrelated-deepgram-sentinel", + }); + const runtimeDate = await evalJsonAsync<{ isDate: boolean; iso: string }>( + obsidian, + `(() => { + const value = app.plugins.plugins.${PLUGIN_ID}.settings.currentEpisode.episodeDate; + return { isDate: value instanceof Date, iso: value.toISOString() }; + })()`, + ); + expect(runtimeDate).toEqual({ isDate: true, iso: isoDate }); + + await obsidian.command(`${PLUGIN_ID}:create-podcast-note`).run(); + const note = await sandbox.waitForContent( + "legacy-date-note.md", + (value) => value.includes("date: 2024-03-01"), + WAIT_OPTS, + ); + expect(note).toContain("date: 2024-03-01"); + + await setVolume(obsidian, 0.42); + const persisted = await plugin.waitForData( + (data) => data.schemaVersion === 2 && data.defaultVolume === 0.42, + WAIT_OPTS, + ); + expect(persisted.legacyExtension).toEqual({ enabled: true }); + expect(persisted.currentEpisode?.episodeDate).toBe(isoDate); + const idsBeforeReload = await obsidian.dev.evalJson( + 'app.secretStorage.listSecrets().filter((id) => id.startsWith("podnotes-openai-api-key") || id.startsWith("podnotes-deepgram-api-key")).sort()', + ); + await plugin.reload(RELOAD_OPTIONS); + await waitForPodNotesReady(obsidian); + const afterReload = await plugin.data().read(); + const idsAfterReload = await obsidian.dev.evalJson( + 'app.secretStorage.listSecrets().filter((id) => id.startsWith("podnotes-openai-api-key") || id.startsWith("podnotes-deepgram-api-key")).sort()', + ); + expect(idsAfterReload).toEqual(idsBeforeReload); + expect(afterReload.openAISecretId).toBe(beforeSave.openAISecretId); + expect(afterReload.deepgramSecretId).toBe(beforeSave.deepgramSecretId); + expect(afterReload.openAIApiKey).toBeUndefined(); + expect(afterReload.diarizationApiKey).toBeUndefined(); + expect(await obsidian.dev.runtimeErrors()).toEqual([]); + }); + + test("refuses a future schema without modifying its data", async () => { + const { obsidian, plugin } = getContext(); + const original = await plugin.data().read(); + const future = { + ...original, + schemaVersion: 3, + legacyExtension: { enabled: true }, + }; + + await plugin.disable(); + await plugin.data().write(future); + try { + await plugin.enable().catch(() => undefined); + await obsidian.sleep(200); + + const after = await plugin.data().read(); + const state = await obsidian.dev.evalJson<{ hasCommand: boolean; ready: boolean }>( + `(() => ({ + hasCommand: Boolean(app.commands.commands[${JSON.stringify(`${PLUGIN_ID}:podnotes-show-leaf`)}]), + ready: app.plugins.plugins.${PLUGIN_ID}?.isReady === true, + }))()`, + ); + expect(after).toEqual(future); + expect(state).toEqual({ hasCommand: false, ready: false }); + } finally { + await plugin.data().write(original); + // A failed onload can leave Obsidian's enabled flag set without a live + // plugin instance. Start the restored enable operation without awaiting + // its manager promise: after a rejected onload Obsidian can leave that + // promise pending even though the plugin becomes live. The readiness poll + // below is the authoritative completion signal. + await evalJsonAsync( + obsidian, + `(async () => { + await app.plugins.disablePlugin(${JSON.stringify(PLUGIN_ID)}); + void app.plugins.enablePlugin(${JSON.stringify(PLUGIN_ID)}).catch((error) => { + console.error("PodNotes E2E: restored enable failed", error); + }); + return true; + })()`, + ); + await waitForPodNotesReady(obsidian); + await obsidian.dev.resetDiagnostics().catch(() => undefined); + } + }); }); async function seedAudio(sandbox: SandboxApi, fileName: string): Promise { @@ -504,6 +656,8 @@ async function seedRuntimeData( note?: { path: string; template: string }; played?: { duration: number; time: number }; timestampTemplate?: string; + legacyData?: boolean; + legacySecrets?: { openAI?: string; deepgram?: string }; } = {}, ): Promise { const placeholderEpisode = createLocalEpisode( @@ -517,6 +671,12 @@ async function seedRuntimeData( } await plugin.updateDataAndReload((data) => { + if (options.legacyData) { + delete data.schemaVersion; + data.legacyExtension = { enabled: true }; + data.openAIApiKey = options.legacySecrets?.openAI; + data.diarizationApiKey = options.legacySecrets?.deepgram; + } data.currentEpisode = options.currentEpisode ?? placeholderEpisode; data.defaultPlaybackRate = options.defaultPlaybackRate ?? 1; data.defaultVolume = 1; diff --git a/tests/mocks/obsidian.ts b/tests/mocks/obsidian.ts index 71608622..1e92c233 100644 --- a/tests/mocks/obsidian.ts +++ b/tests/mocks/obsidian.ts @@ -1,4 +1,22 @@ -export class App {} +export class SecretStorage { + private values = new Map(); + + setSecret(id: string, secret: string): void { + this.values.set(id, secret); + } + + getSecret(id: string): string | null { + return this.values.get(id) ?? null; + } + + listSecrets(): string[] { + return [...this.values.keys()]; + } +} + +export class App { + secretStorage = new SecretStorage(); +} export class Plugin {} export class Component { app?: App; @@ -229,6 +247,15 @@ export class TextComponent extends BaseInteractiveElement { } } +export class SecretComponent extends TextComponent { + constructor( + public app: App, + container: HTMLElement, + ) { + super(container); + } +} + export class Setting { settingEl: HTMLElement; @@ -268,6 +295,11 @@ export class Setting { callback(new ButtonComponent(this.settingEl)); return this; } + + addComponent(callback: (container: HTMLElement) => T) { + callback(this.settingEl); + return this; + } } export class PluginSettingTab {