diff --git a/apps/roam/src/components/RefreshImportedNodeTitleButton.tsx b/apps/roam/src/components/RefreshImportedNodeTitleButton.tsx index 1e6eec39a..2fa6ac04a 100644 --- a/apps/roam/src/components/RefreshImportedNodeTitleButton.tsx +++ b/apps/roam/src/components/RefreshImportedNodeTitleButton.tsx @@ -18,12 +18,13 @@ const RefreshImportedNodeTitleButton = ({ const refresh = async (): Promise => { setRefreshing(true); try { - const result = await refreshImportedNode({ pageUid: uid }); + const result = await refreshImportedNode({ pageUid: uid, force: true }); + const failed = result.status === "failed"; renderToast({ - id: result.success - ? "refresh-imported-node-success" - : "refresh-imported-node-failed", - intent: result.success ? "success" : "danger", + id: failed + ? "refresh-imported-node-failed" + : "refresh-imported-node-success", + intent: failed ? "danger" : "success", content: result.message, }); } finally { diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index 67a546c92..a224327dd 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -5,6 +5,7 @@ import getSubTree from "roamjs-components/util/getSubTree"; import { DiscourseNode } from "~/utils/getDiscourseNodes"; import extractRef from "roamjs-components/util/extractRef"; import { getAllDiscourseNodesSince } from "~/utils/getAllDiscourseNodesSince"; +import { getImportedNodeUids } from "~/utils/importedSourceIdentity"; import { upsertNodesToSupabaseAsContentWithEmbeddings } from "~/utils/syncDgNodesToSupabase"; import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; import { @@ -56,8 +57,11 @@ const DiscourseNodeSuggestiveRules = ({ const context = await getSupabaseContext(); if (context && blockNodesSince) { + const importedNodeUids = await getImportedNodeUids(); await upsertNodesToSupabaseAsContentWithEmbeddings( - blockNodesSince, + blockNodesSince.filter( + (node) => !importedNodeUids.has(node.source_local_id), + ), supabaseClient, context, ); diff --git a/apps/roam/src/utils/__tests__/refreshAllImportedNodes.test.ts b/apps/roam/src/utils/__tests__/refreshAllImportedNodes.test.ts new file mode 100644 index 000000000..021e77dea --- /dev/null +++ b/apps/roam/src/utils/__tests__/refreshAllImportedNodes.test.ts @@ -0,0 +1,54 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { getImportedNodeUids } from "~/utils/importedSourceIdentity"; +import { refreshAllImportedNodes } from "~/utils/refreshAllImportedNodes"; +import { refreshImportedNode } from "~/utils/refreshImportedNode"; + +vi.mock("~/utils/importedSourceIdentity", () => ({ + getImportedNodeUids: vi.fn(), +})); +vi.mock("~/utils/refreshImportedNode", () => ({ + refreshImportedNode: vi.fn(), +})); + +const mockedGetImportedNodeUids = vi.mocked(getImportedNodeUids); +const mockedRefreshImportedNode = vi.mocked(refreshImportedNode); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("refreshAllImportedNodes", () => { + it("refreshes every imported node without forcing and tallies the outcomes", async () => { + mockedGetImportedNodeUids.mockResolvedValue( + new Set(["uid-1", "uid-2", "uid-3", "uid-4"]), + ); + mockedRefreshImportedNode + .mockResolvedValueOnce({ status: "refreshed", message: "Refreshed." }) + .mockResolvedValueOnce({ status: "skipped", message: "Up to date." }) + .mockResolvedValueOnce({ status: "failed", message: "Not shared." }) + .mockResolvedValueOnce({ status: "refreshed", message: "Refreshed." }); + + await expect(refreshAllImportedNodes()).resolves.toEqual({ + refreshed: 2, + skipped: 1, + failed: 1, + }); + expect(mockedRefreshImportedNode.mock.calls).toEqual([ + [{ pageUid: "uid-1", force: false }], + [{ pageUid: "uid-2", force: false }], + [{ pageUid: "uid-3", force: false }], + [{ pageUid: "uid-4", force: false }], + ]); + }); + + it("returns zero counts when the graph has no imported nodes", async () => { + mockedGetImportedNodeUids.mockResolvedValue(new Set()); + + await expect(refreshAllImportedNodes()).resolves.toEqual({ + refreshed: 0, + skipped: 0, + failed: 0, + }); + expect(mockedRefreshImportedNode).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/roam/src/utils/__tests__/refreshImportedNode.test.ts b/apps/roam/src/utils/__tests__/refreshImportedNode.test.ts index a308aa3ee..0ba1b468b 100644 --- a/apps/roam/src/utils/__tests__/refreshImportedNode.test.ts +++ b/apps/roam/src/utils/__tests__/refreshImportedNode.test.ts @@ -80,8 +80,10 @@ beforeEach(() => { describe("refreshImportedNode", () => { it("refreshes the page from its stored source identity", async () => { - await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({ - success: true, + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "refreshed", message: 'Refreshed "EVD - REM sleep and recall" from Research vault.', }); expect(mockedGetSharedNodeByRid).toHaveBeenCalledWith({ @@ -96,11 +98,36 @@ describe("refreshImportedNode", () => { expect(mockedInternalError).not.toHaveBeenCalled(); }); + it("skips an up-to-date import when not forced", async () => { + mockedMaterializeSharedNode.mockResolvedValue({ + success: true, + action: "skipped", + pageUid: PAGE_UID, + sourceModifiedAt: sharedNode.lastModified, + sourceNodeRid: sharedNode.rid, + }); + + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: false }), + ).resolves.toEqual({ + status: "skipped", + message: '"EVD - REM sleep and recall" is already up to date.', + }); + expect(mockedMaterializeSharedNode).toHaveBeenCalledWith({ + client, + sharedNode, + force: false, + }); + expect(mockedInternalError).not.toHaveBeenCalled(); + }); + it("fails when the page has no stored source identity", async () => { mockedReadImportedSourceIdentity.mockReturnValue(undefined); - await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({ - success: false, + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "failed", message: `"${LOCAL_TITLE}" has no stored source identity, so it cannot be refreshed.`, }); expect(mockedGetLoggedInClient).not.toHaveBeenCalled(); @@ -111,8 +138,10 @@ describe("refreshImportedNode", () => { it("fails when the database client is unavailable", async () => { mockedGetLoggedInClient.mockResolvedValue(null); - await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({ - success: false, + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "failed", message: "Could not connect to shared persistence.", }); expect(mockedGetSharedNodeByRid).not.toHaveBeenCalled(); @@ -122,8 +151,10 @@ describe("refreshImportedNode", () => { it("fails when the source node is no longer shared", async () => { mockedGetSharedNodeByRid.mockResolvedValue(null); - await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({ - success: false, + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "failed", message: `The source of "${LOCAL_TITLE}" is no longer shared with your groups, so it cannot be refreshed.`, }); expect(mockedMaterializeSharedNode).not.toHaveBeenCalled(); @@ -141,8 +172,10 @@ describe("refreshImportedNode", () => { }, }); - await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({ - success: false, + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "failed", message: 'Failed to replace the content of "EVD - old local title"', }); expect(mockedInternalError).toHaveBeenCalledTimes(1); @@ -165,8 +198,10 @@ describe("refreshImportedNode", () => { sourceNodeRid: sharedNode.rid, }); - await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({ - success: false, + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "failed", message: `A different page ("${OTHER_PAGE_TITLE}") is linked to the same source and was refreshed instead.`, }); }); @@ -175,8 +210,10 @@ describe("refreshImportedNode", () => { const thrown = new Error("network down"); mockedGetSharedNodeByRid.mockRejectedValue(thrown); - await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({ - success: false, + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "failed", message: "Could not refresh this page: network down", }); expect(mockedInternalError).toHaveBeenCalledTimes(1); diff --git a/apps/roam/src/utils/importedSourceIdentity.ts b/apps/roam/src/utils/importedSourceIdentity.ts index 64588918a..0b3775f25 100644 --- a/apps/roam/src/utils/importedSourceIdentity.ts +++ b/apps/roam/src/utils/importedSourceIdentity.ts @@ -74,6 +74,21 @@ export const getImportedSourceRids = async (): Promise> => { ); }; +export const getImportedNodeUids = async (): Promise> => { + const query = `[:find [?uid ...] + :where + [?page :block/uid ?uid] + [?page :block/props ?props] + [(get ?props :${DISCOURSE_GRAPH_PROP_NAME}) ?dgData] + [(get ?dgData :${IMPORTED_FROM_PROP_KEY}) ?importedFrom] + [(get ?importedFrom :${SOURCE_NODE_RID_KEY}) ?rid]]`; + const result = (await window.roamAlphaAPI.data.async.q(query)) as unknown[]; + + return new Set( + result.filter((uid): uid is string => typeof uid === "string"), + ); +}; + export const findImportedNodeUidBySourceRid = async ( sourceNodeRid: string, ): Promise => { diff --git a/apps/roam/src/utils/refreshAllImportedNodes.ts b/apps/roam/src/utils/refreshAllImportedNodes.ts new file mode 100644 index 000000000..5fbe8e990 --- /dev/null +++ b/apps/roam/src/utils/refreshAllImportedNodes.ts @@ -0,0 +1,23 @@ +import { getImportedNodeUids } from "./importedSourceIdentity"; +import { refreshImportedNode } from "./refreshImportedNode"; + +type RefreshAllImportedNodesResult = { + refreshed: number; + skipped: number; + failed: number; +}; + +export const refreshAllImportedNodes = + async (): Promise => { + const pageUids = await getImportedNodeUids(); + const counts: RefreshAllImportedNodesResult = { + refreshed: 0, + skipped: 0, + failed: 0, + }; + for (const pageUid of pageUids) { + const result = await refreshImportedNode({ pageUid, force: false }); + counts[result.status] += 1; + } + return counts; + }; diff --git a/apps/roam/src/utils/refreshImportedNode.ts b/apps/roam/src/utils/refreshImportedNode.ts index 62e32381d..f355eb383 100644 --- a/apps/roam/src/utils/refreshImportedNode.ts +++ b/apps/roam/src/utils/refreshImportedNode.ts @@ -8,32 +8,34 @@ import { } from "./materializeSharedNode"; import { getLoggedInClient } from "./supabaseContext"; -const REFRESH_ERROR_TYPE = "Imported node refresh failed"; +export const REFRESH_ERROR_TYPE = "Imported node refresh failed"; const REFRESH_ERROR_OPERATION = "refresh-imported-node"; type RefreshImportedNodeResult = { - success: boolean; + status: "refreshed" | "skipped" | "failed"; message: string; }; export const refreshImportedNode = async ({ pageUid, + force, }: { pageUid: string; + force: boolean; }): Promise => { try { const title = getPageTitleByPageUid(pageUid); const identity = readImportedSourceIdentity(pageUid); if (!identity) return { - success: false, + status: "failed", message: `"${title}" has no stored source identity, so it cannot be refreshed.`, }; const client = await getLoggedInClient(); if (!client) return { - success: false, + status: "failed", message: "Could not connect to shared persistence.", }; @@ -43,14 +45,14 @@ export const refreshImportedNode = async ({ }); if (!sharedNode) return { - success: false, + status: "failed", message: `The source of "${title}" is no longer shared with your groups, so it cannot be refreshed.`, }; const result = await materializeSharedNode({ client, sharedNode, - force: true, + force, }); if (!result.success) { internalError({ @@ -63,15 +65,20 @@ export const refreshImportedNode = async ({ }, sendEmail: false, }); - return { success: false, message: result.error.message }; + return { status: "failed", message: result.error.message }; } if (result.pageUid !== pageUid) return { - success: false, + status: "failed", message: `A different page ("${getPageTitleByPageUid(result.pageUid)}") is linked to the same source and was refreshed instead.`, }; + if (result.action === "skipped") + return { + status: "skipped", + message: `"${sharedNode.title}" is already up to date.`, + }; return { - success: true, + status: "refreshed", message: `Refreshed "${sharedNode.title}" from ${sharedNode.spaceName}.`, }; } catch (error) { @@ -82,7 +89,7 @@ export const refreshImportedNode = async ({ sendEmail: false, }); return { - success: false, + status: "failed", message: `Could not refresh this page: ${getErrorMessage(error)}`, }; } diff --git a/apps/roam/src/utils/registerCommandPaletteCommands.ts b/apps/roam/src/utils/registerCommandPaletteCommands.ts index 140488177..d8579fe9c 100644 --- a/apps/roam/src/utils/registerCommandPaletteCommands.ts +++ b/apps/roam/src/utils/registerCommandPaletteCommands.ts @@ -55,6 +55,11 @@ import { insertPageRefAtRange, } from "./advancedSearchFooterUtils"; import { renderDiscoverSharedNodesDialog } from "~/components/DiscoverSharedNodesDialog"; +import { refreshAllImportedNodes } from "~/utils/refreshAllImportedNodes"; +import { REFRESH_ERROR_TYPE } from "~/utils/refreshImportedNode"; +import internalError from "~/utils/internalError"; + +const REFRESH_ALL_TOAST_ID = "refresh-imported-nodes"; export const createDiscourseNodeFromCommand = ( extensionAPI: OnloadArgs["extensionAPI"], @@ -348,6 +353,50 @@ export const registerCommandPaletteCommands = (onloadArgs: OnloadArgs) => { renderDiscoverSharedNodesDialog({}); }; + const refreshAllImportedNodesFromCommand = async () => { + if (!isNodeSharingEnabled()) { + renderToast({ + id: "refresh-imported-nodes-sharing-disabled", + content: "Node sharing must be enabled to refresh imported nodes.", + }); + return; + } + + posthog.capture("Refresh Imported Node: Refresh All Command Triggered"); + renderToast({ + id: REFRESH_ALL_TOAST_ID, + content: "Refreshing imported nodes…", + timeout: 0, + }); + try { + const { refreshed, skipped, failed } = await refreshAllImportedNodes(); + if (refreshed + skipped + failed === 0) { + renderToast({ + id: REFRESH_ALL_TOAST_ID, + content: "No imported nodes to refresh.", + }); + return; + } + renderToast({ + id: REFRESH_ALL_TOAST_ID, + intent: failed > 0 ? "warning" : "success", + content: `${refreshed} refreshed, ${skipped} skipped, ${failed} failed.`, + }); + } catch (error) { + internalError({ + error, + type: REFRESH_ERROR_TYPE, + context: { operation: "refresh-all-imported-nodes" }, + sendEmail: false, + }); + renderToast({ + id: REFRESH_ALL_TOAST_ID, + intent: "danger", + content: "Could not refresh imported nodes.", + }); + } + }; + const toggleDiscourseContextOverlay = async () => { const currentValue = getPersonalSetting([ PERSONAL_KEYS.discourseContextOverlay, @@ -425,6 +474,10 @@ export const registerCommandPaletteCommands = (onloadArgs: OnloadArgs) => { void addCommand("DG: Discover shared nodes", discoverSharedNodes); } if (isNodeSharingEnabled()) { + void addCommand( + "DG: Refresh all imported nodes", + () => void refreshAllImportedNodesFromCommand(), + ); void addCommand("DG: Share current node", shareCurrentNode); } if (getFeatureFlag("Advanced node search enabled")) { diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index 5427398e9..194599002 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -4,6 +4,7 @@ import { nodeTypeSince, } from "./getAllDiscourseNodesSince"; import getDiscourseNodeFormatExpression from "./getDiscourseNodeFormatExpression"; +import { getImportedNodeUids } from "./importedSourceIdentity"; import { cleanupOrphanedNodes } from "./cleanupOrphanedNodes"; import { getLoggedInClient, @@ -1226,11 +1227,19 @@ export const createOrUpdateDiscourseEmbedding = async ( spaceId: activeContext.spaceId, }), }); + const importedNodeUids = await measureSyncPhase({ + phase: "getImportedNodeUids", + phases, + operation: () => getImportedNodeUids(), + }); + const localNodeInstances = changedNodeInstances.filter( + (node) => !importedNodeUids.has(node.source_local_id), + ); const nodeInstancesToSync = sharedNodesOnlySync - ? changedNodeInstances.filter((node) => + ? localNodeInstances.filter((node) => sharedSourceLocalIds.has(node.source_local_id), ) - : changedNodeInstances; + : localNodeInstances; const sharedSourceLocalIdsToBackfill = await measureSyncPhase({ phase: "getSharedSourceLocalIdsMissingFullContent", phases,