[ENG-1861] Add Roam refresh-all for imported nodes

apps/roam/src/components/RefreshImportedNodeTitleButton.tsx

@@ -18,12 +18,13 @@ const RefreshImportedNodeTitleButton = ({
   const refresh = async (): Promise<void> => {
     setRefreshing(true);
     try {
-      const result = await refreshImportedNode({ pageUid: uid });
+      const result = await refreshImportedNode({ pageUid: uid, force: true });
+      const failed = result.status === "failed";
       renderToast({
-        id: result.success
-          ? "refresh-imported-node-success"
-          : "refresh-imported-node-failed",
-        intent: result.success ? "success" : "danger",
+        id: failed
+          ? "refresh-imported-node-failed"
+          : "refresh-imported-node-success",
+        intent: failed ? "danger" : "success",
         content: result.message,
       });
     } finally {

apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx

@@ -5,6 +5,7 @@ import getSubTree from "roamjs-components/util/getSubTree";
 import { DiscourseNode } from "~/utils/getDiscourseNodes";
 import extractRef from "roamjs-components/util/extractRef";
 import { getAllDiscourseNodesSince } from "~/utils/getAllDiscourseNodesSince";
+import { getImportedNodeUids } from "~/utils/importedSourceIdentity";
 import { upsertNodesToSupabaseAsContentWithEmbeddings } from "~/utils/syncDgNodesToSupabase";
 import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext";
 import {
@@ -56,8 +57,11 @@ const DiscourseNodeSuggestiveRules = ({
 
       const context = await getSupabaseContext();
       if (context && blockNodesSince) {
+        const importedNodeUids = await getImportedNodeUids();
         await upsertNodesToSupabaseAsContentWithEmbeddings(
-          blockNodesSince,
+          blockNodesSince.filter(
+            (node) => !importedNodeUids.has(node.source_local_id),
+          ),
           supabaseClient,
           context,
         );

apps/roam/src/utils/__tests__/refreshAllImportedNodes.test.ts

@@ -0,0 +1,54 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { getImportedNodeUids } from "~/utils/importedSourceIdentity";
+import { refreshAllImportedNodes } from "~/utils/refreshAllImportedNodes";
+import { refreshImportedNode } from "~/utils/refreshImportedNode";
+
+vi.mock("~/utils/importedSourceIdentity", () => ({
+  getImportedNodeUids: vi.fn(),
+}));
+vi.mock("~/utils/refreshImportedNode", () => ({
+  refreshImportedNode: vi.fn(),
+}));
+
+const mockedGetImportedNodeUids = vi.mocked(getImportedNodeUids);
+const mockedRefreshImportedNode = vi.mocked(refreshImportedNode);
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("refreshAllImportedNodes", () => {
+  it("refreshes every imported node without forcing and tallies the outcomes", async () => {
+    mockedGetImportedNodeUids.mockResolvedValue(
+      new Set(["uid-1", "uid-2", "uid-3", "uid-4"]),
+    );
+    mockedRefreshImportedNode
+      .mockResolvedValueOnce({ status: "refreshed", message: "Refreshed." })
+      .mockResolvedValueOnce({ status: "skipped", message: "Up to date." })
+      .mockResolvedValueOnce({ status: "failed", message: "Not shared." })
+      .mockResolvedValueOnce({ status: "refreshed", message: "Refreshed." });
+
+    await expect(refreshAllImportedNodes()).resolves.toEqual({
+      refreshed: 2,
+      skipped: 1,
+      failed: 1,
+    });
+    expect(mockedRefreshImportedNode.mock.calls).toEqual([
+      [{ pageUid: "uid-1", force: false }],
+      [{ pageUid: "uid-2", force: false }],
+      [{ pageUid: "uid-3", force: false }],
+      [{ pageUid: "uid-4", force: false }],
+    ]);
+  });
+
+  it("returns zero counts when the graph has no imported nodes", async () => {
+    mockedGetImportedNodeUids.mockResolvedValue(new Set());
+
+    await expect(refreshAllImportedNodes()).resolves.toEqual({
+      refreshed: 0,
+      skipped: 0,
+      failed: 0,
+    });
+    expect(mockedRefreshImportedNode).not.toHaveBeenCalled();
+  });
+});

apps/roam/src/utils/__tests__/refreshImportedNode.test.ts

@@ -80,8 +80,10 @@ beforeEach(() => {
 
 describe("refreshImportedNode", () => {
   it("refreshes the page from its stored source identity", async () => {
-    await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({
-      success: true,
+    await expect(
+      refreshImportedNode({ pageUid: PAGE_UID, force: true }),
+    ).resolves.toEqual({
+      status: "refreshed",
       message: 'Refreshed "EVD - REM sleep and recall" from Research vault.',
     });
     expect(mockedGetSharedNodeByRid).toHaveBeenCalledWith({
@@ -96,11 +98,36 @@ describe("refreshImportedNode", () => {
     expect(mockedInternalError).not.toHaveBeenCalled();
   });
 
+  it("skips an up-to-date import when not forced", async () => {
+    mockedMaterializeSharedNode.mockResolvedValue({
+      success: true,
+      action: "skipped",
+      pageUid: PAGE_UID,
+      sourceModifiedAt: sharedNode.lastModified,
+      sourceNodeRid: sharedNode.rid,
+    });
+
+    await expect(
+      refreshImportedNode({ pageUid: PAGE_UID, force: false }),
+    ).resolves.toEqual({
+      status: "skipped",
+      message: '"EVD - REM sleep and recall" is already up to date.',
+    });
+    expect(mockedMaterializeSharedNode).toHaveBeenCalledWith({
+      client,
+      sharedNode,
+      force: false,
+    });
+    expect(mockedInternalError).not.toHaveBeenCalled();
+  });
+
   it("fails when the page has no stored source identity", async () => {
     mockedReadImportedSourceIdentity.mockReturnValue(undefined);
 
-    await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({
-      success: false,
+    await expect(
+      refreshImportedNode({ pageUid: PAGE_UID, force: true }),
+    ).resolves.toEqual({
+      status: "failed",
       message: `"${LOCAL_TITLE}" has no stored source identity, so it cannot be refreshed.`,
     });
     expect(mockedGetLoggedInClient).not.toHaveBeenCalled();
@@ -111,8 +138,10 @@ describe("refreshImportedNode", () => {
   it("fails when the database client is unavailable", async () => {
     mockedGetLoggedInClient.mockResolvedValue(null);
 
-    await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({
-      success: false,
+    await expect(
+      refreshImportedNode({ pageUid: PAGE_UID, force: true }),
+    ).resolves.toEqual({
+      status: "failed",
       message: "Could not connect to shared persistence.",
     });
     expect(mockedGetSharedNodeByRid).not.toHaveBeenCalled();
@@ -122,8 +151,10 @@ describe("refreshImportedNode", () => {
   it("fails when the source node is no longer shared", async () => {
     mockedGetSharedNodeByRid.mockResolvedValue(null);
 
-    await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({
-      success: false,
+    await expect(
+      refreshImportedNode({ pageUid: PAGE_UID, force: true }),
+    ).resolves.toEqual({
+      status: "failed",
       message: `The source of "${LOCAL_TITLE}" is no longer shared with your groups, so it cannot be refreshed.`,
     });
     expect(mockedMaterializeSharedNode).not.toHaveBeenCalled();
@@ -141,8 +172,10 @@ describe("refreshImportedNode", () => {
       },
     });
 
-    await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({
-      success: false,
+    await expect(
+      refreshImportedNode({ pageUid: PAGE_UID, force: true }),
+    ).resolves.toEqual({
+      status: "failed",
       message: 'Failed to replace the content of "EVD - old local title"',
     });
     expect(mockedInternalError).toHaveBeenCalledTimes(1);
@@ -165,8 +198,10 @@ describe("refreshImportedNode", () => {
       sourceNodeRid: sharedNode.rid,
     });
 
-    await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({
-      success: false,
+    await expect(
+      refreshImportedNode({ pageUid: PAGE_UID, force: true }),
+    ).resolves.toEqual({
+      status: "failed",
       message: `A different page ("${OTHER_PAGE_TITLE}") is linked to the same source and was refreshed instead.`,
     });
   });
@@ -175,8 +210,10 @@ describe("refreshImportedNode", () => {
     const thrown = new Error("network down");
     mockedGetSharedNodeByRid.mockRejectedValue(thrown);
 
-    await expect(refreshImportedNode({ pageUid: PAGE_UID })).resolves.toEqual({
-      success: false,
+    await expect(
+      refreshImportedNode({ pageUid: PAGE_UID, force: true }),
+    ).resolves.toEqual({
+      status: "failed",
       message: "Could not refresh this page: network down",
     });
     expect(mockedInternalError).toHaveBeenCalledTimes(1);

apps/roam/src/utils/importedSourceIdentity.ts

@@ -74,6 +74,21 @@ export const getImportedSourceRids = async (): Promise<Set<string>> => {
   );
 };
 
+export const getImportedNodeUids = async (): Promise<Set<string>> => {
+  const query = `[:find [?uid ...]
+    :where
+      [?page :block/uid ?uid]
+      [?page :block/props ?props]
+      [(get ?props :${DISCOURSE_GRAPH_PROP_NAME}) ?dgData]
+      [(get ?dgData :${IMPORTED_FROM_PROP_KEY}) ?importedFrom]
+      [(get ?importedFrom :${SOURCE_NODE_RID_KEY}) ?rid]]`;
+  const result = (await window.roamAlphaAPI.data.async.q(query)) as unknown[];
+
+  return new Set(
+    result.filter((uid): uid is string => typeof uid === "string"),
+  );
+};
+
 export const findImportedNodeUidBySourceRid = async (
   sourceNodeRid: string,
 ): Promise<string | null> => {

apps/roam/src/utils/refreshAllImportedNodes.ts

@@ -0,0 +1,23 @@
+import { getImportedNodeUids } from "./importedSourceIdentity";
+import { refreshImportedNode } from "./refreshImportedNode";
+
+type RefreshAllImportedNodesResult = {
+  refreshed: number;
+  skipped: number;
+  failed: number;
+};
+
+export const refreshAllImportedNodes =
+  async (): Promise<RefreshAllImportedNodesResult> => {
+    const pageUids = await getImportedNodeUids();
+    const counts: RefreshAllImportedNodesResult = {
+      refreshed: 0,
+      skipped: 0,
+      failed: 0,
+    };
+    for (const pageUid of pageUids) {
+      const result = await refreshImportedNode({ pageUid, force: false });
+      counts[result.status] += 1;
+    }
+    return counts;
+  };

apps/roam/src/utils/refreshImportedNode.ts

@@ -8,32 +8,34 @@ import {
 } from "./materializeSharedNode";
 import { getLoggedInClient } from "./supabaseContext";
 
-const REFRESH_ERROR_TYPE = "Imported node refresh failed";
+export const REFRESH_ERROR_TYPE = "Imported node refresh failed";
 const REFRESH_ERROR_OPERATION = "refresh-imported-node";
 
 type RefreshImportedNodeResult = {
-  success: boolean;
+  status: "refreshed" | "skipped" | "failed";
   message: string;
 };
 
 export const refreshImportedNode = async ({
   pageUid,
+  force,
 }: {
   pageUid: string;
+  force: boolean;
 }): Promise<RefreshImportedNodeResult> => {
   try {
     const title = getPageTitleByPageUid(pageUid);
     const identity = readImportedSourceIdentity(pageUid);
     if (!identity)
       return {
-        success: false,
+        status: "failed",
         message: `"${title}" has no stored source identity, so it cannot be refreshed.`,
       };
 
     const client = await getLoggedInClient();
     if (!client)
       return {
-        success: false,
+        status: "failed",
         message: "Could not connect to shared persistence.",
       };
 
@@ -43,14 +45,14 @@ export const refreshImportedNode = async ({
     });
     if (!sharedNode)
       return {
-        success: false,
+        status: "failed",
         message: `The source of "${title}" is no longer shared with your groups, so it cannot be refreshed.`,
       };
 
     const result = await materializeSharedNode({
       client,
       sharedNode,
-      force: true,
+      force,
     });
     if (!result.success) {
       internalError({
@@ -63,15 +65,20 @@ export const refreshImportedNode = async ({
         },
         sendEmail: false,
       });
-      return { success: false, message: result.error.message };
+      return { status: "failed", message: result.error.message };
     }
     if (result.pageUid !== pageUid)
       return {
-        success: false,
+        status: "failed",
         message: `A different page ("${getPageTitleByPageUid(result.pageUid)}") is linked to the same source and was refreshed instead.`,
       };
+    if (result.action === "skipped")
+      return {
+        status: "skipped",
+        message: `"${sharedNode.title}" is already up to date.`,
+      };
     return {
-      success: true,
+      status: "refreshed",
       message: `Refreshed "${sharedNode.title}" from ${sharedNode.spaceName}.`,
     };
   } catch (error) {
@@ -82,7 +89,7 @@ export const refreshImportedNode = async ({
       sendEmail: false,
     });
     return {
-      success: false,
+      status: "failed",
       message: `Could not refresh this page: ${getErrorMessage(error)}`,
     };
   }