fix: retry Foundry binary downloads

eslint-suppressions.json

@@ -1004,11 +1004,6 @@
       "count": 1
     }
   },
-  "packages/foundryup/src/download.ts": {
-    "@typescript-eslint/explicit-function-return-type": {
-      "count": 1
-    }
-  },
   "packages/foundryup/src/extract.ts": {
     "@typescript-eslint/explicit-function-return-type": {
       "count": 4

packages/foundryup/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Retry transient Foundry archive download failures with exponential backoff and equal jitter, preventing short-lived network, rate-limit, and server errors from immediately aborting installation ([#9854](https://github.com/MetaMask/core/pull/9854))
+  - Configure the retry policy with the `--max-attempts`, `--initial-retry-delay-ms`, and `--max-retry-delay-ms` CLI options or their equivalent `FOUNDRYUP_*` environment variables
+
 ## [1.0.1]
 
 ### Fixed

packages/foundryup/README.md

@@ -18,6 +18,19 @@ This will install the latest version of Foundry things by default.
 
 Try `yarn bin mm-foundryup --help` for more options.
 
+### Retry configuration
+
+Foundry archive downloads retry transient failures with exponential backoff and equal jitter. The retry policy can be
+configured with CLI flags or equivalent environment variables:
+
+| CLI flag                   | Environment variable               | Default | Description                                    |
+| -------------------------- | ---------------------------------- | ------: | ---------------------------------------------- |
+| `--max-attempts`           | `FOUNDRYUP_MAX_ATTEMPTS`           |       5 | Total download attempts, including the first   |
+| `--initial-retry-delay-ms` | `FOUNDRYUP_INITIAL_RETRY_DELAY_MS` |   1,000 | Initial delay before exponential growth, in ms |
+| `--max-retry-delay-ms`     | `FOUNDRYUP_MAX_RETRY_DELAY_MS`     |  30,000 | Maximum delay before applying jitter, in ms    |
+
+Set `--max-attempts 1` to disable retries.
+
 Once you have the binaries installed, you have to figure out how to get to them.
 
 Probably best to just add each as a `package.json` script:

packages/foundryup/src/download.test.ts

@@ -0,0 +1,136 @@
+import {
+  calculateRetryDelay,
+  DEFAULT_DOWNLOAD_RETRY_OPTIONS,
+  DownloadHttpError,
+  isRetryableDownloadError,
+  retryDownload,
+} from './download.js';
+
+describe('calculateRetryDelay', () => {
+  it('applies equal jitter to exponential delays', () => {
+    const random = jest.fn().mockReturnValue(0.5);
+
+    expect(calculateRetryDelay(1, { random })).toBe(750);
+    expect(calculateRetryDelay(2, { random })).toBe(1_500);
+    expect(calculateRetryDelay(3, { random })).toBe(3_000);
+  });
+
+  it('caps the exponential delay', () => {
+    expect(
+      calculateRetryDelay(10, {
+        initialDelayMs: 1_000,
+        maxDelayMs: 10_000,
+        random: () => 0.5,
+      }),
+    ).toBe(7_500);
+  });
+});
+
+describe('isRetryableDownloadError', () => {
+  it.each([408, 425, 429, 500, 503])(
+    'returns true for HTTP status %s',
+    (statusCode) => {
+      expect(
+        isRetryableDownloadError(
+          new DownloadHttpError(
+            new URL('https://example.com/archive.tar.gz'),
+            statusCode,
+            'Request failed',
+          ),
+        ),
+      ).toBe(true);
+    },
+  );
+
+  it.each([400, 401, 403, 404])(
+    'returns false for HTTP status %s',
+    (statusCode) => {
+      expect(
+        isRetryableDownloadError(
+          new DownloadHttpError(
+            new URL('https://example.com/archive.tar.gz'),
+            statusCode,
+            'Request failed',
+          ),
+        ),
+      ).toBe(false);
+    },
+  );
+
+  it('returns true for transient network errors', () => {
+    const error = new Error('socket hang up') as NodeJS.ErrnoException;
+    error.code = 'ECONNRESET';
+
+    expect(isRetryableDownloadError(error)).toBe(true);
+  });
+
+  it('returns false for permanent errors', () => {
+    expect(isRetryableDownloadError(new Error('checksum mismatch'))).toBe(
+      false,
+    );
+  });
+});
+
+describe('retryDownload', () => {
+  it('retries transient failures with exponential backoff and jitter', async () => {
+    const transientError = new Error('socket hang up') as NodeJS.ErrnoException;
+    transientError.code = 'ECONNRESET';
+    const operation = jest
+      .fn<Promise<string>, []>()
+      .mockRejectedValueOnce(transientError)
+      .mockRejectedValueOnce(transientError)
+      .mockResolvedValue('downloaded');
+    const sleep = jest.fn<Promise<void>, [number]>();
+    sleep.mockResolvedValue(undefined);
+    const onRetry = jest.fn();
+
+    const result = await retryDownload(operation, {
+      random: () => 0.5,
+      sleep,
+      onRetry,
+    });
+
+    expect(result).toBe('downloaded');
+    expect(operation).toHaveBeenCalledTimes(3);
+    expect(sleep.mock.calls).toStrictEqual([[750], [1_500]]);
+    expect(onRetry).toHaveBeenNthCalledWith(1, {
+      attempt: 2,
+      maxAttempts: DEFAULT_DOWNLOAD_RETRY_OPTIONS.maxAttempts,
+      delayMs: 750,
+      error: transientError,
+    });
+    expect(onRetry).toHaveBeenNthCalledWith(2, {
+      attempt: 3,
+      maxAttempts: DEFAULT_DOWNLOAD_RETRY_OPTIONS.maxAttempts,
+      delayMs: 1_500,
+      error: transientError,
+    });
+  });
+
+  it('does not retry permanent failures', async () => {
+    const operation = jest.fn().mockRejectedValue(new Error('Invalid URL'));
+    const sleep = jest.fn<Promise<void>, [number]>();
+
+    await expect(retryDownload(operation, { sleep })).rejects.toThrow(
+      'Invalid URL',
+    );
+    expect(operation).toHaveBeenCalledTimes(1);
+    expect(sleep).not.toHaveBeenCalled();
+  });
+
+  it('stops after the default maximum number of attempts', async () => {
+    const transientError = new Error('socket hang up') as NodeJS.ErrnoException;
+    transientError.code = 'ECONNRESET';
+    const operation = jest.fn().mockRejectedValue(transientError);
+    const sleep = jest.fn<Promise<void>, [number]>();
+    sleep.mockResolvedValue(undefined);
+
+    await expect(
+      retryDownload(operation, { random: () => 0.5, sleep }),
+    ).rejects.toBe(transientError);
+    expect(operation).toHaveBeenCalledTimes(
+      DEFAULT_DOWNLOAD_RETRY_OPTIONS.maxAttempts,
+    );
+    expect(sleep.mock.calls).toStrictEqual([[750], [1_500], [3_000], [6_000]]);
+  });
+});

packages/foundryup/src/download.ts

@@ -5,6 +5,170 @@ import { Stream } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
 import type { DownloadOptions } from './types.js';
+import { isCodedError } from './utils.js';
+
+const RETRYABLE_HTTP_STATUS_CODES = new Set([408, 425, 429]);
+const RETRYABLE_ERROR_CODES = new Set([
+  'EAI_AGAIN',
+  'ECONNREFUSED',
+  'ECONNRESET',
+  'EHOSTDOWN',
+  'EHOSTUNREACH',
+  'ENETDOWN',
+  'ENETRESET',
+  'ENETUNREACH',
+  'EPIPE',
+  'ERR_STREAM_PREMATURE_CLOSE',
+  'ETIMEDOUT',
+  'Z_BUF_ERROR',
+]);
+
+export const DEFAULT_DOWNLOAD_RETRY_OPTIONS = {
+  maxAttempts: 5,
+  initialDelayMs: 1_000,
+  maxDelayMs: 30_000,
+} as const;
+
+export type DownloadRetryEvent = {
+  attempt: number;
+  maxAttempts: number;
+  delayMs: number;
+  error: unknown;
+};
+
+export type DownloadRetryConfiguration = {
+  maxAttempts?: number;
+  initialDelayMs?: number;
+  maxDelayMs?: number;
+};
+
+export type DownloadRetryOptions = DownloadRetryConfiguration & {
+  random?: () => number;
+  sleep?: (delayMs: number) => Promise<void>;
+  onRetry?: (event: DownloadRetryEvent) => void;
+};
+
+/**
+ * An unsuccessful HTTP response received while downloading an archive.
+ */
+export class DownloadHttpError extends Error {
+  readonly statusCode: number | undefined;
+
+  constructor(
+    url: URL,
+    statusCode: number | undefined,
+    statusMessage: string | undefined,
+  ) {
+    super(
+      `Request to ${url} failed. Status Code: ${statusCode} - ${statusMessage}`,
+    );
+    this.name = 'DownloadHttpError';
+    this.statusCode = statusCode;
+  }
+}
+
+/**
+ * Determines whether a failed download can reasonably succeed when retried.
+ *
+ * @param error - The download error.
+ * @returns Whether the error is transient.
+ */
+export function isRetryableDownloadError(error: unknown): boolean {
+  if (error instanceof DownloadHttpError) {
+    const { statusCode } = error;
+    return (
+      statusCode !== undefined &&
+      (RETRYABLE_HTTP_STATUS_CODES.has(statusCode) ||
+        (statusCode >= 500 && statusCode <= 599))
+    );
+  }
+
+  return isCodedError(error) && RETRYABLE_ERROR_CODES.has(error.code);
+}
+
+/**
+ * Calculates an exponential retry delay with equal jitter.
+ *
+ * Equal jitter keeps at least half of the exponential delay while spreading
+ * concurrent retry attempts across the remaining half.
+ *
+ * @param failedAttempt - The one-based attempt number that failed.
+ * @param options - Backoff and randomness options.
+ * @param options.initialDelayMs - The delay before exponential growth.
+ * @param options.maxDelayMs - The maximum delay before jitter.
+ * @param options.random - The source of randomness used for jitter.
+ * @returns The delay before the next attempt, in milliseconds.
+ */
+export function calculateRetryDelay(
+  failedAttempt: number,
+  {
+    initialDelayMs = DEFAULT_DOWNLOAD_RETRY_OPTIONS.initialDelayMs,
+    maxDelayMs = DEFAULT_DOWNLOAD_RETRY_OPTIONS.maxDelayMs,
+    random = Math.random,
+  }: Pick<
+    DownloadRetryOptions,
+    'initialDelayMs' | 'maxDelayMs' | 'random'
+  > = {},
+): number {
+  const exponentialDelay = Math.min(
+    maxDelayMs,
+    initialDelayMs * 2 ** (failedAttempt - 1),
+  );
+  const minimumDelay = exponentialDelay / 2;
+  return Math.round(minimumDelay + random() * minimumDelay);
+}
+
+/**
+ * Retries a download operation after transient failures.
+ *
+ * @param operation - The complete download operation to retry.
+ * @param options - Retry, backoff, and observability options.
+ * @param options.maxAttempts - The total number of attempts, including the first.
+ * @param options.initialDelayMs - The delay before exponential growth.
+ * @param options.maxDelayMs - The maximum delay before jitter.
+ * @param options.random - The source of randomness used for jitter.
+ * @param options.sleep - The function used to wait between attempts.
+ * @param options.onRetry - A callback invoked before each retry.
+ * @returns The result of the successful operation.
+ * @throws The first permanent error or the final transient error.
+ */
+export async function retryDownload<Result>(
+  operation: () => Promise<Result>,
+  {
+    maxAttempts = DEFAULT_DOWNLOAD_RETRY_OPTIONS.maxAttempts,
+    initialDelayMs = DEFAULT_DOWNLOAD_RETRY_OPTIONS.initialDelayMs,
+    maxDelayMs = DEFAULT_DOWNLOAD_RETRY_OPTIONS.maxDelayMs,
+    random = Math.random,
+    sleep = async (delayMs: number): Promise<void> =>
+      await new Promise<void>((resolve) => setTimeout(resolve, delayMs)),
+    onRetry,
+  }: DownloadRetryOptions = {},
+): Promise<Result> {
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await operation();
+    } catch (error) {
+      if (attempt === maxAttempts || !isRetryableDownloadError(error)) {
+        throw error;
+      }
+
+      const delayMs = calculateRetryDelay(attempt, {
+        initialDelayMs,
+        maxDelayMs,
+        random,
+      });
+      onRetry?.({
+        attempt: attempt + 1,
+        maxAttempts,
+        delayMs,
+        error,
+      });
+      await sleep(delayMs);
+    }
+  }
+
+  throw new Error('Download retry loop completed unexpectedly');
+}
 
 /**
  * A PassThrough stream that emits a 'response' event when the HTTP(S) response is available.
@@ -35,7 +199,7 @@ export function startDownload(
   url: URL,
   options: DownloadOptions = {},
   redirects: number = 0,
-) {
+): DownloadStream {
   const MAX_REDIRECTS = options.maxRedirects ?? 5;
   const request = url.protocol === 'http:' ? httpRequest : httpsRequest;
   const stream = new DownloadStream();
@@ -72,9 +236,7 @@ export function startDownload(
     else if (!statusCode || statusCode < 200 || statusCode >= 300) {
       stream.emit(
         'error',
-        new Error(
-          `Request to ${url} failed. Status Code: ${statusCode} - ${statusMessage}`,
-        ),
+        new DownloadHttpError(url, statusCode, statusMessage),
       );
       response.destroy();
     } else {