Fix: Prevent attachment to crash when asset-server is not available

specifyweb/backend/attachment_gw/urls.py

@@ -6,6 +6,7 @@
     path('get_settings/', views.get_settings),
     path('get_upload_params/', views.get_upload_params),
     path('get_token/', views.get_token),
+    path('health/', views.health),
     path('proxy/', views.proxy),
     path('download_all/', views.download_all),
     path('dataset/', views.datasets),

specifyweb/backend/attachment_gw/views.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import time
+from threading import Lock
 from tempfile import mkdtemp
 from os.path import splitext
 from uuid import uuid4
@@ -33,6 +34,10 @@
 
 server_urls = None
 server_time_delta = None
+initialization_lock = Lock()
+next_initialization_attempt = 0
+initialization_failures = 0
+MAX_INITIALIZATION_RETRY_DELAY = 60
 
 from .models import Spattachmentdataset
 
@@ -226,33 +231,76 @@ def init():
         logger.info('Asset server is not configured')
         return
 
-    r = requests.get(settings.WEB_ATTACHMENT_URL)
-    if r.status_code != 200:
-        logger.error('Failed fetching asset server configuration')
-        return
-
-    update_time_delta(r)
-
     try:
-        urls_xml = ElementTree.fromstring(r.text)
-    except:
-        logger.error('Failed parsing the response')
-        return
+        r = requests.get(settings.WEB_ATTACHMENT_URL, timeout=settings.WEB_ATTACHMENT_TIMEOUT)
+        if r.status_code != 200:
+            logger.error('Failed fetching asset server configuration')
+            return
+
+        update_time_delta(r)
+
+        try:
+            urls_xml = ElementTree.fromstring(r.text)
+        except ElementTree.ParseError:
+            logger.error('Failed parsing the response')
+            return
+
+        try:
+            urls = {url.attrib['type']: url.text
+                    for url in urls_xml.findall('url')}
+            required_url_types = (
+                'delete',
+                'getmetadata',
+                'read',
+                'testkey',
+                'write',
+            )
+            if any(not urls.get(url_type) for url_type in required_url_types):
+                raise AttachmentError('Incomplete asset server configuration.')
+            test_key(urls)
+            server_urls = urls
+        except (AttachmentError, requests.RequestException, KeyError) as error:
+            logger.error('Invalid asset server configuration: %s', str(error))
+            server_urls = None
+    except requests.RequestException as error:
+        logger.error('Failed to connect to asset server: %s', str(error))
+        server_urls = None
 
-    server_urls = {url.attrib['type']: url.text
-                   for url in urls_xml.findall('url')}
+def retry_initialization():
+    """Attempt a bounded, backoff-limited asset server initialization."""
+    global initialization_failures, next_initialization_attempt
 
-    try:
-        test_key()
-    except AttachmentError as error:
-        logger.error('%s', str(error))
-        server_urls = None
+    now = time.monotonic()
+    if now < next_initialization_attempt or not initialization_lock.acquire(blocking=False):
+        return False
 
-def test_key():
+    try:
+        if server_urls is not None:
+            return True
+        if now < next_initialization_attempt:
+            return False
+
+        init()
+        if server_urls is not None:
+            initialization_failures = 0
+            next_initialization_attempt = 0
+            return True
+
+        initialization_failures += 1
+        next_initialization_attempt = time.monotonic() + min(
+            2 ** initialization_failures,
+            MAX_INITIALIZATION_RETRY_DELAY,
+        )
+        return False
+    finally:
+        initialization_lock.release()
+
+def test_key(urls=None):
     random = str(uuid4())
     token = generate_token(get_timestamp(), random)
-    r = requests.get(server_urls["testkey"],
-                     params={'random': random, 'token': token})
+    r = requests.get((urls or server_urls)["testkey"],
+                     params={'random': random, 'token': token},
+                     timeout=settings.WEB_ATTACHMENT_TIMEOUT)
 
     if r.status_code == 200:
         return
@@ -261,6 +309,24 @@ def test_key():
     else:
         raise AttachmentError("Attachment key test failed.")
 
+@login_maybe_required
+@require_http_methods(['GET', 'HEAD'])
+@never_cache
+def health(request):
+    if server_urls is None:
+        if settings.WEB_ATTACHMENT_URL in (None, ''):
+            return HttpResponse(status=404)
+        if not retry_initialization():
+            return HttpResponse(status=503)
+
+    try:
+        test_key()
+    except (AttachmentError, requests.RequestException) as error:
+        logger.error('Health check failed: %s', str(error))
+        return HttpResponse(status=503)
+
+    return HttpResponse(status=204)
+
 @openapi(schema={
     "get": {
         "parameters": [

specifyweb/frontend/js_src/lib/components/Attachments/Cell.tsx

@@ -22,7 +22,7 @@ import { softFail } from '../Errors/Crash';
 import { Dialog } from '../Molecules/Dialog';
 import { TableIcon } from '../Molecules/TableIcon';
 import { hasTablePermission } from '../Permissions/helpers';
-import { fetchOriginalUrl } from './attachments';
+import { fetchOriginalUrl, useAttachmentServerStatus } from './attachments';
 import { AttachmentPreview } from './Preview';
 import { getAttachmentRelationship, tablesWithAttachments } from './utils';
 
@@ -39,10 +39,17 @@ export function AttachmentCell({
     | ((table: SpecifyTable, recordId: number) => void)
     | undefined;
 }): JSX.Element {
+  const attachmentServerStatus = useAttachmentServerStatus();
   const table = f.maybe(attachment.tableID ?? undefined, getAttachmentTable);
 
   const [originalUrl] = useAsyncState(
-    React.useCallback(async () => fetchOriginalUrl(attachment), [attachment]),
+    React.useCallback(
+      async () =>
+        attachmentServerStatus !== 'unavailable'
+          ? fetchOriginalUrl(attachment)
+          : undefined,
+      [attachment, attachmentServerStatus]
+    ),
     false
   );
 

specifyweb/frontend/js_src/lib/components/Attachments/Preview.tsx

@@ -4,7 +4,11 @@ import { useAsyncState } from '../../hooks/useAsyncState';
 import type { SerializedResource } from '../DataModel/helperTypes';
 import type { Attachment } from '../DataModel/types';
 import type { AttachmentThumbnail } from './attachments';
-import { fetchThumbnail } from './attachments';
+import {
+  fetchThumbnail,
+  reportAttachmentServerFailure,
+  useAttachmentServerStatus,
+} from './attachments';
 
 export function AttachmentPreview({
   attachment,
@@ -13,8 +17,15 @@ export function AttachmentPreview({
   readonly attachment: SerializedResource<Attachment>;
   readonly onOpen: () => void;
 }): JSX.Element {
+  const attachmentServerStatus = useAttachmentServerStatus();
   const [thumbnail] = useAsyncState(
-    React.useCallback(async () => fetchThumbnail(attachment), [attachment]),
+    React.useCallback(
+      async () =>
+        attachmentServerStatus !== 'unavailable'
+          ? fetchThumbnail(attachment)
+          : undefined,
+      [attachment, attachmentServerStatus]
+    ),
     false
   );
 
@@ -56,6 +67,11 @@ export function Thumbnail({
         width: `${thumbnail.width}px`,
         height: `${thumbnail.height}px`,
       }}
+      onError={
+        thumbnail.isServerBacked === true
+          ? reportAttachmentServerFailure
+          : undefined
+      }
     />
   );
 }