check: detect missing packs referenced by the index (#9898)

src/borg/repository.py

@@ -986,6 +986,11 @@ def check(self, repair=False, max_duration=0, max_age=0):
         it. The record clears at the check that finds the pack intact again or gone (removed by
         compact, or salvaged and dropped by repair; refs #8572); prune() does this from packs/.
 
+        It also reports missing packs (refs #9898): pack ids the chunk index references but that are
+        absent from packs/. The chunk index is loaded and its referenced pack ids are compared with the
+        packs present in the store. A corrupt or invalid index is rebuilt from the packs on next use, so
+        its references are not checked here.
+
         max_age (seconds, 0 = verify every pack): skip packs whose intact record is younger than
         max_age, accepting a future timestamp up to MAX_CLOCK_SKEW (clock skew). Results are recorded
         regardless of max_age.
@@ -1027,15 +1032,17 @@ def store_list(namespace):
         t_last_checkpoint = t_start
         index_files = index_errors = 0
         pack_files = pack_errors = pack_skipped = 0
+        missing_pack_ids = []  # packs referenced by the index but absent from packs/ (refs #9898)
         # index and packs get separate progress indicators, each running from 0% to 100%.
         # the index is checked first and in full, on partial checks too: it is small, and index errors
         # stop the pack check below.
         index_infos = store_list("index")
         # an invalid chunk index means an interrupted fragment deletion; it will be rebuilt on next
         # use, so warn rather than verify the leftover fragments.
-        from .cache import chunkindex_is_invalid
+        from .cache import chunkindex_is_invalid, build_chunkindex_from_repo
 
-        if chunkindex_is_invalid(self):
+        index_invalid = chunkindex_is_invalid(self)
+        if index_invalid:
             logger.warning("chunk index is invalid (interrupted operation); it will be rebuilt on next use.")
         index_pi = ProgressIndicatorPercent(total=len(index_infos), msg="Checking index %3.0f%%", msgid="check.index")
         for info in index_infos:
@@ -1101,19 +1108,40 @@ def recorded_ts(info):
                 if pack_infos:
                     pack_pi.show(current=len(pack_infos))  # finish at 100%
                 logger.info("Finished checking packs.")
-            tracker.prune({hex_to_bin(info.name) for info in pack_infos})
+            present_pack_ids = {hex_to_bin(info.name) for info in pack_infos}
+            tracker.prune(present_pack_ids)
             pack_pi.finish()
+            # report packs the index references but that are absent from packs/ (refs #9898). an
+            # invalid index is rebuilt from the packs on next use, so its references are not checked.
+            if not index_invalid:
+                chunks = build_chunkindex_from_repo(self)
+                try:
+                    referenced_pack_ids = {
+                        entry.pack_id
+                        for _, entry in chunks.iteritems()
+                        if not (entry.flags & ChunkIndex.F_PENDING)  # pending: pack_id not resolved yet
+                    }
+                finally:
+                    chunks.clear()
+                missing_pack_ids = sorted(referenced_pack_ids - present_pack_ids)
         else:
             # TODO: --repair will rebuild the index from the packs here instead of stopping (refs #8572).
             logger.error("Repository index is corrupted and must be repaired; skipping the pack check.")
-        objs_errors = index_errors + pack_errors
+        objs_errors = index_errors + pack_errors + len(missing_pack_ids)
         summary = (
             f"Checked {index_files} index files ({index_errors} errors) "
             f"and {pack_files} packs ({pack_errors} errors)."
         )
+        if missing_pack_ids:
+            summary += f" {len(missing_pack_ids)} pack(s) referenced by the index are missing."
         if pack_skipped:
             summary += f" Reused {pack_skipped} recent pack check result(s)."
         logger.info(summary)
+        if missing_pack_ids:
+            # one id per line (the list can be long).
+            logger.error(f"Found {len(missing_pack_ids)} missing pack(s) referenced by the index:")
+            for pack_id in missing_pack_ids:
+                logger.error(f"Missing pack: {bin_to_hex(pack_id)}")
         # corrupt_ids() is every pack recorded corrupt, including from earlier runs. with a corrupt
         # index the packs were not scanned, so report nothing.
         corrupt_ids = tracker.corrupt_ids() if index_errors == 0 else []

src/borg/testsuite/repository_test.py

@@ -1024,6 +1024,15 @@ def boom(*args, **kwargs):
         assert H(1) not in repository._chunks
 
 
+def _serialized_chunkindex(chunks=None):
+    # Serialize a ChunkIndex to bytes, as stored under index/<sha256(content)>. check() parses index
+    # fragments, so a fragment must be a real ChunkIndex serialization.
+    chunks = chunks if chunks is not None else ChunkIndex()
+    with io.BytesIO() as f:
+        chunks.write(f)
+        return f.getvalue()
+
+
 def test_check_detects_corruption_in_later_object(tmp_path):
     # Corruption anywhere in a multi-object pack must be caught, not just in the first object: the pack
     # is named by sha256(content), so flipping any byte makes its stored hash differ from its name.
@@ -1044,7 +1053,7 @@ def test_check_detects_corruption_in_later_object(tmp_path):
 
 def test_check_detects_index_corruption(tmp_path):
     # index/ objects are named by sha256(content) like packs, so check verifies them the same way.
-    content = b"pretend this is a serialized chunk index"
+    content = _serialized_chunkindex()
     index_name = "index/" + bin_to_hex(sha256(content).digest())
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         repository.store_store(index_name, content)
@@ -1094,6 +1103,43 @@ def test_check_intact_multi_object_pack_passes(tmp_path):
         assert repository.check(repair=False) is True
 
 
+def test_check_detects_missing_pack_referenced_by_index(tmp_path, caplog):
+    # check must report a pack the chunk index references but that is absent from packs/ (refs #9898).
+    # put+flush+close persists the index; then delete the pack, keeping its index entry.
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        for x in range(3):
+            repository.put(H(x), fchunk(b"DATA-%02d" % x, chunk_id=H(x)))
+        repository.flush()  # flush before close persists the index
+    with Repository(location, exclusive=True) as repository:
+        assert repository.check(repair=False) is True  # index and pack both present
+        pack_id = repository.chunks[H(0)].pack_id
+        repository.store_delete("packs/" + bin_to_hex(pack_id))  # pack gone, index entry kept
+        with caplog.at_level(logging.ERROR):
+            assert repository.check(repair=False) is False
+        assert f"Missing pack: {bin_to_hex(pack_id)}" in caplog.text
+
+
+def test_check_missing_pack_detection_skipped_when_index_invalid(tmp_path, caplog):
+    # an invalid index is rebuilt from the packs on next use, so check does not report missing packs
+    # from it (refs #9898); it only warns about the invalid index.
+    from ..cache import write_chunkindex_invalid
+
+    location = os.fspath(tmp_path / "repo")
+    with Repository(location, exclusive=True, create=True) as repository:
+        for x in range(3):
+            repository.put(H(x), fchunk(b"DATA-%02d" % x, chunk_id=H(x)))
+        repository.flush()
+    with Repository(location, exclusive=True) as repository:
+        pack_id = repository.chunks[H(0)].pack_id
+        repository.store_delete("packs/" + bin_to_hex(pack_id))  # pack gone
+        write_chunkindex_invalid(repository)  # mark the index invalid
+        with caplog.at_level(logging.WARNING):
+            assert repository.check(repair=False) is True
+        assert "Missing pack" not in caplog.text
+        assert "chunk index is invalid" in caplog.text
+
+
 def test_check_checked_packs_roundtrip(tmp_path):
     # the set survives a store/load round-trip; a rotted blob loads as empty.
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
@@ -1574,7 +1620,7 @@ def finish(self, *args, **kwargs):
     monkeypatch.setattr("borg.repository.ProgressIndicatorPercent", FakePI)
     pack = fchunk(b"A", chunk_id=H(1))
     pack_name = "packs/" + bin_to_hex(sha256(pack).digest())
-    index_content = b"serialized chunk index"
+    index_content = _serialized_chunkindex()
     index_name = "index/" + bin_to_hex(sha256(index_content).digest())
     with Repository(str(tmp_path / "repo"), exclusive=True, create=True) as repository:
         repository.store_store(pack_name, pack)