fix(lora): only drop adapter from _merged_adapters when unfused from all components

src/diffusers/loaders/lora_base.py

@@ -669,11 +669,21 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs):
                 if issubclass(model.__class__, (ModelMixin, PreTrainedModel)):
                     for module in model.modules():
                         if isinstance(module, BaseTunerLayer):
-                            for adapter in set(module.merged_adapters):
-                                if adapter and adapter in self._merged_adapters:
-                                    self._merged_adapters = self._merged_adapters - {adapter}
                             module.unmerge()
 
+        # Only remove an adapter from _merged_adapters once it is no longer
+        # physically merged in any remaining loadable component. Removing it
+        # on the first unfused component would desync the set when the adapter
+        # is still fused into other components.
+        remaining_merged: set[str] = set()
+        for component_name in self._lora_loadable_modules:
+            component_model = getattr(self, component_name, None)
+            if component_model is not None and issubclass(component_model.__class__, (ModelMixin, PreTrainedModel)):
+                for module in component_model.modules():
+                    if isinstance(module, BaseTunerLayer):
+                        remaining_merged.update(module.merged_adapters)
+        self._merged_adapters = self._merged_adapters & remaining_merged
+
     def set_adapters(
         self,
         adapter_names: list[str] | str,

tests/lora/test_lora_loader_utils.py

@@ -100,3 +100,53 @@ def test_local_directory_with_multiple_files_warns_and_uses_first(tmp_path, monk
 
     assert weight_name == first_path.name
     assert "contains more than one weights file" in caplog.text
+
+
+def test_unfuse_lora_partial_components_keeps_merged_adapters_in_sync():
+    """Regression test for gh-14214.
+
+    Unfusing only a subset of components must keep _merged_adapters in sync
+    with the adapters still physically fused in the remaining components.
+    """
+    import torch.nn as nn
+    from peft import LoraConfig
+    from peft.tuners.tuners_utils import BaseTunerLayer
+    from diffusers.loaders.lora_base import LoraBaseMixin
+    from diffusers.loaders.peft import PeftAdapterMixin
+    from diffusers.models.modeling_utils import ModelMixin
+    from diffusers.configuration_utils import ConfigMixin
+
+    class TinyModel(ModelMixin, ConfigMixin, PeftAdapterMixin):
+        config_name = "config.json"
+        def __init__(self):
+            super().__init__()
+            self.linear = nn.Linear(8, 8)
+
+    class FakePipeline(LoraBaseMixin):
+        _lora_loadable_modules = ["unet", "text_encoder"]
+        def __init__(self, unet, text_encoder):
+            self._merged_adapters = set()
+            self.unet, self.text_encoder = unet, text_encoder
+
+    unet = TinyModel()
+    text_encoder = TinyModel()
+    config = LoraConfig(r=4, lora_alpha=4, target_modules=["linear"], init_lora_weights=False)
+    unet.add_adapter(config, adapter_name="adapter")
+    text_encoder.add_adapter(config, adapter_name="adapter")
+
+    pipe = FakePipeline(unet, text_encoder)
+    pipe.fuse_lora(components=["unet", "text_encoder"], adapter_names=["adapter"])
+    assert pipe.num_fused_loras == 1
+
+    pipe.unfuse_lora(components=["text_encoder"])
+    assert "adapter" in pipe.fused_loras, "adapter should remain tracked while unet is still fused"
+    assert pipe.num_fused_loras == 1
+
+    unet_still_merged = any(
+        isinstance(m, BaseTunerLayer) and len(m.merged_adapters) > 0
+        for m in unet.modules()
+    )
+    assert unet_still_merged, "unet should still be physically merged at the PEFT level"
+
+    pipe.unfuse_lora(components=["unet"])
+    assert pipe.num_fused_loras == 0

tests/lora/utils.py

@@ -2548,3 +2548,4 @@ def test_lora_group_offloading_delete_adapters(self):
             # Clean up the hooks to prevent state leak
             if hasattr(denoiser, "_diffusers_hook"):
                 denoiser._diffusers_hook.remove_hook(_GROUP_OFFLOADING, recurse=True)
+