From 1d0d437433777339bcf6d13d1a20eda3fa93d84d Mon Sep 17 00:00:00 2001 From: christopher5106 Date: Wed, 23 Sep 2026 02:36:44 +0200 Subject: [PATCH] [LoRA] strip the stale text_model. prefix when loading kohya CLIP text-encoder LoRAs on transformers >= 5.6 transformers 5.6 flattened CLIPTextModel, so named_modules() no longer yields `text_model.*` names while kohya-converted FLUX LoRAs still carry that prefix. The rank dict came out empty and _create_lora_config raised IndexError (huggingface/diffusers#13984). Upstream closed the fix (#14029) wanting a general solution; this carries it on the scenario branch until one lands. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 4148971da123019e949bd34bc881b04745832c15) --- src/diffusers/loaders/lora_base.py | 8 ++++++++ tests/pipelines/flux/test_pipeline_flux.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index 0009f04df9e2..0b811e90957b 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -383,6 +383,14 @@ def _load_lora_into_text_encoder( # convert state dict state_dict = convert_state_dict_to_peft(state_dict) + # Under transformers>=5, CLIPTextModel was flattened and no longer has the + # `text_model.` wrapper module, so named_modules() returns unprefixed names + # like "encoder.layers.0.self_attn.q_proj". Kohya-sourced LoRA state dict + # keys still carry the "text_model." prefix from the old module layout. + # Strip it so that rank-key matching works regardless of the transformers version. + if not hasattr(text_encoder, "text_model"): + state_dict = {k.removeprefix("text_model."): v for k, v in state_dict.items()} + for name, _ in text_encoder.named_modules(): if name.endswith((".q_proj", ".k_proj", ".v_proj", ".out_proj", ".fc1", ".fc2")): rank_key = f"{name}.lora_B.weight" diff --git a/tests/pipelines/flux/test_pipeline_flux.py b/tests/pipelines/flux/test_pipeline_flux.py index 70cd88a48c9b..16cb3a942be9 100644 --- a/tests/pipelines/flux/test_pipeline_flux.py +++ b/tests/pipelines/flux/test_pipeline_flux.py @@ -242,6 +242,24 @@ class TestFluxPipelineMagCache(FluxPipelineTesterConfig, MagCacheTesterMixin): class TestFluxPipelineLoRA(FluxPipelineTesterConfig, LoraTesterMixin): """LoRA tests for the Flux pipeline.""" + def test_kohya_clip_text_encoder_lora_under_flattened_clip(self): + # Regression for huggingface/diffusers#13984: transformers >= 5.6 removed the + # `text_model.` wrapper from CLIPTextModel, but kohya-converted state dicts still + # carry that prefix, which left the rank dict empty and raised IndexError. + pipe = self.get_pipeline().to(torch_device) + if hasattr(pipe.text_encoder, "text_model"): + pytest.skip("transformers still exposes the text_model wrapper") + + state_dict = {} + for name, module in pipe.text_encoder.named_modules(): + if name.endswith((".q_proj", ".k_proj", ".v_proj", ".out_proj", ".fc1", ".fc2")): + state_dict[f"text_encoder.text_model.{name}.lora_A.weight"] = torch.randn(2, module.in_features) + state_dict[f"text_encoder.text_model.{name}.lora_B.weight"] = torch.randn(module.out_features, 2) + + pipe.load_lora_weights(state_dict) + assert check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + pipe.unload_lora_weights() + def test_with_alpha_in_state_dict(self, tmp_path): pipe = self.get_pipeline().to(torch_device) self.add_adapters_to_pipeline(pipe, components=["transformer"])