add_weighted_adapter with rank_pattern produces adapters that cannot be reloaded (linear, ties, dare, magnitude_prune)

import tempfile

import torch
from torch import nn

from peft import LoraConfig, PeftModel, get_peft_model


class MLP(nn.Module):
    def __init__(self):
        super().__init__()
        self.lin0 = nn.Linear(20, 20)
        self.lin1 = nn.Linear(20, 20)

    def forward(self, x):
        return self.lin1(self.lin0(x))


def make_config(rank_pattern):
    return LoraConfig(target_modules=["lin0", "lin1"], r=8, rank_pattern=rank_pattern, init_lora_weights=False)


# 1. Both adapters use the same rank_pattern: combining works, but the result cannot be reloaded
for combination_type in ["linear", "ties", "dare_linear", "dare_ties", "magnitude_prune", "svd", "cat"]:
    torch.manual_seed(0)
    model = get_peft_model(MLP(), make_config({"lin1": 4}), adapter_name="a")
    model.add_adapter("b", make_config({"lin1": 4}))
    kwargs = {} if combination_type in ("linear", "svd", "cat") else {"density": 0.5}
    model.add_weighted_adapter(["a", "b"], [0.5, 0.5], "merged", combination_type=combination_type, **kwargs)
    with tempfile.TemporaryDirectory() as tmp_dir:
        model.save_pretrained(tmp_dir, selected_adapters=["merged"])
        try:
            PeftModel.from_pretrained(MLP(), f"{tmp_dir}/merged")
            print(f"{combination_type}: reload OK")
        except RuntimeError as e:
            print(f"{combination_type}: reload FAILED: {str(e).splitlines()[1].strip()}")

# 2. Same maximum rank, but different ranks for the same module: combining fails immediately
torch.manual_seed(0)
model = get_peft_model(MLP(), make_config({"lin1": 4}), adapter_name="a")
model.add_adapter("b", make_config({}))
try:
    model.add_weighted_adapter(["a", "b"], [0.5, 0.5], "merged", combination_type="linear")
except RuntimeError as e:
    print(f"linear with different per-module ranks: FAILED: {e}")

linear: reload FAILED: size mismatch for base_model.model.lin1.lora_A.default.weight: copying a param with shape torch.Size([4, 20]) from checkpoint, the shape in current model is torch.Size([8, 20]).
ties: reload FAILED: size mismatch for base_model.model.lin1.lora_A.default.weight: copying a param with shape torch.Size([4, 20]) from checkpoint, the shape in current model is torch.Size([8, 20]).
dare_linear: reload FAILED: size mismatch for base_model.model.lin1.lora_A.default.weight: copying a param with shape torch.Size([4, 20]) from checkpoint, the shape in current model is torch.Size([8, 20]).
dare_ties: reload FAILED: size mismatch for base_model.model.lin1.lora_A.default.weight: copying a param with shape torch.Size([4, 20]) from checkpoint, the shape in current model is torch.Size([8, 20]).
magnitude_prune: reload FAILED: size mismatch for base_model.model.lin1.lora_A.default.weight: copying a param with shape torch.Size([4, 20]) from checkpoint, the shape in current model is torch.Size([8, 20]).
svd: reload OK
cat: reload OK
linear with different per-module ranks: FAILED: stack expects each tensor to be equal size, but got [4, 20] at entry 0 and [8, 20] at entry 1