Mixing modalities in `encode()` doesn't work for CLIP

I'm trying to get CLIP to work with the new multimodal setup, and it seems that it misinterprets lists containing multiple modalities as messages:

from datasets import load_dataset
from sentence_transformers import SentenceTransformer

ds = load_dataset("nbadrinath/ikea_dataset_5.0", split="train[:1000]")
images = list(ds["image"])
texts = list(ds["desc"])
print(images[0], texts[0])
# <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=512x512 at 0x7818178354C0>  Product: Small Outdoor Balcony Set...

inputs = images + texts
encoder = SentenceTransformer("clip-ViT-B-32")
encoder.encode(inputs)

ValueError                                Traceback (most recent call last)
Cell In[61], line 1
----> 1 encoder.encode(inputs)

File ~/.conda/envs/turftopic/lib/python3.12/site-packages/torch/utils/_contextlib.py:120, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    117 @functools.wraps(func)
    118 def decorate_context(*args, **kwargs):
    119     with ctx_factory():
--> 120         return func(*args, **kwargs)

File ~/.conda/envs/turftopic/lib/python3.12/site-packages/sentence_transformers/util/decorators.py:41, in deprecated_kwargs.<locals>.decorator.<locals>.wrappe
r(*args, **kwargs)
     39         else:
     40             kwargs.pop(old_name)
---> 41 return func(*args, **kwargs)

File ~/.conda/envs/turftopic/lib/python3.12/site-packages/sentence_transformers/sentence_transformer/model.py:644, in SentenceTransformer.encode(self, inputs,
 prompt_name, prompt, batch_size, show_progress_bar, output_value, precision, convert_to_numpy, convert_to_tensor, device, normalize_embeddings, truncate_dim,
 pool, chunk_size, **kwargs)
    642 for start_index in trange(0, len(inputs_sorted), batch_size, desc="Batches", disable=not show_progress_bar):
    643     inputs_batch = inputs_sorted[start_index : start_index + batch_size]
--> 644     features = self.preprocess(inputs_batch, prompt=prompt, **kwargs)
    646     if is_hpu:
    647         features = self._pad_features_for_hpu(features)

File ~/.conda/envs/turftopic/lib/python3.12/site-packages/sentence_transformers/base/model.py:532, in BaseModel.preprocess(self, inputs, prompt, **kwargs)
    527     if isinstance(modality, tuple) and all(part in self.modalities for part in modality):
    528         message += (
    529             f"\nThis model supports {' and '.join(modality)} individually, "
    530             "but not in the same input. Please process each modality separately."
    531         )
--> 532     raise ValueError(message)
    534 # Backwards compatibility: fall back to preprocess/tokenize without prompt if the
    535 # input module doesn't accept it. Only the main path (preprocess with prompt) will
    536 # be supported in the future.
    537 try:

ValueError: Modality 'message' is not supported by this SentenceTransformer model. Supported modalities: text, image