LlamaCppTokenizer 会丢弃每个片长超过 32 字节的令牌: llama_token_to_piece 会报告缓冲区过小并返回负值,因此 n > size 重试是无法实现的

repro_llamacpp_longpiece.py

"""Offline repro: fake llama_cpp module that mimics LLaMA.cpp's real return contract.""" import ctypes from unittest.mock import MagicMock, patch from outlines.models.llamacpp import LlamaCppTokenizer LONG = "a" * 40 # 40-byte piece, > the hardcoded 32-byte buffer PIECES = {0: "hello", 1: " world", 2: "", 3: LONG, 4: "bye"} calls = []

def fake_llama_token_to_piece(vocab, token, buf, buf_size, lstrip, special): # verbatim semantics of LLaMA.cpp llama_vocab::impl::token_to_piece # (src/LLaMA-vocab.cpp): "if (length < (int32_t) size) return -(int32_t) size;" data = PIECES[token].encode() calls.append((token, buf_size, len(data))) if buf_size < len(data): return -len(data) ctypes.memmove(buf, data, len(data)) return len(data)

def make_tokenizer(): model = MagicMock() model.token_eos.return_value = 2 model.n_vocab.return_value = len(PIECES) del model.tokenizer_ # llama_cpp default: tokenizer_ has no hf_tokenizer with patch.dict("sys.modules", {"llama_cpp": …