Panic on settings update after a document update adds a nested object (`Expected field … in the fields IDs map`)

panicked at crates/milli/src/update/new/extract/searchable/extract_word_docids.rs:609:37: Expected field `extra.x` in the fields IDs map
panicked at crates/milli/src/update/new/indexer/mod.rs:371:37: called `Result::unwrap()` on an `Err` value: Any { .. }

{"message": "An unexpected crash occurred when processing the task: called `Result::unwrap()` on an `Err` value: Any { .. }", "code": "internal", "type": "internal", "link": "https://docs.meilisearch.com/errors#internal"}

M=http://localhost:7700; J='Content-Type: application/json'

curl -X PATCH $M/indexes/repro/settings -H "$J" --data '{"searchableAttributes":["title"]}'; sleep 1
curl -X POST $M/indexes/repro/documents -H "$J" --data '[{"id":1,"title":"hello"}]'; sleep 1
curl -X POST $M/indexes/repro/documents -H "$J" --data '[{"id":1,"title":"hello","extra":{"x":1}}]'; sleep 1
curl -X PATCH $M/indexes/repro/settings -H "$J" --data '{"filterableAttributes":["title"]}'; sleep 1
curl "$M/tasks?indexUids=repro&types=settingsUpdate"

unzip repro_6210-v1.53.1.snapshot.zip
meilisearch --import-snapshot data.ms.snapshot
curl -X PATCH localhost:7700/indexes/repro_6210/settings -H 'Content-Type: application/json' --data '{"filterableAttributes":["title"]}'
# the task fails with the panic above

#!/usr/bin/env python3
"""Reproduction for https://github.com/meilisearch/meilisearch/issues/6210 (Python 3 stdlib only).

With an explicit `searchableAttributes` list, updating an EXISTING document so that it gains a
nested object (or an array of objects) leaves the nested paths out of the fields IDs map. The next
settings update that re-extracts documents (new filterable or sortable attributes) panics at
extract_word_docids.rs:609 ("Expected field `extra.x` in the fields IDs map"), and so does every
later one on that index.

    docker run -d --rm -p 7700:7700 getmeili/meilisearch:v1.53.2
    MEILI_URL=http://localhost:7700 python3 repro.py            # add MEILI_KEY=<master key> if one is set
    python3 repro.py --arm-only   # leave index `repro_6210` right before the failing settings update

Exit code: 1 when the bug reproduces, 0 when every scenario behaves as expected.
Regression in v1.43.0 (still in v1.53.2); MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_SETTINGS=true avoids it.
"""
import json
import os
import sys
import time
import urllib.request

BASE = os.environ.get("MEILI_URL", "http://localhost:7700").rstrip("/")
KEY = os.environ.get("MEILI_KEY", "")
RUN = str(int(time.time() * 1000))


def call(method, path, body=None):
    headers = {"Content-Type": "application/json"}
    if KEY:
        headers["Authorization"] = "Bearer " + KEY
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(BASE + path, method=method, data=data, headers=headers)
    with urllib.request.urlopen(req, timeout=60) as resp:
        return json.load(resp)


def run_task(method, path, body):
    task = call(method, path, body)
    while True:
        t = call("GET", f"/tasks/{task['taskUid']}")
        if t["status"] in ("succeeded", "failed", "canceled"):
            return t
        time.sleep(0.05)


def step(uid, method, path, body, must_succeed=True):
    t = run_task(method, f"/indexes/{uid}{path}", body)
    if must_succeed and t["status"] != "succeeded":
        raise RuntimeError(f"{uid}: setup step {method} {path} {t['status']}: {t.get('error')}")
    return t


def new_index(name, searchable=("title",)):
    uid = f"{name}_{RUN}"
    t = run_task("POST", "/indexes", {"uid": uid, "primaryKey": "id"})
    if t["status"] != "succeeded":
        raise RuntimeError(f"create {uid}: {t.get('error')}")
    if searchable is not None:
        step(uid, "PATCH", "/settings", {"searchableAttributes": list(searchable)})
    return uid


FILTERABLE = {"filterableAttributes": ["title"]}
SORTABLE = {"sortableAttributes": ["title"]}
DOC = {"id": 1, "title": "hello"}
DOC_WITH_OBJECT = {"id": 1, "title": "hello", "extra": {"x": 1}}

results = []


def check(name, task, expected="succeeded"):
    error = (task.get("error") or {}).get("message", "")
    ok = task["status"] == expected
    results.append((name, ok))
    print(f"{'ok  ' if ok else 'BUG '} {name}: settings update {task['status']}" + (f" ({error})" if error else ""))


def main():
    version = call("GET", "/version")
    print(f"Meilisearch {version.get('pkgVersion')} ({version.get('commitSha', '')[:7]}) at {BASE}\n")

    if "--arm-only" in sys.argv:
        uid = "repro_6210"
        run_task("POST", "/indexes", {"uid": uid, "primaryKey": "id"})
        step(uid, "PATCH", "/settings", {"searchableAttributes": ["title"]})
        step(uid, "POST", "/documents", [DOC])
        step(uid, "POST", "/documents", [DOC_WITH_OBJECT])
        print(f"Index `{uid}` is armed. This settings update now fails:\n"
              f"  curl -X PATCH '{BASE}/indexes/{uid}/settings' -H 'Content-Type: application/json' "
              f"--data '{json.dumps(FILTERABLE)}'")
        return 0

    uid = new_index("bug")
    step(uid, "POST", "/documents", [DOC])
    step(uid, "POST", "/documents", [DOC_WITH_OBJECT])  # update an existing document: it gains a nested object
    check("update adds a nested object, then new filterable attribute", step(uid, "PATCH", "/settings", FILTERABLE, False))
    check("  ...and every later settings update on that index (new sortable attribute)", step(uid, "PATCH", "/settings", SORTABLE, False))
    step(uid, "POST", "/documents", [DOC])  # workaround: replace the document without the object
    check("  ...until the document is replaced without the object", step(uid, "PATCH", "/settings", SORTABLE, False))

    uid = new_index("partial")
    step(uid, "POST", "/documents", [DOC])
    step(uid, "PUT", "/documents", [{"id": 1, "lines": [{"a": 1}]}])  # partial update adding an array of objects
    check("PUT partial update adds an array of objects, then new filterable attribute", step(uid, "PATCH", "/settings", FILTERABLE, False))

    print("\ncontrols (expected to succeed):")
    uid = new_index("control_insert")
    step(uid, "POST", "/documents", [DOC])
    step(uid, "POST", "/documents", [dict(DOC_WITH_OBJECT, id=2)])  # same object, but on a NEW document
    check("same object inserted with a new document id", step(uid, "PATCH", "/settings", FILTERABLE, False))

    uid = new_index("control_first_batch")
    step(uid, "POST", "/documents", [DOC_WITH_OBJECT])
    check("object already present in the first batch", step(uid, "PATCH", "/settings", FILTERABLE, False))

    uid = new_index("control_default_searchable", searchable=None)
    step(uid, "POST", "/documents", [DOC])
    step(uid, "POST", "/documents", [DOC_WITH_OBJECT])
    check("default searchableAttributes [\"*\"]", step(uid, "PATCH", "/settings", FILTERABLE, False))

    uid = new_index("control_scalar")
    step(uid, "POST", "/documents", [DOC])
    step(uid, "POST", "/documents", [dict(DOC, extra="scalar")])
    check("update adds a top-level scalar field instead", step(uid, "PATCH", "/settings", FILTERABLE, False))

    for name in ("bug", "partial", "control_insert", "control_first_batch", "control_default_searchable", "control_scalar"):
        call("DELETE", f"/indexes/{name}_{RUN}")

    reproduced = not all(ok for _, ok in results)
    print("\nRESULT:", "bug reproduced" if reproduced else "no bug: every settings update succeeded")
    return 1 if reproduced else 0


if __name__ == "__main__":
    sys.exit(main())