refactor(backends/chroma): coerce None metadatas at the boundary (cherry-pick of MemPalace#1094)

Fork main was carrying the per-site `meta = meta or {}` guards from MemPalace#999
in eight read paths but didn't have the boundary coercion that closes
the issue once for all callers.

Cherry-picked MemPalace#1094 (open upstream, jp-authored) so the typed
QueryResult/GetResult contract — both declare `metadatas: list[dict]`,
never `list[Optional[dict]]` — is honored at the chromadb adapter
boundary. Three same-family fork-ahead PRs we filed since (MemPalace#1198
_tokenize None-doc, MemPalace#1201 palace_graph None metadata, MemPalace#1199 review)
all pointed at gaps that would have been impossible if this pattern
had been in place. Future None-metadata bugs of the same shape are
now catchable at one site instead of N.

Per-site guards are kept as belt-and-suspenders for paths that might
bypass the typed wrappers (e.g. legacy `import chromadb` direct calls).
A future cleanup PR can remove them once we're confident no caller
reaches chromadb directly.

6 new tests in test_backends.py (MemPalace#1094 originals); composes cleanly with
fork main's _segment_appears_healthy + _quarantined_paths from earlier
today. Full suite 1383/1383.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: jphein

mempalace/backends/chroma.py

@@ -480,10 +480,21 @@ def query(
         def _none_list_to_empty(outer):
             return [(inner or []) for inner in outer]
 
+        # Coerce individual None metadata dicts to {} at the backend
+        # boundary. ChromaDB 1.5.x can return None inside metadatas[i][j]
+        # even when the write stored a dict — #999, #1013, and other
+        # fixes added per-site `meta = meta or {}` guards scattered
+        # across searcher.py, layers.py, and miner.py to tolerate this.
+        # Handling the coercion once here means every downstream caller
+        # receives a guaranteed list[dict], matching the type contract
+        # QueryResult.metadatas declares.
+        def _coerce_none_metas(outer):
+            return [[(m if m is not None else {}) for m in (inner or [])] for inner in outer]
+
         return QueryResult(
             ids=_none_list_to_empty(ids),
             documents=_none_list_to_empty(documents),
-            metadatas=_none_list_to_empty(metadatas),
+            metadatas=_coerce_none_metas(metadatas),
             distances=_none_list_to_empty(distances),
             embeddings=(
                 [list(inner) for inner in embeddings_raw]
@@ -538,6 +549,16 @@ def get(
         if spec.metadatas and len(out_metas) < len(out_ids):
             out_metas = out_metas + [{}] * (len(out_ids) - len(out_metas))
 
+        # Coerce any individual None metadata dict to {} at the backend
+        # boundary. Same rationale as the parallel path in query():
+        # chromadb 1.5.x can return None inside the metadatas list even
+        # when writes stored a dict, and per-site `meta = meta or {}`
+        # guards across #999 / #1013 compensated. Doing this once here
+        # means callers receive a guaranteed list[dict], matching the
+        # type contract GetResult.metadatas declares.
+        if spec.metadatas:
+            out_metas = [(m if m is not None else {}) for m in out_metas]
+
         return GetResult(
             ids=out_ids,
             documents=out_docs,

tests/test_backends.py

@@ -129,6 +129,107 @@ def test_chroma_collection_rejects_unknown_where_operator():
         collection.query(query_texts=["q"], where={"$regex": "foo"})
 
 
+def test_chroma_collection_query_coerces_none_metadatas_to_empty_dict():
+    """ChromaDB 1.5.x can return None inside metadatas[i][j] even when
+    the write stored a dict. The backend boundary coerces each None to
+    {} so downstream callers receive a guaranteed list[dict]. Prevents
+    `AttributeError: 'NoneType' object has no attribute 'get'` at every
+    read site (searcher.py, layers.py, miner.status, etc.)."""
+    raw = {
+        "ids": [["a", "b", "c"]],
+        "documents": [["da", "db", "dc"]],
+        "metadatas": [[{"wing": "w1"}, None, {"wing": "w3"}]],
+        "distances": [[0.1, 0.2, 0.3]],
+    }
+    collection = ChromaCollection(_FakeCollection(query_response=raw))
+
+    result = collection.query(query_texts=["q"])
+
+    assert result.metadatas == [[{"wing": "w1"}, {}, {"wing": "w3"}]]
+    # Ids/docs/distances unaffected — their inner None handling is already
+    # covered by the outer-list coercion.
+    assert result.ids == [["a", "b", "c"]]
+    assert result.distances == [[0.1, 0.2, 0.3]]
+
+
+def test_chroma_collection_query_coerces_all_none_inner_metadatas():
+    """Degenerate case: every metadata dict in the batch is None. Each
+    position still yields {} rather than leaving Nones for callers."""
+    raw = {
+        "ids": [["a", "b"]],
+        "documents": [["da", "db"]],
+        "metadatas": [[None, None]],
+        "distances": [[0.1, 0.2]],
+    }
+    collection = ChromaCollection(_FakeCollection(query_response=raw))
+
+    result = collection.query(query_texts=["q"])
+
+    assert result.metadatas == [[{}, {}]]
+
+
+def test_chroma_collection_query_coerces_none_across_multiple_queries():
+    """Two queries in one call, None dicts mixed through both."""
+    raw = {
+        "ids": [["a", "b"], ["c", "d"]],
+        "documents": [["da", "db"], ["dc", "dd"]],
+        "metadatas": [[{"wing": "w1"}, None], [None, {"wing": "w4"}]],
+        "distances": [[0.1, 0.2], [0.3, 0.4]],
+    }
+    collection = ChromaCollection(_FakeCollection(query_response=raw))
+
+    result = collection.query(query_texts=["q1", "q2"])
+
+    assert result.metadatas == [[{"wing": "w1"}, {}], [{}, {"wing": "w4"}]]
+
+
+def test_chroma_collection_get_coerces_none_metadatas_to_empty_dict():
+    """Same None-coercion guarantee on the get() path. Same rationale as
+    query(): callers zip ids with metadatas and reach `.get("wing")`
+    without first checking for None."""
+    raw = {
+        "ids": ["a", "b", "c"],
+        "documents": ["da", "db", "dc"],
+        "metadatas": [{"wing": "w1"}, None, {"wing": "w3"}],
+    }
+    collection = ChromaCollection(_FakeCollection(get_response=raw))
+
+    result = collection.get()
+
+    assert result.metadatas == [{"wing": "w1"}, {}, {"wing": "w3"}]
+
+
+def test_chroma_collection_get_coerces_padding_remains_dict():
+    """Padding short metadata lists to match ids length already uses {}.
+    Regression guard: the new None-coercion step must not convert those
+    padded {} into something else or re-introduce Nones."""
+    raw = {
+        "ids": ["a", "b", "c"],
+        "documents": ["da", "db", "dc"],
+        "metadatas": [{"wing": "w1"}],  # short — needs 2 padded {}
+    }
+    collection = ChromaCollection(_FakeCollection(get_response=raw))
+
+    result = collection.get()
+
+    assert result.metadatas == [{"wing": "w1"}, {}, {}]
+
+
+def test_chroma_collection_get_without_metadatas_requested_stays_empty():
+    """When the caller omits metadatas from include=, the boundary must
+    not fabricate coerced dicts — empty list in, empty list out."""
+    raw = {
+        "ids": ["a"],
+        "documents": ["da"],
+        "metadatas": [{"wing": "w1"}],  # present in raw but not requested
+    }
+    collection = ChromaCollection(_FakeCollection(get_response=raw))
+
+    result = collection.get(include=["documents"])
+
+    assert result.metadatas == []
+
+
 def test_chroma_collection_delegates_writes():
     fake = _FakeCollection()
     collection = ChromaCollection(fake)