Drop short / excess raw terms in index-search parser

ILIKE '%term%' on unindexed columns is linear in table size, so every raw
term in a whitespace-split search string is one full scan. Terms shorter
than 4 characters ('of', 'and', 'by', ...) match too much to be useful
and, once joined with all the others, dominate query time. Cap raw
terms at 7 and drop any shorter than 4; filtered terms (tag:foo) and
quoted raw terms ('ab') are preserved as explicit user intent.

Author: mvdbeek

lib/galaxy/managers/workflows.py

@@ -102,6 +102,7 @@
 )
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.util.search import (
+    filter_terms,
     FilteredTerm,
     parse_filters_structured,
     RawTextTerm,
@@ -210,7 +211,7 @@ def index_query(
         stmt = stmt.where(StoredWorkflow.hidden == (true() if show_hidden else false()))
         if payload.search:
             search_query = payload.search
-            parsed_search = parse_filters_structured(search_query, INDEX_SEARCH_FILTERS)
+            parsed_search = filter_terms(parse_filters_structured(search_query, INDEX_SEARCH_FILTERS))
 
             def w_tag_exists(term_text: str, quoted: bool):
                 return tag_exists_filter(

lib/galaxy/util/search.py

@@ -12,6 +12,12 @@
 ParseFilterResultT = Tuple[Optional[List["FilteredTerm"]], Optional[str]]
 QUOTE_PATTERN = re.compile(r"\'(.*?)\'")
 
+# Defaults for `filter_terms` used by index-search callers. A whitespace-rich
+# query turns into one WHERE clause (and, pre-trigram-index, one seq scan per
+# matching table) per raw term, so both floors are there to bound query cost.
+DEFAULT_MIN_RAW_TERM_LENGTH = 4
+DEFAULT_MAX_RAW_TERMS = 7
+
 
 def parse_filters(search_term: str, filters: Optional[Dict[str, str]] = None) -> ParseFilterResultT:
     """Support github-like filters for narrowing the results.
@@ -110,7 +116,39 @@ def simple_result(self) -> ParseFilterResultT:
         return None if len(self.filter_terms) == 0 else self.filter_terms, " ".join([t.text for t in self.text_terms])
 
 
+def filter_terms(
+    parsed: "ParsedSearch",
+    min_raw_term_length: int = DEFAULT_MIN_RAW_TERM_LENGTH,
+    max_raw_terms: Optional[int] = DEFAULT_MAX_RAW_TERMS,
+) -> "ParsedSearch":
+    """Return a new ParsedSearch with short / excess raw text terms dropped.
+
+    Raw (unquoted, non-keyed) terms shorter than ``min_raw_term_length`` are
+    dropped, and the surviving raw terms are capped at ``max_raw_terms``.
+    Filtered terms (``key:value``) and quoted raw terms ('foo bar') are
+    always kept — those are explicit user intent.
+    """
+    out = ParsedSearch()
+    raw_kept = 0
+    for term in parsed.terms:
+        if isinstance(term, RawTextTerm) and not term.quoted:
+            if len(term.text) < min_raw_term_length:
+                continue
+            if max_raw_terms is not None and raw_kept >= max_raw_terms:
+                continue
+            raw_kept += 1
+            out.add_unfiltered_text(term.text, term.quoted)
+        elif isinstance(term, RawTextTerm):
+            out.add_unfiltered_text(term.text, term.quoted)
+        else:
+            out.add_keyed_term(term.filter, term.text, term.quoted)
+    return out
+
+
 __all__ = (
+    "DEFAULT_MAX_RAW_TERMS",
+    "DEFAULT_MIN_RAW_TERM_LENGTH",
+    "filter_terms",
     "parse_filters",
     "parse_filters_structured",
 )

test/unit/util/test_search.py

@@ -1,6 +1,9 @@
 from galaxy.util.search import (
+    filter_terms,
+    FilteredTerm,
     parse_filters,
     parse_filters_structured,
+    RawTextTerm,
 )
 
 
@@ -94,3 +97,55 @@ def test_parse_filters_structured():
     assert text_terms[1].quoted is True
     assert text_terms[2].text == "foo"
     assert text_terms[2].quoted is False
+
+
+def test_filter_terms_drops_short_raw_terms():
+    parsed = parse_filters_structured("Copy of Genomic Assembly and analysis", {})
+    filtered = filter_terms(parsed, min_raw_term_length=4, max_raw_terms=None)
+    kept = [t.text for t in filtered.terms]
+    assert kept == ["Copy", "Genomic", "Assembly", "analysis"]
+
+
+def test_filter_terms_preserves_quoted_raw_terms():
+    parsed = parse_filters_structured("'ab' Copy 'de'", {})
+    filtered = filter_terms(parsed, min_raw_term_length=4, max_raw_terms=None)
+    assert [(t.text, t.quoted) for t in filtered.terms] == [
+        ("ab", True),
+        ("Copy", False),
+        ("de", True),
+    ]
+
+
+def test_filter_terms_preserves_filtered_terms_of_any_length():
+    parsed = parse_filters_structured("tag:ab user:cd Copy", {"tag": "tag", "user": "user"})
+    filtered = filter_terms(parsed, min_raw_term_length=4, max_raw_terms=None)
+    kinds = [(t.__class__.__name__, t.text) for t in filtered.terms]
+    # Both filtered terms are preserved even though their text is shorter than 4;
+    # "Copy" (4) is preserved too.
+    assert ("FilteredTerm", "ab") in kinds
+    assert ("FilteredTerm", "cd") in kinds
+    assert ("RawTextTerm", "Copy") in kinds
+
+
+def test_filter_terms_caps_raw_terms_only():
+    parsed = parse_filters_structured(
+        "tag:foo Copy Genomic Assembly analysis shared user nedflanders extra1 extra2",
+        {"tag": "tag"},
+    )
+    filtered = filter_terms(parsed, min_raw_term_length=4, max_raw_terms=3)
+    raw = [t.text for t in filtered.terms if isinstance(t, RawTextTerm)]
+    filt = [t.text for t in filtered.terms if isinstance(t, FilteredTerm)]
+    assert raw == ["Copy", "Genomic", "Assembly"]
+    assert filt == ["foo"]
+
+
+def test_filter_terms_defaults():
+    # Default behaviour: 4-char min length, 7-term cap on raw terms.
+    parsed = parse_filters_structured(
+        "Copy of Genomic Assembly and analysis - RDH shared by user nedflanders",
+        {},
+    )
+    filtered = filter_terms(parsed)
+    kept = [t.text for t in filtered.terms]
+    # of, and, -, RDH, by dropped for length; everything else survives.
+    assert kept == ["Copy", "Genomic", "Assembly", "analysis", "shared", "user", "nedflanders"]