Merge pull request #21495 from bernt-matthias/test-output_collections_min_max

Tool testing: add min/max attributes to test output collections

Author: jmchilton

lib/galaxy/tool_util/linters/tests.py

@@ -426,9 +426,14 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
                 discover_datasets = corresponding_output.find(".//discover_datasets")
                 if discover_datasets is None:
                     continue
-                if "count" not in output.attrib and output.find("./discovered_dataset") is None:
+                if (
+                    "count" not in output.attrib
+                    and "min" not in output.attrib
+                    and "max" not in output.attrib
+                    and output.find("./discovered_dataset") is None
+                ):
                     lint_ctx.error(
-                        f"Test {test_idx}: test output '{name}' must have a 'count' attribute and/or 'discovered_dataset' children",
+                        f"Test {test_idx}: test output '{name}' must have a 'count/min/max' attribute and/or 'discovered_dataset' children",
                         linter=cls.name(),
                         node=output,
                     )
@@ -456,12 +461,16 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
                     continue
                 # - test/collection to outputs/output_collection
                 corresponding_output = output_data_or_collection[name]
-                discover_datasets = corresponding_output.find(".//discover_datasets")
-                if discover_datasets is None:
+                if corresponding_output.find(".//discover_datasets") is None:
                     continue
-                if "count" not in output.attrib and output.find("./element") is None:
+                if (
+                    "count" not in output.attrib
+                    and "min" not in output.attrib
+                    and "max" not in output.attrib
+                    and output.find("./element") is None
+                ):
                     lint_ctx.error(
-                        f"Test {test_idx}: test collection '{name}' must have a 'count' attribute or 'element' children",
+                        f"Test {test_idx}: test collection '{name}' must have a 'count/min/max' attribute or 'element' children",
                         linter=cls.name(),
                         node=output,
                     )
@@ -491,10 +500,10 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
                     continue
                 if corresponding_output.get("type", "") in ["list:list", "list:paired"]:
                     nested_elements = output.find("./element/element")
-                    element_with_count = output.find("./element[@count]")
-                    if nested_elements is None and element_with_count is None:
+                    elements_with_count = output.xpath("./element[@count or @min or @max]")
+                    if nested_elements is None and not elements_with_count:
                         lint_ctx.error(
-                            f"Test {test_idx}: test collection '{name}' must contain nested 'element' tags and/or element children with a 'count' attribute",
+                            f"Test {test_idx}: test collection '{name}' must contain nested 'element' tags and/or element children with a 'count/min/max' attribute",
                             linter=cls.name(),
                             node=output,
                         )

lib/galaxy/tool_util/parser/interface.py

@@ -89,6 +89,8 @@ class ToolSourceTestOutputAttributes(TypedDict):
     metric: str
     pin_labels: Optional[Any]
     count: Optional[int]
+    min: Optional[int]
+    max: Optional[int]
     metadata: Dict[str, Any]
     md5: Optional[str]
     checksum: Optional[str]
@@ -865,6 +867,10 @@ def __init__(self, name, attrib, element_tests, element_count: Optional[int] = N
         else:
             count = attrib.get("count")
         self.count = int(count) if count is not None else None
+        min = attrib.get("min")
+        self.min = int(min) if min is not None else None
+        max = attrib.get("max")
+        self.max = int(max) if max is not None else None
         self.attrib = attrib
         self.element_tests = element_tests
 

lib/galaxy/tool_util/parser/xml.py

@@ -922,6 +922,17 @@ def __parse_test_attributes(
         count = int(attrib.pop("count"))
     except KeyError:
         pass
+    min: Optional[int] = None
+    try:
+        min = int(attrib.pop("min"))
+    except KeyError:
+        pass
+    max: Optional[int] = None
+    try:
+        max = int(attrib.pop("max"))
+    except KeyError:
+        pass
+    has_count_assertions = count is not None or min is not None or max is not None
     extra_files: List[Dict[str, Any]] = []
     ftype: Optional[str] = None
     if "ftype" in attrib:
@@ -949,7 +960,7 @@ def __parse_test_attributes(
     has_checksum = md5sum or checksum
     has_nested_tests = extra_files or element_tests or primary_datasets
     has_object = value_object is not VALUE_OBJECT_UNSET
-    if not (assert_list or file or metadata or has_checksum or has_nested_tests or has_object):
+    if not (assert_list or file or metadata or has_checksum or has_nested_tests or has_object or has_count_assertions):
         raise Exception(
             "Test output defines nothing to check (e.g. must have a 'file' check against, assertions to check, metadata or checksum tests, etc...)"
         )
@@ -966,6 +977,8 @@ def __parse_test_attributes(
         pin_labels=pin_labels,
         location=location,
         count=count,
+        min=min,
+        max=max,
         metadata=metadata,
         md5=md5sum,
         checksum=checksum,

lib/galaxy/tool_util/verify/interactor.py

@@ -332,6 +332,8 @@ def verify_output(self, history_id, jobs, output_data, output_testdef, tool_id,
         attributes = output_testdef.attributes
         name = output_testdef.name
         expected_count = attributes.get("count")
+        min_count = attributes.get("min")
+        max_count = attributes.get("max")
         hid = self.__output_id(output_data)
         # TODO: Twill version verifies dataset is 'ok' in here.
         try:
@@ -357,6 +359,14 @@ def verify_output(self, history_id, jobs, output_data, output_testdef, tool_id,
             raise AssertionError(
                 f"Output '{name}': expected to have '{expected_count}' datasets, but it had '{found_datasets}'"
             )
+        if min_count is not None and min_count > found_datasets:
+            raise AssertionError(
+                f"Output '{name}': expected to have at least '{min_count}' datasets, but it had '{found_datasets}'"
+            )
+        if max_count is not None and max_count < found_datasets:
+            raise AssertionError(
+                f"Output '{name}': expected to have at most '{max_count}' datasets, but it had '{found_datasets}'"
+            )
         for designation, (primary_outfile, primary_attributes) in primary_datasets.items():
             primary_output = None
             for output in outputs:
@@ -1335,12 +1345,16 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
             message = f"Output collection '{name}': expected to be of type [{expected_collection_type}], was of type [{collection_type}]."
             raise AssertionError(message)
 
-    expected_element_count = output_collection_def.count
-    if expected_element_count is not None:
-        actual_element_count = len(data_collection["elements"])
-        if expected_element_count != actual_element_count:
-            message = f"Output collection '{name}': expected to have {expected_element_count} elements, but it had {actual_element_count}."
-            raise AssertionError(message)
+    actual_element_count = len(data_collection["elements"])
+    if output_collection_def.count and output_collection_def.count != actual_element_count:
+        message = f"Output collection '{name}': expected to have {output_collection_def.count} elements, but it had {actual_element_count}."
+        raise AssertionError(message)
+    if output_collection_def.min and output_collection_def.min > actual_element_count:
+        message = f"Output collection '{name}': expected to have at least {output_collection_def.min} elements, but it had {actual_element_count}."
+        raise AssertionError(message)
+    if output_collection_def.max and output_collection_def.max < actual_element_count:
+        message = f"Output collection '{name}': expected to have at most {output_collection_def.max} elements, but it had {actual_element_count}."
+        raise AssertionError(message)
 
     def get_element(elements, id):
         for element in elements:
@@ -1356,7 +1370,6 @@ def verify_elements(element_objects, element_tests):
                 element_outfile, element_attrib = None, element_test
             else:
                 element_outfile, element_attrib = element_test
-            expected_count = element_attrib.get("count")
             if "expected_sort_order" in element_attrib:
                 expected_sort_order[element_attrib["expected_sort_order"]] = element_identifier
 
@@ -1373,10 +1386,21 @@ def verify_elements(element_objects, element_tests):
                 elements = element["object"]["elements"]
                 element_count = len(elements)
                 verify_elements(elements, element_attrib.get("elements", {}))
+            expected_count = element_attrib.get("count")
             if expected_count is not None and expected_count != element_count:
                 raise AssertionError(
                     f"Element '{element_identifier}': expected to have {expected_count} elements, but it had {element_count}"
                 )
+            max = element_attrib.get("max")
+            if max is not None and max < element_count:
+                raise AssertionError(
+                    f"Element '{element_identifier}': expected to have at most {max} elements, but it had {element_count}"
+                )
+            min = element_attrib.get("min")
+            if min is not None and min > element_count:
+                raise AssertionError(
+                    f"Element '{element_identifier}': expected to have at least {min} elements, but it had {element_count}"
+                )
 
         if len(expected_sort_order) > 0:
             generated_sort_order = [_["element_identifier"] for _ in element_objects]

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -1996,6 +1996,16 @@ This is available in Galaxy since release 17.05 and was introduced in [pull requ
         <xs:documentation xml:lang="en">Number or datasets for this output. Should be used for outputs with ``discover_datasets``</xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="min" type="xs:integer" gxdocs:added="26.0">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Minimum number or datasets for this output. Should be used for outputs with ``discover_datasets``</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="max" type="xs:integer" gxdocs:added="26.0">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Maximum number or datasets for this output. Should be used for outputs with ``discover_datasets``</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="location" type="xs:anyURI" gxdocs:added="23.1">
       <xs:annotation>
         <xs:documentation xml:lang="en">URL that points to a remote output file that will downloaded and used for output comparison.
@@ -2348,6 +2358,16 @@ This value is the same as the value of the ``name`` attribute of the
         <xs:documentation xml:lang="en">Number of elements in output collection.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="min" type="xs:integer" gxdocs:added="26.0">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Minimum number of elements in output collection.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="max" type="xs:integer" gxdocs:added="26.0">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Maximum number of elements in output collection.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
   </xs:complexType>
   <xs:complexType name="TestAssertions">
     <xs:annotation>

test/functional/tools/expect_num_outputs.xml

@@ -157,6 +157,18 @@ true
                 </discovered_dataset>
             </output>
         </test>
+        <test expect_num_outputs="1">
+            <param name="produce_discovered_dataset_invisible" value="true" />
+            <output name="discovered_dataset_invisible" min="3" max="3"/>
+        </test>
+        <test expect_num_outputs="1" expect_test_failure="true">
+            <param name="produce_discovered_dataset_invisible" value="true" />
+            <output name="discovered_dataset_invisible" min="4"/>
+        </test>
+        <test expect_num_outputs="1" expect_test_failure="true">
+            <param name="produce_discovered_dataset_invisible" value="true" />
+            <output name="discovered_dataset_invisible" max="2"/>
+        </test>
 
         <!-- discovered datasets invisible -->
         <test expect_num_outputs="1">
@@ -322,5 +334,49 @@ true
                 </element>
             </output_collection>
         </test>
+
+        <!-- successful test of min/max of output_collection and elements -->
+        <test expect_num_outputs="1">
+            <param name="produce_paired_list" value="true" />
+            <output_collection name="paired_list" type="list:paired" min="2" max="2">
+                <element name="p1" min="2" max="2">
+                    <element name="forward" min="1" max="1"/>
+                    <element name="reverse" min="1" max="1"/>
+                </element>
+                <element name="p2" min="2" max="2">
+                    <element name="forward" min="1" max="1"/>
+                    <element name="reverse" min="1" max="1"/>
+                </element>
+            </output_collection>
+        </test>
+
+        <!-- unsuccessful test of min of output_collection -->
+        <test expect_num_outputs="1" expect_test_failure="true">
+            <param name="produce_paired_list" value="true" />
+            <output_collection name="paired_list" type="list:paired" min="3"/>
+        </test>
+        <!-- unsuccessful test of max of output_collection -->
+        <test expect_num_outputs="1" expect_test_failure="true">
+            <param name="produce_paired_list" value="true" />
+            <output_collection name="paired_list" type="list:paired" max="1"/>
+        </test>
+
+        <!-- unsuccessful test of min of element -->
+        <test expect_num_outputs="1" expect_test_failure="true">
+            <param name="produce_paired_list" value="true" />
+            <output_collection name="paired_list" type="list:paired" min="2" max="2">
+                <element name="p1" min="3"/>
+                <element name="p2" min="3"/>
+            </output_collection>
+        </test>
+        <!-- unsuccessful test of min of element -->
+        <test expect_num_outputs="1" expect_test_failure="true">
+            <param name="produce_paired_list" value="true" />
+            <output_collection name="paired_list" type="list:paired" min="2" max="2">
+                <element name="p1" max="1"/>
+                <element name="p2" max="1"/>
+            </output_collection>
+        </test>
+
     </tests>
 </tool>

test/unit/tool_util/test_tool_linters.py

@@ -2098,19 +2098,19 @@ def test_tests_discover_outputs(lint_ctx):
     tool_source = get_xml_tool_source(TESTS_DISCOVER_OUTPUTS)
     run_lint_module(lint_ctx, tests, tool_source)
     assert (
-        "Test 3: test output 'data_name' must have a 'count' attribute and/or 'discovered_dataset' children"
+        "Test 3: test output 'data_name' must have a 'count/min/max' attribute and/or 'discovered_dataset' children"
         in lint_ctx.error_messages
     )
     assert (
-        "Test 3: test collection 'collection_name' must have a 'count' attribute or 'element' children"
+        "Test 3: test collection 'collection_name' must have a 'count/min/max' attribute or 'element' children"
         in lint_ctx.error_messages
     )
     assert (
-        "Test 3: test collection 'collection_name' must contain nested 'element' tags and/or element children with a 'count' attribute"
+        "Test 3: test collection 'collection_name' must contain nested 'element' tags and/or element children with a 'count/min/max' attribute"
         in lint_ctx.error_messages
     )
     assert (
-        "Test 5: test collection 'collection_name' must contain nested 'element' tags and/or element children with a 'count' attribute"
+        "Test 5: test collection 'collection_name' must contain nested 'element' tags and/or element children with a 'count/min/max' attribute"
         in lint_ctx.error_messages
     )
     assert len(lint_ctx.error_messages) == 4