Allow sample size of 1 for percentage threshold test (#819)

- Lower `SampleSize::MIN` from `2` to `1` and add per-test minimum
enforcement:
[percentage](https://bencher.dev/docs/explanation/thresholds/#percentage)
test accepts >= 1 (only needs a mean), variance-based tests
([z_score](https://bencher.dev/docs/explanation/thresholds/#z-score),
[t_test](https://bencher.dev/docs/explanation/thresholds/#t-test),
[log_normal](https://bencher.dev/docs/explanation/thresholds/#log-normal),
[iqr](https://bencher.dev/docs/explanation/thresholds/#interquartile-range),
[delta_iqr](https://bencher.dev/docs/explanation/thresholds/#delta-interquartile-range))
require >= 2
- Clarify that sample size counts are historical-only, the new Metric
being tested is not counted
- Update docs across all 9 languages to specify per-test minimums
- Update the
[static](https://bencher.dev/docs/explanation/thresholds/#static) test
callout to recommend max_sample_size: `1`

Author: epompeii

lib/api_projects/tests/thresholds.rs

@@ -430,3 +430,161 @@ async fn thresholds_get_with_wrong_threshold_model() {
         .expect("Request failed");
     assert_eq!(resp.status(), StatusCode::NOT_FOUND);
 }
+
+async fn create_project_with_branch_testbed_measure(
+    server: &TestServer,
+    user: &TestUser,
+    org_name: &str,
+    project_name: &str,
+) -> String {
+    let org = server.create_org(user, org_name).await;
+    let project = server.create_project(user, &org, project_name).await;
+    let project_slug: String = AsRef::<str>::as_ref(&project.slug).to_owned();
+
+    for (path, body) in [
+        (
+            "branches",
+            serde_json::json!({"name": "ssize-branch", "slug": "ssize-branch"}),
+        ),
+        (
+            "testbeds",
+            serde_json::json!({"name": "ssize-testbed", "slug": "ssize-testbed"}),
+        ),
+        (
+            "measures",
+            serde_json::json!({
+                "name": "latency",
+                "slug": "latency",
+                "units": "ns",
+            }),
+        ),
+    ] {
+        let resp = server
+            .client
+            .post(server.api_url(&format!("/v0/projects/{project_slug}/{path}")))
+            .header(
+                bencher_json::AUTHORIZATION,
+                bencher_json::bearer_header(&user.token),
+            )
+            .json(&body)
+            .send()
+            .await
+            .expect("Request failed");
+        assert_eq!(resp.status(), StatusCode::CREATED);
+    }
+
+    project_slug
+}
+
+#[tokio::test]
+async fn create_threshold_percentage_max_sample_size_one_succeeds() {
+    let server = TestServer::new().await;
+    let user = server
+        .signup("Test User", "ssize-percentage@example.com")
+        .await;
+    let project_slug = create_project_with_branch_testbed_measure(
+        &server,
+        &user,
+        "Sample Size Percentage Org",
+        "Sample Size Percentage Project",
+    )
+    .await;
+
+    let body = serde_json::json!({
+        "branch": "ssize-branch",
+        "testbed": "ssize-testbed",
+        "measure": "latency",
+        "test": "percentage",
+        "max_sample_size": 1,
+        "upper_boundary": 0.05,
+    });
+    let resp = server
+        .client
+        .post(server.api_url(&format!("/v0/projects/{project_slug}/thresholds")))
+        .header(
+            bencher_json::AUTHORIZATION,
+            bencher_json::bearer_header(&user.token),
+        )
+        .json(&body)
+        .send()
+        .await
+        .expect("Request failed");
+    assert_eq!(resp.status(), StatusCode::CREATED);
+    let _threshold: JsonThreshold = resp.json().await.expect("Failed to parse threshold");
+}
+
+#[tokio::test]
+async fn create_threshold_t_test_max_sample_size_one_rejected() {
+    let server = TestServer::new().await;
+    let user = server.signup("Test User", "ssize-ttest@example.com").await;
+    let project_slug = create_project_with_branch_testbed_measure(
+        &server,
+        &user,
+        "Sample Size TTest Org",
+        "Sample Size TTest Project",
+    )
+    .await;
+
+    let body = serde_json::json!({
+        "branch": "ssize-branch",
+        "testbed": "ssize-testbed",
+        "measure": "latency",
+        "test": "t_test",
+        "max_sample_size": 1,
+        "upper_boundary": 0.99,
+    });
+    let resp = server
+        .client
+        .post(server.api_url(&format!("/v0/projects/{project_slug}/thresholds")))
+        .header(
+            bencher_json::AUTHORIZATION,
+            bencher_json::bearer_header(&user.token),
+        )
+        .json(&body)
+        .send()
+        .await
+        .expect("Request failed");
+    assert!(
+        resp.status().is_client_error(),
+        "Expected 4xx, got: {}",
+        resp.status()
+    );
+}
+
+#[tokio::test]
+async fn create_threshold_iqr_max_sample_size_one_rejected() {
+    let server = TestServer::new().await;
+    let user = server.signup("Test User", "ssize-iqr@example.com").await;
+    let project_slug = create_project_with_branch_testbed_measure(
+        &server,
+        &user,
+        "Sample Size IQR Org",
+        "Sample Size IQR Project",
+    )
+    .await;
+
+    let body = serde_json::json!({
+        "branch": "ssize-branch",
+        "testbed": "ssize-testbed",
+        "measure": "latency",
+        "test": "iqr",
+        "max_sample_size": 1,
+        "upper_boundary": 1.5,
+    });
+    let resp = server
+        .client
+        .post(server.api_url(&format!("/v0/projects/{project_slug}/thresholds")))
+        .header(
+            bencher_json::AUTHORIZATION,
+            bencher_json::bearer_header(&user.token),
+        )
+        .json(&body)
+        .send()
+        .await
+        .expect("Request failed");
+    assert!(
+        resp.status().is_client_error(),
+        "Expected 4xx, got: {}",
+        resp.status()
+    );
+}

lib/bencher_boundary/src/limits/mod.rs

@@ -957,7 +957,7 @@ mod tests {
         );
     }
 
-    // Edge case: at the minimum allowed sample size (`SampleSize::MIN = 2`), the t-test
+    // Edge case: at the minimum allowed sample size (`SampleSize::TWO = 2`), the t-test
     // uses `freedom = 1` (a Cauchy distribution) and `sqrt(1 + 1/2) = sqrt(1.5)` scaling.
     // The interval must still be finite, and it must be strictly wider than the
     // z-score interval, since both the t-quantile with 1 dof and the PI scaling factor

lib/bencher_valid/src/error.rs

@@ -1,4 +1,4 @@
-use crate::{Boundary, SampleSize, Window};
+use crate::{Boundary, ModelTest, SampleSize, Window};
 
 pub(crate) const REGEX_ERROR: &str = "Failed to compile regex.";
 
@@ -151,6 +151,14 @@ pub enum ValidError {
         "Invalid model, minimum sample size ({min}) is greater than maximum sample size ({max})"
     )]
     SampleSizes { min: SampleSize, max: SampleSize },
+    #[error(
+        "Invalid model, sample size ({sample_size}) is below the minimum ({min}) required for the {test} test"
+    )]
+    TestSampleSize {
+        test: ModelTest,
+        sample_size: SampleSize,
+        min: SampleSize,
+    },
     #[error("Invalid model, lower boundary ({lower}) is greater than upper boundary ({upper})")]
     Boundaries { lower: Boundary, upper: Boundary },
     #[error("Invalid model, no boundary provided")]