Skip to content

Commit 48c54ef

Browse files
autarchautarch
authored
TOOLS-4148 Add a new sharded test type for sharded cluster integration tests (#916)
This adds a new `TOOLS_TESTING_SHARDED_INTEGRATION` test type, wires it up in the build script via -topology=sharded, adds a "create sharded cluster" Evergreen function (mirroring "create repl_set" but using ShardingTest), and adds integration-X.Y-sharded tasks for all server versions that have cluster tasks (4.4 through latest). This will be used as I convert JS tests to Go. Some of those tests are specific to sharded clusters, so we need a way to write Go tests against sharded clusters too. This PR also includes the plan that Claude created for the conversion of all JS tests.
1 parent ddde0e1 commit 48c54ef

5 files changed

Lines changed: 183 additions & 1 deletion

File tree

‎build.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ func init() {
6666
Description("runs all integration tests").
6767
OptionalArgs("pkgs", "ssl", "auth", "kerberos", "topology", "race").
6868
Do(buildscript.TestIntegration)
69+
taskRegistry.Declare("test:sharded-integration").
70+
Description("runs tests requiring a sharded cluster topology").
71+
OptionalArgs("pkgs", "ssl", "race").
72+
Do(buildscript.TestShardedIntegration)
6973
taskRegistry.Declare("test:kerberos").
7074
Description("runs all kerberos tests").
7175
OptionalArgs("race").

‎buildscript/build.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,13 @@ func TestIntegration(ctx *task.Context) error {
122122
return runTests(ctx, selectedPkgs(ctx), testtype.IntegrationTestType)
123123
}
124124

125+
// TestShardedIntegration runs tests that require a sharded cluster (mongos) topology.
126+
// It sets only ShardedIntegrationTestType, intentionally not setting IntegrationTestType,
127+
// so regular integration tests (which expect a standalone mongod) are not run against mongos.
128+
func TestShardedIntegration(ctx *task.Context) error {
129+
return runTests(ctx, selectedPkgs(ctx), testtype.ShardedIntegrationTestType)
130+
}
131+
125132
// TestAWSAuth is an Executor that runs all AWS auth tests for the provided packages.
126133
func TestAWSAuth(ctx *task.Context) error {
127134
return runTests(ctx, selectedPkgs(ctx), testtype.AWSAuthTestType)
@@ -191,6 +198,9 @@ func runTests(ctx *task.Context, pkgs []string, testType string) error {
191198
if ctx.Get("topology") == "replSet" {
192199
env = append(env, testtype.ReplSetTestType+"=true")
193200
}
201+
if ctx.Get("topology") == "sharded" {
202+
env = append(env, testtype.ShardedIntegrationTestType+"=true")
203+
}
194204

195205
if ctx.Get("race") == "true" {
196206
args = append(args, "-race")

‎common.yml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,33 @@ functions:
461461
fi;
462462
./bin/mongo $MONGO_ARGS --nodb --eval 'assert.soon(function(x){try{var d = new Mongo("localhost:${mongod_port}"); return true} catch(e){return false}}, "timed out connection")'
463463
464+
"create sharded cluster":
465+
- command: shell.exec
466+
params:
467+
working_dir: src/github.com/mongodb/mongo-tools
468+
background: true
469+
script: |
470+
set -x
471+
set -v
472+
set -e
473+
echo "starting sharded cluster"
474+
mkdir -p /data/db/
475+
# use jsconfig.json to set baseUrl to find libs
476+
mv test/shell_common/jsconfig.json ./
477+
if [ -n "${load_libs_version}" ]; then
478+
IMPORT_LOAD_LIBS="await import(\"../shell_common/libs/load_libs-${load_libs_version}.js\");"
479+
fi
480+
PATH=./bin:$PATH ./bin/mongo --port ${mongod_port} --nodb --eval "$IMPORT_LOAD_LIBS; var st = new ShardingTest({name: \"tools_sharded_cluster\", shards: 1, mongos: [{port: ${mongod_port}}], other: {rsOptions: {}, configOptions: {}}}); while(true){sleep(1000);}"
481+
482+
- command: shell.exec
483+
params:
484+
working_dir: src/github.com/mongodb/mongo-tools
485+
script: |
486+
set -x
487+
set -v
488+
set -e
489+
./bin/mongo --port ${mongod_port} --nodb --eval 'assert.soon(function(x){try{var d = new Mongo("localhost:${mongod_port}"); return true} catch(e){return false}}, "timed out connection")'
490+
464491
"add-aws-auth-variables-to-file":
465492
- command: shell.exec
466493
type: test
@@ -1624,6 +1651,140 @@ tasks:
16241651
vars:
16251652
target: test:integration -ssl=true -topology=replSet ${testArgs}
16261653

1654+
- name: integration-4.4-sharded
1655+
tags: ["4.4", "sharded"]
1656+
commands:
1657+
- func: "fetch source"
1658+
- command: expansions.update
1659+
- func: "download mongod and shell"
1660+
vars:
1661+
mongo_version: "4.4"
1662+
- func: "create sharded cluster"
1663+
- func: "run make target"
1664+
vars:
1665+
target: build
1666+
- func: "run make target"
1667+
vars:
1668+
target: test:sharded-integration ${testArgs}
1669+
1670+
- name: integration-5.0-sharded
1671+
tags: ["5.0", "sharded"]
1672+
commands:
1673+
- func: "fetch source"
1674+
- command: expansions.update
1675+
- func: "download mongod and shell"
1676+
vars:
1677+
mongo_version: "5.0"
1678+
- func: "create sharded cluster"
1679+
- func: "run make target"
1680+
vars:
1681+
target: build
1682+
- func: "run make target"
1683+
vars:
1684+
target: test:sharded-integration ${testArgs}
1685+
1686+
- name: integration-6.0-sharded
1687+
tags: ["6.0", "sharded"]
1688+
commands:
1689+
- func: "fetch source"
1690+
- command: expansions.update
1691+
- func: "download mongod and shell"
1692+
vars:
1693+
mongo_version: "6.0"
1694+
- func: "create sharded cluster"
1695+
- func: "run make target"
1696+
vars:
1697+
target: build
1698+
- func: "run make target"
1699+
vars:
1700+
target: test:sharded-integration ${testArgs}
1701+
1702+
- name: integration-7.0-sharded
1703+
tags: ["7.0", "sharded"]
1704+
commands:
1705+
- func: "fetch source"
1706+
- command: expansions.update
1707+
- func: "download mongod and shell"
1708+
vars:
1709+
mongo_version: "7.0"
1710+
- func: "create sharded cluster"
1711+
- func: "run make target"
1712+
vars:
1713+
target: build
1714+
- func: "run make target"
1715+
vars:
1716+
target: test:sharded-integration ${testArgs}
1717+
1718+
- name: integration-8.0-sharded
1719+
tags: ["8.0", "sharded"]
1720+
commands:
1721+
- func: "fetch source"
1722+
- command: expansions.update
1723+
- func: "download mongod and shell"
1724+
vars:
1725+
mongo_version: "8.0"
1726+
- func: "create sharded cluster"
1727+
- func: "run make target"
1728+
vars:
1729+
target: build
1730+
- func: "run make target"
1731+
vars:
1732+
target: test:sharded-integration ${testArgs}
1733+
1734+
- name: integration-8.1-sharded
1735+
tags: ["8.1", "sharded"]
1736+
commands:
1737+
- func: "fetch source"
1738+
- command: expansions.update
1739+
- func: "download mongod and shell"
1740+
vars:
1741+
mongo_version: "8.1.0"
1742+
- func: "create sharded cluster"
1743+
vars:
1744+
load_libs_version: "8.1"
1745+
- func: "run make target"
1746+
vars:
1747+
target: build
1748+
- func: "run make target"
1749+
vars:
1750+
target: test:sharded-integration ${testArgs}
1751+
1752+
- name: integration-8.2-sharded
1753+
tags: ["8.2", "sharded"]
1754+
commands:
1755+
- func: "fetch source"
1756+
- command: expansions.update
1757+
- func: "download mongod and shell"
1758+
vars:
1759+
mongo_version: "8.2.0"
1760+
- func: "create sharded cluster"
1761+
vars:
1762+
load_libs_version: "8.2"
1763+
- func: "run make target"
1764+
vars:
1765+
target: build
1766+
- func: "run make target"
1767+
vars:
1768+
target: test:sharded-integration ${testArgs}
1769+
1770+
- name: integration-latest-sharded
1771+
tags: ["latest", "sharded"]
1772+
commands:
1773+
- func: "fetch source"
1774+
- command: expansions.update
1775+
- func: "download mongod and shell"
1776+
vars:
1777+
mongo_version: "latest"
1778+
- func: "create sharded cluster"
1779+
vars:
1780+
load_libs_version: *max_server_version
1781+
- func: "run make target"
1782+
vars:
1783+
target: build
1784+
- func: "run make target"
1785+
vars:
1786+
target: test:sharded-integration ${testArgs}
1787+
16271788
- name: aws-auth-latest
16281789
tags: ["latest", "aws-auth"]
16291790
commands:

‎common/testtype/types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ const (
1818
// First checks for a URI for a Mongod in the env variable TOOLS_TESTING_MONGOD. If it does not find it, looks on localhost:33333.
1919
IntegrationTestType = "TOOLS_TESTING_INTEGRATION"
2020

21+
// Testing the tools against a sharded cluster (mongos) topology.
22+
ShardedIntegrationTestType = "TOOLS_TESTING_SHARDED_INTEGRATION"
23+
2124
// Unit tests don't require a real mongod. They may still do file I/O.
2225
UnitTestType = "TOOLS_TESTING_UNIT"
2326

‎precious.toml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@
66
# for example from editors on save. Slower tidiers should only be invoked manually, either by
77
# running the relevant mage command or by explicitly running `precious` from the CLI.
88

9-
exclude = "vendor/**/*"
9+
exclude = [
10+
".ai-plans/**/*",
11+
"vendor/**/*",
12+
]
1013

1114
[commands.golangci-lint]
1215
type = "both"
@@ -96,6 +99,7 @@ invoke = "once"
9699
working-dir = "root"
97100
path-args = "file"
98101
include = [ "*.md" ]
102+
exclude = [ "docs/superpowers/**/*.md" ]
99103
cmd = [
100104
"$PRECIOUS_ROOT/node_modules/.bin/prettier",
101105
"--print-width", "100",

0 commit comments

Comments
 (0)
⚡