Skip to content

Commit 5a5a1cf

Browse files
sentrivanasentrivana
authored
tests: Generate Flags tox entries with toxgen script (#3974)
- remove hardcoded entries for `openfeature`, `launchdarkly`, `statsig`, and `unleash` from the tox template - remove them from the ignore list in `populate_tox.py` - run `populate_tox.py` to fill in entries for them - run `split_gh_tox_actions.py` to generate the CI yaml files so that they correspond to the new `tox.ini` Note that this effectively eliminates the `-latest` tests for the Flags group. The script doesn't generate any `-latest` tests since it always makes sure to add a pinned entry for the latest version. So in case all of the integrations in a single group are using the script, the whole `-latest` test category is removed.
1 parent 5a66a04 commit 5a5a1cf

5 files changed

Lines changed: 47 additions & 141 deletions

File tree

‎.github/workflows/test-integrations-flags.yml‎

Lines changed: 1 addition & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -22,82 +22,14 @@ env:
2222
CACHED_BUILD_PATHS: |
2323
${{ github.workspace }}/dist-serverless
2424
jobs:
25-
test-flags-latest:
26-
name: Flags (latest)
27-
timeout-minutes: 30
28-
runs-on: ${{ matrix.os }}
29-
strategy:
30-
fail-fast: false
31-
matrix:
32-
python-version: ["3.8","3.12","3.13"]
33-
# python3.6 reached EOL and is no longer being supported on
34-
# new versions of hosted runners on Github Actions
35-
# ubuntu-20.04 is the last version that supported python3.6
36-
# see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
37-
os: [ubuntu-20.04]
38-
steps:
39-
- uses: actions/checkout@v4.2.2
40-
- uses: actions/setup-python@v5
41-
with:
42-
python-version: ${{ matrix.python-version }}
43-
allow-prereleases: true
44-
- name: Setup Test Env
45-
run: |
46-
pip install "coverage[toml]" tox
47-
- name: Erase coverage
48-
run: |
49-
coverage erase
50-
- name: Test launchdarkly latest
51-
run: |
52-
set -x # print commands that are executed
53-
./scripts/runtox.sh "py${{ matrix.python-version }}-launchdarkly-latest"
54-
- name: Test openfeature latest
55-
run: |
56-
set -x # print commands that are executed
57-
./scripts/runtox.sh "py${{ matrix.python-version }}-openfeature-latest"
58-
- name: Test statsig latest
59-
run: |
60-
set -x # print commands that are executed
61-
./scripts/runtox.sh "py${{ matrix.python-version }}-statsig-latest"
62-
- name: Test unleash latest
63-
run: |
64-
set -x # print commands that are executed
65-
./scripts/runtox.sh "py${{ matrix.python-version }}-unleash-latest"
66-
- name: Generate coverage XML (Python 3.6)
67-
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
68-
run: |
69-
export COVERAGE_RCFILE=.coveragerc36
70-
coverage combine .coverage-sentry-*
71-
coverage xml --ignore-errors
72-
- name: Generate coverage XML
73-
if: ${{ !cancelled() && matrix.python-version != '3.6' }}
74-
run: |
75-
coverage combine .coverage-sentry-*
76-
coverage xml
77-
- name: Upload coverage to Codecov
78-
if: ${{ !cancelled() }}
79-
uses: codecov/codecov-action@v5.3.1
80-
with:
81-
token: ${{ secrets.CODECOV_TOKEN }}
82-
files: coverage.xml
83-
# make sure no plugins alter our coverage reports
84-
plugin: noop
85-
verbose: true
86-
- name: Upload test results to Codecov
87-
if: ${{ !cancelled() }}
88-
uses: codecov/test-results-action@v1
89-
with:
90-
token: ${{ secrets.CODECOV_TOKEN }}
91-
files: .junitxml
92-
verbose: true
9325
test-flags-pinned:
9426
name: Flags (pinned)
9527
timeout-minutes: 30
9628
runs-on: ${{ matrix.os }}
9729
strategy:
9830
fail-fast: false
9931
matrix:
100-
python-version: ["3.8","3.12","3.13"]
32+
python-version: ["3.7","3.8","3.9","3.12","3.13"]
10133
# python3.6 reached EOL and is no longer being supported on
10234
# new versions of hosted runners on Github Actions
10335
# ubuntu-20.04 is the last version that supported python3.6

‎scripts/populate_tox/config.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,25 @@
2323
"py3.6": ["aiocontextvars"],
2424
},
2525
},
26+
"launchdarkly": {
27+
"package": "launchdarkly-server-sdk",
28+
},
29+
"openfeature": {
30+
"package": "openfeature-sdk",
31+
},
32+
"statsig": {
33+
"package": "statsig",
34+
"deps": {
35+
"*": ["typing_extensions"],
36+
},
37+
},
2638
"strawberry": {
2739
"package": "strawberry-graphql[fastapi,flask]",
2840
"deps": {
2941
"*": ["httpx"],
3042
},
3143
},
44+
"unleash": {
45+
"package": "UnleashClient",
46+
},
3247
}

‎scripts/populate_tox/populate_tox.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,12 +77,10 @@
7777
"huggingface_hub",
7878
"langchain",
7979
"langchain_notiktoken",
80-
"launchdarkly",
8180
"litestar",
8281
"loguru",
8382
"openai",
8483
"openai_notiktoken",
85-
"openfeature",
8684
"pure_eval",
8785
"pymongo",
8886
"pyramid",
@@ -96,12 +94,10 @@
9694
"spark",
9795
"starlette",
9896
"starlite",
99-
"statsig",
10097
"sqlalchemy",
10198
"tornado",
10299
"trytond",
103100
"typer",
104-
"unleash",
105101
}
106102

107103

‎scripts/populate_tox/tox.jinja‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -163,10 +163,6 @@ envlist =
163163
{py3.9,py3.11,py3.12}-langchain-latest
164164
{py3.9,py3.11,py3.12}-langchain-notiktoken
165165

166-
# LaunchDarkly
167-
{py3.8,py3.12,py3.13}-launchdarkly-v9.8.0
168-
{py3.8,py3.12,py3.13}-launchdarkly-latest
169-
170166
# Litestar
171167
{py3.8,py3.11}-litestar-v{2.0}
172168
{py3.8,py3.11,py3.12}-litestar-v{2.6}
@@ -184,10 +180,6 @@ envlist =
184180
{py3.9,py3.11,py3.12}-openai-latest
185181
{py3.9,py3.11,py3.12}-openai-notiktoken
186182

187-
# OpenFeature
188-
{py3.8,py3.12,py3.13}-openfeature-v0.7
189-
{py3.8,py3.12,py3.13}-openfeature-latest
190-
191183
# OpenTelemetry (OTel)
192184
{py3.7,py3.9,py3.12,py3.13}-opentelemetry
193185

@@ -259,10 +251,6 @@ envlist =
259251
{py3.8,py3.11}-starlite-v{1.48,1.51}
260252
# 1.51.14 is the last starlite version; the project continues as litestar
261253

262-
# Statsig
263-
{py3.8,py3.12,py3.13}-statsig-v0.55.3
264-
{py3.8,py3.12,py3.13}-statsig-latest
265-
266254
# SQL Alchemy
267255
{py3.6,py3.9}-sqlalchemy-v{1.2,1.4}
268256
{py3.7,py3.11}-sqlalchemy-v{2.0}
@@ -284,10 +272,6 @@ envlist =
284272
{py3.7,py3.12,py3.13}-typer-v{0.15}
285273
{py3.7,py3.12,py3.13}-typer-latest
286274

287-
# Unleash
288-
{py3.8,py3.12,py3.13}-unleash-v6.0.1
289-
{py3.8,py3.12,py3.13}-unleash-latest
290-
291275
# === Integrations - Auto-generated ===
292276
# These come from the populate_tox.py script. Eventually we should move all
293277
# integration tests there.
@@ -557,18 +541,6 @@ deps =
557541
openai-latest: tiktoken~=0.6.0
558542
openai-notiktoken: openai
559543
560-
# OpenFeature
561-
openfeature-v0.7: openfeature-sdk~=0.7.1
562-
openfeature-latest: openfeature-sdk
563-
564-
# LaunchDarkly
565-
launchdarkly-v9.8.0: launchdarkly-server-sdk~=9.8.0
566-
launchdarkly-latest: launchdarkly-server-sdk
567-
568-
# Unleash
569-
unleash-v6.0.1: UnleashClient~=6.0.1
570-
unleash-latest: UnleashClient
571-
572544
# OpenTelemetry (OTel)
573545
opentelemetry: opentelemetry-distro
574546
@@ -693,11 +665,6 @@ deps =
693665
starlite-v{1.48}: starlite~=1.48.0
694666
starlite-v{1.51}: starlite~=1.51.0
695667
696-
# Statsig
697-
statsig: typing_extensions
698-
statsig-v0.55.3: statsig~=0.55.3
699-
statsig-latest: statsig
700-
701668
# SQLAlchemy
702669
sqlalchemy-v1.2: sqlalchemy~=1.2.0
703670
sqlalchemy-v1.4: sqlalchemy~=1.4.0

‎tox.ini‎

Lines changed: 31 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -163,10 +163,6 @@ envlist =
163163
{py3.9,py3.11,py3.12}-langchain-latest
164164
{py3.9,py3.11,py3.12}-langchain-notiktoken
165165

166-
# LaunchDarkly
167-
{py3.8,py3.12,py3.13}-launchdarkly-v9.8.0
168-
{py3.8,py3.12,py3.13}-launchdarkly-latest
169-
170166
# Litestar
171167
{py3.8,py3.11}-litestar-v{2.0}
172168
{py3.8,py3.11,py3.12}-litestar-v{2.6}
@@ -184,10 +180,6 @@ envlist =
184180
{py3.9,py3.11,py3.12}-openai-latest
185181
{py3.9,py3.11,py3.12}-openai-notiktoken
186182

187-
# OpenFeature
188-
{py3.8,py3.12,py3.13}-openfeature-v0.7
189-
{py3.8,py3.12,py3.13}-openfeature-latest
190-
191183
# OpenTelemetry (OTel)
192184
{py3.7,py3.9,py3.12,py3.13}-opentelemetry
193185

@@ -259,10 +251,6 @@ envlist =
259251
{py3.8,py3.11}-starlite-v{1.48,1.51}
260252
# 1.51.14 is the last starlite version; the project continues as litestar
261253

262-
# Statsig
263-
{py3.8,py3.12,py3.13}-statsig-v0.55.3
264-
{py3.8,py3.12,py3.13}-statsig-latest
265-
266254
# SQL Alchemy
267255
{py3.6,py3.9}-sqlalchemy-v{1.2,1.4}
268256
{py3.7,py3.11}-sqlalchemy-v{2.0}
@@ -284,14 +272,24 @@ envlist =
284272
{py3.7,py3.12,py3.13}-typer-v{0.15}
285273
{py3.7,py3.12,py3.13}-typer-latest
286274

287-
# Unleash
288-
{py3.8,py3.12,py3.13}-unleash-v6.0.1
289-
{py3.8,py3.12,py3.13}-unleash-latest
290-
291275
# === Integrations - Auto-generated ===
292276
# These come from the populate_tox.py script. Eventually we should move all
293277
# integration tests there.
294278

279+
# ~~~ Flags ~~~
280+
{py3.8,py3.12,py3.13}-launchdarkly-v9.8.1
281+
{py3.8,py3.12,py3.13}-launchdarkly-v9.9.0
282+
283+
{py3.8,py3.12,py3.13}-openfeature-v0.7.5
284+
{py3.9,py3.12,py3.13}-openfeature-v0.8.0
285+
286+
{py3.7,py3.12,py3.13}-statsig-v0.55.3
287+
{py3.7,py3.12,py3.13}-statsig-v0.56.0
288+
289+
{py3.8,py3.12,py3.13}-unleash-v6.0.1
290+
{py3.8,py3.12,py3.13}-unleash-v6.1.0
291+
292+
295293
# ~~~ GraphQL ~~~
296294
{py3.8,py3.10,py3.11}-ariadne-v0.20.1
297295
{py3.8,py3.11,py3.12}-ariadne-v0.22
@@ -307,7 +305,7 @@ envlist =
307305
{py3.8,py3.10,py3.11}-strawberry-v0.209.8
308306
{py3.8,py3.11,py3.12}-strawberry-v0.226.2
309307
{py3.8,py3.11,py3.12}-strawberry-v0.243.1
310-
{py3.9,py3.12,py3.13}-strawberry-v0.260.0
308+
{py3.9,py3.12,py3.13}-strawberry-v0.260.1
311309

312310

313311

@@ -565,18 +563,6 @@ deps =
565563
openai-latest: tiktoken~=0.6.0
566564
openai-notiktoken: openai
567565

568-
# OpenFeature
569-
openfeature-v0.7: openfeature-sdk~=0.7.1
570-
openfeature-latest: openfeature-sdk
571-
572-
# LaunchDarkly
573-
launchdarkly-v9.8.0: launchdarkly-server-sdk~=9.8.0
574-
launchdarkly-latest: launchdarkly-server-sdk
575-
576-
# Unleash
577-
unleash-v6.0.1: UnleashClient~=6.0.1
578-
unleash-latest: UnleashClient
579-
580566
# OpenTelemetry (OTel)
581567
opentelemetry: opentelemetry-distro
582568

@@ -701,11 +687,6 @@ deps =
701687
starlite-v{1.48}: starlite~=1.48.0
702688
starlite-v{1.51}: starlite~=1.51.0
703689

704-
# Statsig
705-
statsig: typing_extensions
706-
statsig-v0.55.3: statsig~=0.55.3
707-
statsig-latest: statsig
708-
709690
# SQLAlchemy
710691
sqlalchemy-v1.2: sqlalchemy~=1.2.0
711692
sqlalchemy-v1.4: sqlalchemy~=1.4.0
@@ -737,6 +718,21 @@ deps =
737718
# These come from the populate_tox.py script. Eventually we should move all
738719
# integration tests there.
739720

721+
# ~~~ Flags ~~~
722+
launchdarkly-v9.8.1: launchdarkly-server-sdk==9.8.1
723+
launchdarkly-v9.9.0: launchdarkly-server-sdk==9.9.0
724+
725+
openfeature-v0.7.5: openfeature-sdk==0.7.5
726+
openfeature-v0.8.0: openfeature-sdk==0.8.0
727+
728+
statsig-v0.55.3: statsig==0.55.3
729+
statsig-v0.56.0: statsig==0.56.0
730+
statsig: typing_extensions
731+
732+
unleash-v6.0.1: UnleashClient==6.0.1
733+
unleash-v6.1.0: UnleashClient==6.1.0
734+
735+
740736
# ~~~ GraphQL ~~~
741737
ariadne-v0.20.1: ariadne==0.20.1
742738
ariadne-v0.22: ariadne==0.22
@@ -760,7 +756,7 @@ deps =
760756
strawberry-v0.209.8: strawberry-graphql[fastapi,flask]==0.209.8
761757
strawberry-v0.226.2: strawberry-graphql[fastapi,flask]==0.226.2
762758
strawberry-v0.243.1: strawberry-graphql[fastapi,flask]==0.243.1
763-
strawberry-v0.260.0: strawberry-graphql[fastapi,flask]==0.260.0
759+
strawberry-v0.260.1: strawberry-graphql[fastapi,flask]==0.260.1
764760
strawberry: httpx
765761

766762

0 commit comments

Comments
 (0)
⚡