Skip to content

Commit cd44aaf

Browse files
PyatakovPyatakov
authored
chore: CI actions and e2e tests optimizations
*Updated GitHub Actions workflows to add Yarn caching and more reliable service startup. - Added steps to locate Yarn cache and use actions/cache to persist it; changed yarn setup to only report version and run yarn install --frozen-lockfile. - Replaced repetitive npm start/pushd blocks with a reusable start() function to parallelize service starts, reduce static sleeps, and wait for the API gateway via TCP probe for robustness. - Added Docker Buildx setup and switched to step-security/docker-build-push-action to build the Cypress image with GHA cache support. - Removed redundant top-level yarn in builds and adjusted sleep timings. - In main workflow, include build/lint/test steps for queue-service, topic-listener-service, and worker-service to ensure those packages are validated. - In UI workflow, wait for the frontend dev server on :4200 before proceeding. *Replaced hardcoded IPFS messageId literals with Cypress.env variables to make test inputs configurable. *Added local IPFS setup to CI workflow. - Introduced an IPFS matrix entry and a setup step to run a local IPFS daemon in the CI. - The workflow now includes matrix.ipfs-version and uses oduwsdl/setup-ipfs to start a local node. - Environment variables were updated to use IPFS_PROVIDER=local. - IPFS_PUBLIC_GATEWAY envs were updated to use https://ipfs.filebase.io/ipfs/{cid} in CI/workflow files to avoid resolver/runner issues and use a stable public gateway. *Replaced localhost with 127.0.0.1 for local curl and IPFS node endpoints. *Updated test resource IDs in e2e-tests/cypress.env.json to match refreshed test data and fixtures used by the end-to-end tests. --------- Signed-off-by: Alex Piatakov <alex.piatakov@swirldslabs.com>
1 parent 1e76959 commit cd44aaf

17 files changed

Lines changed: 317 additions & 266 deletions

‎.github/workflows/add-documentation-to-repo.yaml‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,21 @@ jobs:
3131
- name: Setup Yarn
3232
uses: Borales/actions-yarn@3766bb1335b98fb13c60eaf358fe20811b730a88 # v5.0.0
3333
with:
34-
cmd: install
34+
cmd: --version
35+
36+
- name: Locate yarn cache
37+
id: yarncache
38+
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
39+
40+
- name: Cache yarn
41+
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
42+
with:
43+
path: ${{ steps.yarncache.outputs.dir }}
44+
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
45+
restore-keys: ${{ runner.os }}-yarn-
3546

3647
- name: Install dependencies
37-
run: yarn install
48+
run: yarn install --frozen-lockfile
3849

3950
- name: Start NatsMQ
4051
uses: step-security/nats-action@2254b4cc4958120e36da3a096e8200dac968be1c # v0.1.4
@@ -55,7 +66,6 @@ jobs:
5566

5667
- name: Build
5768
run: |
58-
yarn
5969
pushd interfaces
6070
yarn run build
6171
popd
@@ -91,7 +101,7 @@ jobs:
91101
pushd analytics-service
92102
yarn start &
93103
popd
94-
sleep 30
104+
sleep 20
95105
env:
96106
QM_VERIFICATION: false
97107

@@ -100,9 +110,9 @@ jobs:
100110
rm -fv swagger.yaml
101111
rm -fv swagger-indexer.yaml
102112
rm -fv swagger-analytics.yaml
103-
curl -o swagger.yaml http://localhost:3002/api-docs-yaml
104-
curl -o swagger-indexer.yaml http://localhost:3021/api-docs-yaml
105-
curl -o swagger-analytics.yaml http://localhost:3020/api-docs-yaml
113+
curl -o swagger.yaml http://127.0.0.1:3002/api-docs-yaml
114+
curl -o swagger-indexer.yaml http://127.0.0.1:3021/api-docs-yaml
115+
curl -o swagger-analytics.yaml http://127.0.0.1:3020/api-docs-yaml
106116
git add swagger.yaml
107117
git add swagger-indexer.yaml
108118
git add swagger-analytics.yaml

‎.github/workflows/api-after-commit.yml‎

Lines changed: 49 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ jobs:
1717
matrix:
1818
node-version: [ 20.20.2 ]
1919
mongodb-version: [ 7.0.21 ]
20+
ipfs-version: [ 0.41.0 ]
2021
steps:
2122
- name: Harden Runner
2223
uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
@@ -34,10 +35,21 @@ jobs:
3435
- name: Setup Yarn
3536
uses: Borales/actions-yarn@3766bb1335b98fb13c60eaf358fe20811b730a88 # v5.0.0
3637
with:
37-
cmd: install
38+
cmd: --version
39+
40+
- name: Locate yarn cache
41+
id: yarncache
42+
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
43+
44+
- name: Cache yarn
45+
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
46+
with:
47+
path: ${{ steps.yarncache.outputs.dir }}
48+
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
49+
restore-keys: ${{ runner.os }}-yarn-
3850

3951
- name: Install dependencies
40-
run: yarn install
52+
run: yarn install --frozen-lockfile
4153

4254
- name: Build packages
4355
run: |
@@ -80,54 +92,35 @@ jobs:
8092
with:
8193
port: '4222'
8294

95+
- name: Set up IPFS ${{ matrix.ipfs-version }}
96+
uses: oduwsdl/setup-ipfs@678755ac20f92d2dfca7e16138e40ae75f7a0f6f # v0.8.0
97+
with:
98+
ipfs_version: ${{ matrix.ipfs-version }}
99+
run_daemon: true
100+
83101
- name: Start MongoDB
84102
uses: step-security/mongodb-github-action@ca72004b9c8ad6d9ed996c3174edbe62f9f7424a
85103
with:
86104
mongodb-version: ${{ matrix.mongodb-version }}
87105

88106
- name: Run Guardian
89107
run: |
90-
pushd notification-service
91-
npm start &
92-
sleep 20
93-
popd
94-
pushd logger-service
95-
npm start &
96-
sleep 20
97-
popd
98-
pushd auth-service
99-
npm start &
100-
sleep 20
101-
popd
102-
pushd guardian-service
103-
npm start &
104-
sleep 20
105-
popd
106-
pushd policy-service
107-
npm start &
108-
sleep 20
109-
popd
110-
pushd queue-service
111-
npm start &
112-
sleep 20
113-
popd
114-
pushd topic-listener-service
115-
npm start &
116-
sleep 20
117-
popd
118-
pushd worker-service
119-
npm start &
120-
npm start &
121-
npm start &
122-
npm start &
123-
npm start &
124-
sleep 20
125-
popd
126-
pushd api-gateway
127-
npm start &
128-
sleep 20
129-
popd
130-
sleep 60
108+
start() { for s in "$@"; do (cd "$s" && npm start) & done; }
109+
110+
start logger-service notification-service
111+
sleep 5
112+
start auth-service queue-service
113+
sleep 5
114+
start policy-service topic-listener-service
115+
for _ in {1..5}; do (cd worker-service && npm start) & done
116+
sleep 5
117+
start guardian-service
118+
sleep 5
119+
start api-gateway
120+
121+
echo "Waiting for API gateway on :3002..."
122+
timeout 180 bash -c 'until (exec 3<>/dev/tcp/127.0.0.1/3002) 2>/dev/null; do sleep 2; done'
123+
sleep 15 # grace for backend NATS subscriptions
131124
env:
132125
CI: true
133126
NODE_OPTIONS: '--openssl-legacy-provider'
@@ -137,9 +130,9 @@ jobs:
137130
JWT_PUBLIC_KEY: ${{ secrets.CI_JWT_PUBLIC_KEY }}
138131
SERVICE_JWT_PUBLIC_KEY_ALL: ${{ secrets.CI_JWT_PUBLIC_KEY }}
139132
SERVICE_JWT_SECRET_KEY_ALL: ${{ secrets.CI_JWT_PRIVATE_KEY }}
140-
IPFS_NODE_ADDRESS: http://localhost:5001
141-
IPFS_PROVIDER: filebase
142-
IPFS_PUBLIC_GATEWAY: https://ipfs.io/ipfs/{cid}
133+
IPFS_NODE_ADDRESS: http://127.0.0.1:5001
134+
IPFS_PROVIDER: local
135+
IPFS_PUBLIC_GATEWAY: https://ipfs.filebase.io/ipfs/{cid}
143136
IPFS_STORAGE_KEY: ${{ secrets.E2E_IPFS_STORAGE_KEY }}
144137
IPFS_STORAGE_PROOF: ${{ secrets.E2E_IPFS_STORAGE_PROOF }}
145138
IPFS_STORAGE_API_KEY: ${{ secrets.IPFS_STORAGE_API_KEY }}
@@ -149,8 +142,17 @@ jobs:
149142
INITIAL_BALANCE: 2
150143
INITIAL_STANDARD_REGISTRY_BALANCE: 20
151144

145+
- name: Set up Docker Buildx
146+
uses: step-security/setup-buildx-action@f931205d68723ad9589fd2a7e2ece238bf9de341 # v4.0.0
147+
152148
- name: Build Cypress Docker image
153-
run: docker build -t cypress-runner ./e2e-tests
149+
uses: step-security/docker-build-push-action@846549baaf047e867d038826129a64d81df0f704 # v7.1.0
150+
with:
151+
context: ./e2e-tests
152+
tags: cypress-runner
153+
load: true
154+
cache-from: type=gha
155+
cache-to: type=gha,mode=max
154156

155157
- name: Run Cypress Tests in Docker
156158
run: |

‎.github/workflows/api-manual.yml‎

Lines changed: 50 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ jobs:
2525
matrix:
2626
node-version: [20.20.2]
2727
mongodb-version: [7.0.21]
28+
ipfs-version: [ 0.41.0 ]
2829
steps:
2930
- name: Harden Runner
3031
uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
@@ -42,10 +43,21 @@ jobs:
4243
- name: Setup Yarn
4344
uses: Borales/actions-yarn@3766bb1335b98fb13c60eaf358fe20811b730a88 # v5.0.0
4445
with:
45-
cmd: install
46+
cmd: --version
47+
48+
- name: Locate yarn cache
49+
id: yarncache
50+
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
51+
52+
- name: Cache yarn
53+
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
54+
with:
55+
path: ${{ steps.yarncache.outputs.dir }}
56+
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
57+
restore-keys: ${{ runner.os }}-yarn-
4658

4759
- name: Install dependencies
48-
run: yarn install
60+
run: yarn install --frozen-lockfile
4961

5062
- name: Build packages
5163
run: |
@@ -82,59 +94,41 @@ jobs:
8294
pushd api-gateway
8395
yarn run build
8496
popd
97+
8598
- name: Start NatsMQ
8699
uses: step-security/nats-action@2254b4cc4958120e36da3a096e8200dac968be1c # v0.1.4
87100
with:
88101
port: "4222"
89102

103+
- name: Set up IPFS ${{ matrix.ipfs-version }}
104+
uses: oduwsdl/setup-ipfs@678755ac20f92d2dfca7e16138e40ae75f7a0f6f # v0.8.0
105+
with:
106+
ipfs_version: ${{ matrix.ipfs-version }}
107+
run_daemon: true
108+
90109
- name: Start MongoDB
91110
uses: step-security/mongodb-github-action@ca72004b9c8ad6d9ed996c3174edbe62f9f7424a
92111
with:
93112
mongodb-version: ${{ matrix.mongodb-version }}
94113

95114
- name: Run Guardian
96115
run: |
97-
pushd notification-service
98-
npm start &
99-
sleep 20
100-
popd
101-
pushd logger-service
102-
npm start &
103-
sleep 20
104-
popd
105-
pushd auth-service
106-
npm start &
107-
sleep 20
108-
popd
109-
pushd guardian-service
110-
npm start &
111-
sleep 20
112-
popd
113-
pushd policy-service
114-
npm start &
115-
sleep 20
116-
popd
117-
pushd queue-service
118-
npm start &
119-
sleep 20
120-
popd
121-
pushd topic-listener-service
122-
npm start &
123-
sleep 20
124-
popd
125-
pushd worker-service
126-
npm start &
127-
npm start &
128-
npm start &
129-
npm start &
130-
npm start &
131-
sleep 20
132-
popd
133-
pushd api-gateway
134-
npm start &
135-
sleep 20
136-
popd
137-
sleep 60
116+
start() { for s in "$@"; do (cd "$s" && npm start) & done; }
117+
118+
start logger-service notification-service
119+
sleep 5
120+
start auth-service queue-service
121+
sleep 5
122+
start policy-service topic-listener-service
123+
for _ in {1..5}; do (cd worker-service && npm start) & done
124+
sleep 5
125+
start guardian-service
126+
sleep 5
127+
start api-gateway
128+
129+
echo "Waiting for API gateway on :3002..."
130+
timeout 180 bash -c 'until (exec 3<>/dev/tcp/127.0.0.1/3002) 2>/dev/null; do sleep 2; done'
131+
sleep 15 # grace for backend NATS subscriptions
138132
env:
139133
CI: true
140134
NODE_OPTIONS: '--openssl-legacy-provider'
@@ -144,9 +138,9 @@ jobs:
144138
JWT_PUBLIC_KEY: ${{ secrets.CI_JWT_PUBLIC_KEY }}
145139
SERVICE_JWT_PUBLIC_KEY_ALL: ${{ secrets.CI_JWT_PUBLIC_KEY }}
146140
SERVICE_JWT_SECRET_KEY_ALL: ${{ secrets.CI_JWT_PRIVATE_KEY }}
147-
IPFS_NODE_ADDRESS: http://localhost:5001
148-
IPFS_PROVIDER: filebase
149-
IPFS_PUBLIC_GATEWAY: https://ipfs.io/ipfs/{cid}
141+
IPFS_NODE_ADDRESS: http://127.0.0.1:5001
142+
IPFS_PROVIDER: local
143+
IPFS_PUBLIC_GATEWAY: https://ipfs.filebase.io/ipfs/{cid}
150144
IPFS_STORAGE_KEY: ${{ secrets.E2E_IPFS_STORAGE_KEY }}
151145
IPFS_STORAGE_PROOF: ${{ secrets.E2E_IPFS_STORAGE_PROOF }}
152146
IPFS_STORAGE_API_KEY: ${{ secrets.IPFS_STORAGE_API_KEY }}
@@ -156,8 +150,17 @@ jobs:
156150
INITIAL_BALANCE: 10
157151
INITIAL_STANDARD_REGISTRY_BALANCE: 200
158152

153+
- name: Set up Docker Buildx
154+
uses: step-security/setup-buildx-action@f931205d68723ad9589fd2a7e2ece238bf9de341 # v4.0.0
155+
159156
- name: Build Cypress Docker image
160-
run: docker build -t cypress-runner ./e2e-tests
157+
uses: step-security/docker-build-push-action@846549baaf047e867d038826129a64d81df0f704 # v7.1.0
158+
with:
159+
context: ./e2e-tests
160+
tags: cypress-runner
161+
load: true
162+
cache-from: type=gha
163+
cache-to: type=gha,mode=max
161164

162165
- name: Run Cypress API Tests in Docker
163166
run: |

0 commit comments

Comments
 (0)
⚡