feat(config)!: migrate zebrad to use a layered configuration

.config/nextest.toml

@@ -6,9 +6,22 @@ fail-fast = true
 status-level = "pass"
 
 # --- All Tests profile ---
-# Runs all tests except 'check_no_git_dependencies'
+# CI-friendly test selection.
+#
+# Runs fast unit/integration tests and explicitly excludes:
+# - 'check_no_git_dependencies' (it has its own profile)
+# - stateful or long-running tests that require a local cache dir or external
+#   services (lightwalletd RPC/GRPC, full sync, checkpoint generation, mining
+#   RPCs, etc.).
+#
+# Notes:
+# - CI now provides a default cache directory, so stateful tests are no longer
+#   auto-skipped when the cache dir is undefined. To keep this profile fast and
+#   deterministic, we exclude those tests here and offer dedicated profiles
+#   below for running them when needed.
+# TODO: We need a better test architecture to run all non-stateful
 [profile.all-tests]
-default-filter = 'not test(check_no_git_dependencies)'
+default-filter = "not test(check_no_git_dependencies) and not test(=fully_synced_rpc_z_getsubtreesbyindex_snapshot_test) and not test(=lwd_rpc_test) and not test(=lwd_rpc_send_tx) and not test(=lwd_grpc_wallet) and not test(=lwd_integration) and not test(=lwd_sync_full) and not test(=lwd_sync_update) and not test(=lightwalletd_test_suite) and not test(=rpc_get_block_template) and not test(=rpc_submit_block) and not test(=get_peer_info) and not test(~generate_checkpoints_) and not test(=trusted_chain_sync_handles_forks_correctly) and not test(=sync_update_mainnet) and not test(=trusted_chain_sync_handles_forks_correctly) and not test(=activate_mempool_mainnet) and not test(=downgrade_state_format) and not test(=invalidate_and_reconsider_block) and not test(=nu6_funding_streams_and_coinbase_balance)"
 
 # --- Individual Test Profiles ---
 
@@ -103,3 +116,16 @@ default-filter = 'package(zebrad) and test(=lightwalletd_test_suite)'
 [profile.rpc-z-getsubtreesbyindex-snapshot]
 slow-timeout = { period = "30m", terminate-after = 2 }
 default-filter = 'package(zebrad) and test(=fully_synced_rpc_z_getsubtreesbyindex_snapshot_test)'
+
+# TODO: This are not being tested in CI (they run forever)
+[profile.downgrade-state-format]
+slow-timeout = { period = "30m", terminate-after = 2 }
+default-filter = 'package(zebrad) and test(=downgrade_state_format)'
+
+[profile.invalidate-and-reconsider-block]
+slow-timeout = { period = "30m", terminate-after = 2 }
+default-filter = 'package(zebrad) and test(=invalidate_and_reconsider_block)'
+
+[profile.nu6-funding-streams-and-coinbase-balance]
+slow-timeout = { period = "30m", terminate-after = 2 }
+default-filter = 'package(zebrad) and test(=nu6_funding_streams_and_coinbase_balance)'

.github/workflows/cd-deploy-nodes-gcp.yml

@@ -328,7 +328,7 @@ jobs:
           --container-stdin \
           --container-tty \
           --container-image ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
-          --container-env "NETWORK=${{ matrix.network }},LOG_FILE=${{ vars.CD_LOG_FILE }},SENTRY_DSN=${{ vars.SENTRY_DSN }}" \
+          --container-env "ZEBRA_NETWORK__NETWORK=${{ matrix.network }},LOG_FILE=${{ vars.CD_LOG_FILE }},SENTRY_DSN=${{ vars.SENTRY_DSN }}" \
           --service-account ${{ vars.GCP_DEPLOYMENTS_SA }} \
           --scopes cloud-platform \
           --metadata google-logging-enabled=true,google-logging-use-fluentbit=true,google-monitoring-enabled=true \

.github/workflows/ci-coverage.yml

@@ -84,7 +84,7 @@
       - name: Skip network tests on Ubuntu
         # Ubuntu runners don't have reliable network or DNS during test steps.
         shell: bash
-        run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV
+        run: echo "SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV
 
       - name: Minimise proptest cases in Coverage tests
         # We set cases to 1, because some tests already run 1 case by default.

.github/workflows/ci-unit-tests-os.yml

@@ -144,12 +144,13 @@
           echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
           echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 
-      - name: Skip network tests on Ubuntu and Windows
+      - name: Skip network tests on all OSes
+        # TODO: remove this step once we have reliable network on all OSes.
         # Ubuntu runners don't have reliable network or DNS during test steps.
         # Windows runners have an unreliable network.
+        # macOS is affected locally and in the CI, so we skip the tests on all OSes.
         shell: bash
-        if: matrix.os != 'macos-latest'
-        run: echo "ZEBRA_SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV
+        run: echo "SKIP_NETWORK_TESTS=1" >> $GITHUB_ENV
 
       - name: Minimise proptest cases on macOS and Windows
         # We set cases to 1, because some tests already run 1 case by default.
@@ -173,7 +174,7 @@
       # Explicitly run any tests that are usually #[ignored]
 
       - name: Run zebrad large sync tests${{ matrix.features }}
-        # Skip the entire step on Ubuntu and Windows, because the test would be skipped anyway due to ZEBRA_SKIP_NETWORK_TESTS
+        # Skip the entire step on Ubuntu and Windows, because the test would be skipped anyway due to SKIP_NETWORK_TESTS
         if: matrix.os == 'macos-latest'
         run: |
           cargo test --features "${{ matrix.features }}" --release --verbose --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_

.github/workflows/sub-ci-integration-tests-gcp.yml

@@ -32,9 +32,6 @@ on:
 #! The job names in `ci-integration-tests-gcp.yml`, `ci-integration-tests-gcp.patch.yml` and
 #! `ci-integration-tests-gcp.patch-external.yml` must be kept in sync.
 #!
-#! The test variables ZEBRA_CACHE_DIR and LWD_CACHE_DIR used in some steps are set in the
-#! `sub-deploy-integration-tests-gcp.yml` workflow file as inputs. If modified in this file, they must
-#! also be updated in the `sub-deploy-integration-tests-gcp.yml` file.
 jobs:
   # Check if the cached state disks used by the tests are available for the default network.
   #
@@ -82,7 +79,7 @@ jobs:
       app_name: zebrad
       test_id: sync-to-mandatory-checkpoint
       test_description: Test sync up to mandatory checkpoint
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=sync-to-mandatory-checkpoint,ZEBRA_TEST_SYNC_TO_CHECKPOINT=1,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=sync-to-mandatory-checkpoint,TEST_SYNC_TO_CHECKPOINT=1"
       network: ${{ inputs.network || vars.ZCASH_NETWORK }}
       # This test commonly less than 3 hours by October 2024, but now it takes longer
       is_long_test: true
@@ -106,7 +103,7 @@ jobs:
       app_name: zebrad
       test_id: sync-past-mandatory-checkpoint
       test_description: Test full validation sync from a cached state
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=sync-past-mandatory-checkpoint-${{ (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && 'mainnet' || 'testnet' }},ZEBRA_TEST_SYNC_PAST_CHECKPOINT=1,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=sync-past-mandatory-checkpoint-${{ (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && 'mainnet' || 'testnet' }},TEST_SYNC_PAST_CHECKPOINT=1"
       needs_zebra_state: true
       saves_to_disk: false
       disk_suffix: checkpoint
@@ -138,7 +135,7 @@ jobs:
       test_id: sync-full-mainnet
       test_description: Test a full sync up to the tip
       # TODO: update the test to use {{ input.network }} instead?
-      test_variables: NETWORK=Mainnet,NEXTEST_PROFILE=sync-full-mainnet,SYNC_FULL_MAINNET_TIMEOUT_MINUTES=0,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra
+      test_variables: ZEBRA_NETWORK__NETWORK=Mainnet,NEXTEST_PROFILE=sync-full-mainnet,SYNC_FULL_MAINNET_TIMEOUT_MINUTES=0
       # This test runs for longer than 6 hours, so it needs multiple jobs
       is_long_test: true
       needs_zebra_state: false
@@ -165,7 +162,7 @@ jobs:
       app_name: zebrad
       test_id: sync-update-mainnet
       test_description: Test syncing to tip with a Zebra tip state
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=sync-update-mainnet,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=sync-update-mainnet"
       needs_zebra_state: true
       # update the disk on every PR, to increase CI speed
       saves_to_disk: true
@@ -196,7 +193,7 @@ jobs:
       test_id: generate-checkpoints-mainnet
       test_description: Generate Zebra checkpoints on mainnet
       # TODO: update the test to use {{ input.network }} instead?
-      test_variables: NETWORK=Mainnet,NEXTEST_PROFILE=generate-checkpoints-mainnet,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra
+      test_variables: ZEBRA_NETWORK__NETWORK=Mainnet,NEXTEST_PROFILE=generate-checkpoints-mainnet
       needs_zebra_state: true
       # sync-update-mainnet updates the disk on every PR, so we don't need to do it here
       saves_to_disk: false
@@ -230,7 +227,7 @@ jobs:
       app_name: zebrad
       test_id: sync-full-testnet
       test_description: Test a full sync up to the tip on testnet
-      test_variables: NETWORK=Testnet,NEXTEST_PROFILE=sync-full-testnet,SYNC_FULL_TESTNET_TIMEOUT_MINUTES=1,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra
+      test_variables: ZEBRA_NETWORK__NETWORK=Testnet,NEXTEST_PROFILE=sync-full-testnet,SYNC_FULL_TESTNET_TIMEOUT_MINUTES=1
       network: Testnet
       # A full testnet sync could take 2-10 hours in April 2023.
       # The time varies a lot due to the small number of nodes.
@@ -261,7 +258,7 @@ jobs:
       app_name: zebrad
       test_id: generate-checkpoints-testnet
       test_description: Generate Zebra checkpoints on testnet
-      test_variables: NETWORK=Testnet,NEXTEST_PROFILE=generate-checkpoints-testnet,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra
+      test_variables: ZEBRA_NETWORK__NETWORK=Testnet,NEXTEST_PROFILE=generate-checkpoints-testnet
       network: Testnet
       needs_zebra_state: true
       # update the disk on every PR, to increase CI speed
@@ -295,7 +292,7 @@ jobs:
       app_name: lightwalletd
       test_id: lwd-sync-full
       test_description: Test lightwalletd full sync
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-sync-full,ZEBRA_TEST_LIGHTWALLETD=1,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra,LWD_CACHE_DIR=/home/zebra/.cache/lwd"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-sync-full,TEST_LIGHTWALLETD=1"
       # This test runs for longer than 6 hours, so it needs multiple jobs
       is_long_test: true
       needs_zebra_state: true
@@ -322,7 +319,7 @@ jobs:
       app_name: lightwalletd
       test_id: lwd-sync-update
       test_description: Test lightwalletd update sync with both states
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-sync-update,ZEBRA_TEST_LIGHTWALLETD=1,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra,LWD_CACHE_DIR=/home/zebra/.cache/lwd"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-sync-update,TEST_LIGHTWALLETD=1"
       needs_zebra_state: true
       needs_lwd_state: true
       saves_to_disk: true
@@ -350,7 +347,7 @@ jobs:
       app_name: lightwalletd
       test_id: lwd-rpc-test
       test_description: Test lightwalletd RPC with a Zebra tip state
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-rpc-test,ZEBRA_TEST_LIGHTWALLETD=1,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-rpc-test,TEST_LIGHTWALLETD=1"
       needs_zebra_state: true
       saves_to_disk: false
     secrets: inherit
@@ -372,7 +369,7 @@ jobs:
       app_name: lightwalletd
       test_id: lwd-rpc-send-tx
       test_description: Test sending transactions via lightwalletd
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-rpc-send-tx,ZEBRA_TEST_LIGHTWALLETD=1,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra,LWD_CACHE_DIR=/home/zebra/.cache/lwd"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-rpc-send-tx,TEST_LIGHTWALLETD=1"
       needs_zebra_state: true
       needs_lwd_state: true
       saves_to_disk: false
@@ -395,7 +392,7 @@ jobs:
       app_name: lightwalletd
       test_id: lwd-grpc-wallet
       test_description: Test gRPC calls via lightwalletd
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-grpc-wallet,ZEBRA_TEST_LIGHTWALLETD=1,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra,LWD_CACHE_DIR=/home/zebra/.cache/lwd"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=lwd-grpc-wallet,TEST_LIGHTWALLETD=1"
       needs_zebra_state: true
       needs_lwd_state: true
       saves_to_disk: false
@@ -422,7 +419,7 @@ jobs:
       app_name: zebrad
       test_id: rpc-get-block-template
       test_description: Test getblocktemplate RPC method via Zebra's rpc server
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=rpc-get-block-template,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=rpc-get-block-template"
       needs_zebra_state: true
       needs_lwd_state: false
       saves_to_disk: false
@@ -445,7 +442,7 @@ jobs:
       app_name: zebrad
       test_id: rpc-submit-block
       test_description: Test submitting blocks via Zebra's rpc server
-      test_variables: "NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=rpc-submit-block,ZEBRA_CACHE_DIR=/home/zebra/.cache/zebra"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=rpc-submit-block"
       needs_zebra_state: true
       needs_lwd_state: false
       saves_to_disk: false

.github/workflows/sub-ci-unit-tests-docker.yml

@@ -52,7 +52,7 @@ jobs:
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }}
           docker run --tty \
           -e NEXTEST_PROFILE=all-tests \
-          -e NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
+          -e ZEBRA_NETWORK__NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
           -e RUST_LOG=${{ env.RUST_LOG }} \
           -e RUST_BACKTRACE=${{ env.RUST_BACKTRACE }} \
           -e RUST_LIB_BACKTRACE=${{ env.RUST_LIB_BACKTRACE }} \
@@ -74,14 +74,12 @@ jobs:
           short-length: 7
 
       - name: Run zebrad large sync tests
-        env:
-          NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
         run: |
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }}
           docker run --tty \
           -e NEXTEST_PROFILE=sync-large-checkpoints-empty \
-          -e ZEBRA_TEST_LARGE_CHECKPOINTS=1 \
-          -e NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
+          -e TEST_LARGE_CHECKPOINTS=1 \
+          -e ZEBRA_NETWORK__NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
           -e RUST_LOG=${{ env.RUST_LOG }} \
           -e RUST_BACKTRACE=${{ env.RUST_BACKTRACE }} \
           -e RUST_LIB_BACKTRACE=${{ env.RUST_LIB_BACKTRACE }} \
@@ -103,14 +101,12 @@ jobs:
           short-length: 7
 
       - name: Run tests with empty lightwalletd launch
-        env:
-          NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
         run: |
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }}
           docker run --tty \
           -e NEXTEST_PROFILE=lwd-integration \
-          -e ZEBRA_TEST_LIGHTWALLETD=1 \
-          -e NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
+          -e TEST_LIGHTWALLETD=1 \
+          -e ZEBRA_NETWORK__NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
           -e RUST_LOG=${{ env.RUST_LOG }} \
           -e RUST_BACKTRACE=${{ env.RUST_BACKTRACE }} \
           -e RUST_LIB_BACKTRACE=${{ env.RUST_LIB_BACKTRACE }} \
@@ -163,7 +159,7 @@ jobs:
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }}
           docker run --tty \
           -e NEXTEST_PROFILE=check-no-git-dependencies \
-          -e NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
+          -e ZEBRA_NETWORK__NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
           -e RUST_LOG=${{ env.RUST_LOG }} \
           -e RUST_BACKTRACE=${{ env.RUST_BACKTRACE }} \
           -e RUST_LIB_BACKTRACE=${{ env.RUST_LIB_BACKTRACE }} \