feat(config)!: migrate zebrad to use a layered configuration

.config/nextest.toml

@@ -21,7 +21,7 @@ status-level = "pass"
 #   below for running them when needed.
 # TODO: We need a better test architecture to run all non-stateful
 [profile.all-tests]
-default-filter = "not test(check_no_git_dependencies) and not test(=fully_synced_rpc_z_getsubtreesbyindex_snapshot_test) and not test(=lwd_rpc_test) and not test(=lwd_rpc_send_tx) and not test(=lwd_grpc_wallet) and not test(=lwd_integration) and not test(=lwd_sync_full) and not test(=lwd_sync_update) and not test(=lightwalletd_test_suite) and not test(=rpc_get_block_template) and not test(=rpc_submit_block) and not test(=get_peer_info) and not test(~generate_checkpoints_) and not test(=trusted_chain_sync_handles_forks_correctly) and not test(=sync_update_mainnet) and not test(=trusted_chain_sync_handles_forks_correctly) and not test(=activate_mempool_mainnet) and not test(=downgrade_state_format) and not test(=invalidate_and_reconsider_block) and not test(=nu6_funding_streams_and_coinbase_balance)"
+default-filter = "not test(check_no_git_dependencies) and not test(=fully_synced_rpc_z_getsubtreesbyindex_snapshot_test) and not test(=lwd_rpc_test) and not test(=lwd_rpc_send_tx) and not test(=lwd_grpc_wallet) and not test(=lwd_integration) and not test(=lwd_sync_full) and not test(=lwd_sync_update) and not test(=lightwalletd_test_suite) and not test(=rpc_get_block_template) and not test(=rpc_submit_block) and not test(=get_peer_info) and not test(~generate_checkpoints_) and not test(=sync_update_mainnet) and not test(=activate_mempool_mainnet)"
 
 # --- Individual Test Profiles ---
 
@@ -42,10 +42,15 @@ slow-timeout = { period = "1500m", terminate-after = 1 }
 success-output = "immediate"
 default-filter = 'package(zebrad) and test(=sync_full_testnet)'
 
-[profile.sync-to-mandatory-checkpoint]
+[profile.sync-to-mandatory-checkpoint-mainnet]
 slow-timeout = { period = "1500m", terminate-after = 1 }
 success-output = "immediate"
-default-filter = 'package(zebrad) and test(~sync_to_mandatory_checkpoint_)'
+default-filter = 'package(zebrad) and test(=sync_to_mandatory_checkpoint_mainnet)'
+
+[profile.sync-to-mandatory-checkpoint-testnet]
+slow-timeout = { period = "1500m", terminate-after = 1 }
+success-output = "immediate"
+default-filter = 'package(zebrad) and test(=sync_to_mandatory_checkpoint_testnet)'
 
 [profile.sync-update-mainnet]
 slow-timeout = { period = "30m", terminate-after = 2 }
@@ -116,16 +121,3 @@ default-filter = 'package(zebrad) and test(=lightwalletd_test_suite)'
 [profile.rpc-z-getsubtreesbyindex-snapshot]
 slow-timeout = { period = "30m", terminate-after = 2 }
 default-filter = 'package(zebrad) and test(=fully_synced_rpc_z_getsubtreesbyindex_snapshot_test)'
-
-# TODO: This are not being tested in CI (they run forever)
-[profile.downgrade-state-format]
-slow-timeout = { period = "30m", terminate-after = 2 }
-default-filter = 'package(zebrad) and test(=downgrade_state_format)'
-
-[profile.invalidate-and-reconsider-block]
-slow-timeout = { period = "30m", terminate-after = 2 }
-default-filter = 'package(zebrad) and test(=invalidate_and_reconsider_block)'
-
-[profile.nu6-funding-streams-and-coinbase-balance]
-slow-timeout = { period = "30m", terminate-after = 2 }
-default-filter = 'package(zebrad) and test(=nu6_funding_streams_and_coinbase_balance)'

.github/workflows/sub-ci-integration-tests-gcp.yml

@@ -79,7 +79,7 @@ jobs:
       app_name: zebrad
       test_id: sync-to-mandatory-checkpoint
       test_description: Test sync up to mandatory checkpoint
-      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=sync-to-mandatory-checkpoint,TEST_SYNC_TO_CHECKPOINT=1"
+      test_variables: "ZEBRA_NETWORK__NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }},NEXTEST_PROFILE=sync-to-mandatory-checkpoint-${{ (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && 'mainnet' || 'testnet' }},TEST_SYNC_TO_CHECKPOINT=1"
       network: ${{ inputs.network || vars.ZCASH_NETWORK }}
       # This test commonly less than 3 hours by October 2024, but now it takes longer
       is_long_test: true

.github/workflows/sub-ci-unit-tests-docker.yml

@@ -52,7 +52,6 @@ jobs:
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }}
           docker run --tty \
           -e NEXTEST_PROFILE=all-tests \
-          -e ZEBRA_NETWORK__NETWORK="${{ inputs.network || vars.ZCASH_NETWORK }}" \
           -e RUST_LOG=${{ env.RUST_LOG }} \
           -e RUST_BACKTRACE=${{ env.RUST_BACKTRACE }} \
           -e RUST_LIB_BACKTRACE=${{ env.RUST_LIB_BACKTRACE }} \

Cargo.lock

@@ -1975,7 +1975,7 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
 dependencies = [
- "hashbrown 0.15.4",
+ "hashbrown 0.15.5",
 ]
 
 [[package]]
@@ -7087,7 +7087,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "toml 0.9.5",
- "tonic 0.14.0",
+ "tonic 0.14.1",
  "tonic-prost",
  "tonic-prost-build",
  "tower 0.4.13",

Cargo.toml

@@ -136,7 +136,7 @@ tokio-util = "0.7.14"
 toml = "0.9.5"
 tonic = "0.14"
 tonic-prost = "0.14"
-tonic-prost-build = "0.14" 
+tonic-prost-build = "0.14"
 tonic-reflection = "0.14"
 tower = "0.4.13"
 tower-test = "0.4"
@@ -153,7 +153,7 @@ vergen-git2 = { version = "1.0", default-features = false }
 x25519-dalek = "2.0.1"
 zcash_note_encryption = "0.4.1"
 zcash_script = "0.3.2"
-config = { version = "0.15.14", features = ["toml", "json"] }
+config = { version = "0.15.14", features = ["toml"] }
 
 [patch.crates-io]
 sapling-crypto = { git = "https://github.com/zcash/sapling-crypto", branch = "access-inner-param" }

book/src/user/docker.md

@@ -80,7 +80,7 @@ All available Cargo features are listed at
 
 ## Configuring Zebra
 
-Zebra uses config-rs to layer configuration from defaults, an optional TOML file, and `ZEBRA_`-prefixed environment variables. When running with Docker, configure Zebra using any of the following (later items override earlier ones):
+Zebra uses [config-rs](https://crates.io/crates/config) to layer configuration from defaults, an optional TOML file, and `ZEBRA_`-prefixed environment variables. When running with Docker, configure Zebra using any of the following (later items override earlier ones):
 
 1. **Provide a specific config file path:** Set the `CONFIG_FILE_PATH` environment variable to point to your config file within the container. The entrypoint will pass it to `zebrad` via `--config`.
 2. **Use the default config file:** Mount a config file to `/home/zebra/.config/zebrad.toml` (for example using the `configs:` mapping in `docker-compose.yml`). This file is loaded if `CONFIG_FILE_PATH` is not set.
@@ -90,7 +90,7 @@ You can verify your configuration by inspecting Zebra's logs at startup.
 
 ### RPC
 
-Zebra's RPC server is disabled by default. Enable and configure it via TOML or environment variables:
+Zebra's RPC server is disabled by default. Enable and configure it via the TOML configuration file, or configuration environment variables:
 
 * **Using a config file:** Add or uncomment the `[rpc]` section in your `zebrad.toml`. Set `listen_addr` (e.g., `"0.0.0.0:8232"` for Mainnet).
 * **Using environment variables:** Set `ZEBRA_RPC__LISTEN_ADDR` (e.g., `0.0.0.0:8232`). To disable cookie auth, set `ZEBRA_RPC__ENABLE_COOKIE_AUTH=false`. To change the cookie directory, set `ZEBRA_RPC__COOKIE_DIR=/path/inside/container`.
@@ -135,7 +135,7 @@ docker compose -f docker/docker-compose.lwd.yml up
 ```
 
 Note that Docker will run Zebra with the RPC server enabled and the cookie
-authentication mechanism disabled since Lightwalletd doesn't support it. In this
+authentication mechanism disabled when running `docker compose -f docker/docker-compose.lwd.yml up`, since Lightwalletd doesn't support cookie authentication. In this
 example, the RPC server is configured by setting `ZEBRA_` environment variables
 directly in `docker/docker-compose.lwd.yml` (or an accompanying `.env` file).
 

zebrad/src/application.rs

@@ -248,7 +248,13 @@ impl Application for ZebradApp {
             // Ignore errors loading the config for some commands.
             Err(_e) if command.cmd().should_ignore_load_config_error() => Default::default(),
             Err(e) => {
-                status_err!("Zebra could not load its configuration: {}", e);
+                status_err!(
+                    "Zebra could not load the provided configuration file and/or environment variables.\
+                     This might mean you are using a deprecated format of the file, or are attempting to
+                     configure deprecated or unknown fields via environment variables.\
+                     You can generate a valid config by running \"zebrad generate\", \
+                     and diff it against yours to examine any format inconsistencies."
+                );
                 // Convert config::ConfigError to FrameworkError using a generic IO error
                 let io_error = std::io::Error::new(
                     std::io::ErrorKind::InvalidData,

zebrad/tests/acceptance.rs

@@ -30,7 +30,8 @@
 //!   will allow this test to run or give up. Value for the Mainnet full sync tests.
 //! - `SYNC_FULL_TESTNET_TIMEOUT_MINUTES` env variable: The total number of minutes we
 //!   will allow this test to run or give up. Value for the Testnet full sync tests.
-//! - `ZEBRA_STATE__CACHE_DIR` env variable: The path to a Zebra cached state directory.
+//! - A zebrad state cache directory is required for some tests, either at the default state cache
+//!   directory path, or at the path defined in the `ZEBRA_STATE__CACHE_DIR` env variable.
 //!   For some sync tests, this directory needs to be created in the file system
 //!   with write permissions.
 //!
@@ -903,7 +904,9 @@ fn invalid_generated_config() -> Result<()> {
     let output = child.wait_with_output()?;
 
     // Check that Zebra produced an informative message.
-    output.stderr_contains("Zebra could not load its configuration: invalid type: map, expected a string for key `mempool.eviction_memory_time`")?;
+    output.stderr_contains(
+        "Zebra could not load the provided configuration file and/or environment variables",
+    )?;
 
     Ok(())
 }
@@ -1784,7 +1787,7 @@ fn lwd_integration() -> Result<()> {
 
 /// Make sure `zebrad` can sync from peers, but don't actually launch `lightwalletd`.
 ///
-/// This test only runs when a persistent cached state is configured
+/// This test only runs when a persistent cached state directory path is configured
 /// (for example, by setting `ZEBRA_STATE__CACHE_DIR`).
 ///
 /// This test might work on Windows.
@@ -1797,7 +1800,7 @@ fn sync_update_mainnet() -> Result<()> {
 ///
 /// This test only runs when:
 /// - `TEST_LIGHTWALLETD` is set,
-/// - a persistent cached state is configured (e.g., via `ZEBRA_STATE__CACHE_DIR`), and
+/// - a persistent cached state directory path is configured (e.g., via `ZEBRA_STATE__CACHE_DIR`), and
 /// - Zebra is compiled with `--features=lightwalletd-grpc-tests`.
 ///
 /// This test doesn't work on Windows, so it is always skipped on that platform.
@@ -1831,9 +1834,9 @@ fn lwd_sync_full() -> Result<()> {
 ///
 /// Runs the tests in this order:
 /// - launch lightwalletd with empty states,
-/// - if a cached Zebra state is configured:
+/// - if a cached Zebra state directory path is configured:
 ///   - run a full sync
-/// - if a cached Zebra state is configured:
+/// - if a cached Zebra state directory path is configured:
 ///   - run a quick update sync,
 ///   - run a send transaction gRPC test,
 ///   - run read-only gRPC tests.
@@ -1849,7 +1852,7 @@ async fn lightwalletd_test_suite() -> Result<()> {
         launches_lightwalletd: true,
     })?;
 
-    // Only runs when a cached Zebra state is configured.
+    // Only runs when a cached Zebra state directory path is configured with an environment variable.
     lwd_integration_test(UpdateZebraCachedStateNoRpc)?;
 
     // These tests need the compile-time gRPC feature

zebrad/tests/common/cached_state.rs

@@ -31,9 +31,6 @@ use crate::common::{
     test_type::TestType,
 };
 
-// Deprecated: tests should read the cache dir from `ZEBRA_STATE__CACHE_DIR` if they need
-// an external cached state directory. The actual application reads this via config-rs.
-
 /// In integration tests, the interval between database format checks for newly added blocks.
 ///
 /// This should be short enough that format bugs cause CI test failures,

zebrad/tests/common/config.rs

@@ -68,9 +68,9 @@ pub fn default_test_config(net: &Network) -> Result<ZebradConfig> {
 
     // Determine the effective network kind: prefer environment override if present.
     let effective_kind: NetworkKind = match env::var("ZEBRA_NETWORK__NETWORK").ok().as_deref() {
-        Some("Mainnet") | Some("mainnet") | Some("Main") | Some("main") => NetworkKind::Mainnet,
-        Some("Regtest") | Some("regtest") => NetworkKind::Regtest,
-        Some("Testnet") | Some("testnet") => NetworkKind::Testnet,
+        Some("Mainnet") => NetworkKind::Mainnet,
+        Some("Regtest") => NetworkKind::Regtest,
+        Some("Testnet") => NetworkKind::Testnet,
         _ => net.kind(),
     };
 

zebrad/tests/common/sync.rs

@@ -326,9 +326,9 @@ pub fn check_sync_logs_until(
     Ok(zebrad)
 }
 
-/// Returns the cache directory for Zebra's state, as configured via config-rs.
+/// Returns the cache directory for Zebra's state, as configured with [`ZebradConfig::load`] with config-rs.
 ///
-/// Uses the resolved configuration from `ZebradConfig::load(None)`, which
+/// Uses the resolved configuration from [`ZebradConfig::load(None)`](ZebradConfig::load), which
 /// incorporates defaults, optional TOML, and environment overrides.
 fn get_zebra_cached_state_dir() -> PathBuf {
     ZebradConfig::load(None)

zebrad/tests/common/test_type.rs

@@ -296,7 +296,7 @@ impl TestType {
             if default_path.exists() {
                 Some(default_path)
             } else {
-                tracing::info!(
+                tracing::warn!(
                     ?default_path,
                     "skipped {test_name:?} {self:?} lightwalletd test: no cached state found at default path",
                 );
@@ -307,7 +307,7 @@ impl TestType {
             if default_path.exists() {
                 Some(default_path)
             } else {
-                tracing::info!(
+                tracing::warn!(
                     ?default_path,
                     "running {test_name:?} {self:?} lightwalletd test without cached state (no default found)",
                 );