Merge remote-tracking branch 'origin/main' into rk/effect-logging

Signed-off-by: Roland Kuhn <rk@rkuhn.info>

Author: rkuhn

.github/workflows/CI.yml

@@ -67,10 +67,10 @@ jobs:
               rustup toolchain add nightly-x86_64-unknown-linux-gnu
               rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu
 
-          - runner: ubuntu-latest
+          - runner: ubuntu-24.04
             target: aarch64-unknown-linux-musl
             command: test
-            setup: rustup target add aarch64-unknown-linux-musl
+            setup: rustup target add aarch64-unknown-linux-musl && sudo apt-get install -y pkg-config libssl-dev
             cross-compile: true
 
     timeout-minutes: 30
@@ -161,8 +161,11 @@ jobs:
       AMARU_PEER_ADDRESS: 127.0.0.1:3001
     strategy:
       matrix:
-        network: [preprod]
+        network:
+          - name: preprod
+            magic: 1
         cardano_node_version: [10.1.4]
+    continue-on-error: true 
     steps:
       - uses: actions/checkout@v4
 
@@ -176,11 +179,11 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           # The path should match the one used for the 'Nightly Sync' workflow.
-          path: ${{ runner.temp }}/db-${{ matrix.network }}
+          path: ${{ runner.temp }}/db-${{ matrix.network.name }}
           # The key should also match
-          key: cardano-node-ogmios-${{ matrix.network }}
+          key: cardano-node-ogmios-${{ matrix.network.name }}
           restore-keys: |
-            cardano-node-ogmios-${{ matrix.network }}
+            cardano-node-ogmios-${{ matrix.network.name }}
 
       - name: Check if cardano-node-db is available
         if: steps.cache-cardano-node-db.outputs.cache-hit == ''
@@ -193,9 +196,9 @@ jobs:
         shell: bash
         run: |
           docker pull ghcr.io/intersectmbo/cardano-node:${{ matrix.cardano_node_version }}
-          make HASKELL_NODE_CONFIG_DIR=cardano-node-config NETWORK=${{ matrix.network }} download-haskell-config
+          make HASKELL_NODE_CONFIG_DIR=cardano-node-config NETWORK=${{ matrix.network.name }} download-haskell-config
           docker run -d --name cardano-node \
-            -v ${{ runner.temp }}/db-${{ matrix.network }}:/db \
+            -v ${{ runner.temp }}/db-${{ matrix.network.name }}:/db \
             -v ${{ runner.temp }}/ipc:/ipc \
             -v ./cardano-node-config:/config \
             -v ./cardano-node-config:/genesis \
@@ -217,11 +220,11 @@ jobs:
         run: |
           cargo test --no-run -p amaru
 
-      - name: Cache Amaru's ledger.db
+      - name: Cache Amaru's ledger.${{ matrix.network.name }}.db
         id: cache-ledger-db
         uses: actions/cache/restore@v4
         with:
-          path: ./ledger.db
+          path: ./ledger.${{ matrix.network.name }}.db
           # If the ledger store serialisation format changes and become
           # incompatible, it is necessary to bump the index below to invalidate
           # the cached ledger snapshots, and recompute them from the CBOR ones
@@ -233,29 +236,51 @@ jobs:
       - name: Full bootstrap amaru
         if: steps.cache-ledger-db.outputs.cache-hit == ''
         run: |
-          make BUILD_PROFILE=test bootstrap
+          make BUILD_PROFILE=test NETWORK=${{ matrix.network.name }} bootstrap
 
       - name: Light bootstrap amaru
         if: steps.cache-ledger-db.outputs.cache-hit != ''
         run: |
-          make BUILD_PROFILE=test import-headers
-          make BUILD_PROFILE=test import-nonces
+          make BUILD_PROFILE=test NETWORK=${{ matrix.network.name }} import-headers
+          make BUILD_PROFILE=test NETWORK=${{ matrix.network.name }} import-nonces
 
       - uses: actions/cache/save@v4
         if: github.event_name == 'push' || steps.cache-ledger-db.outputs.cache-hit == ''
         with:
-          path: ./ledger.db
+          path: ./ledger.${{ matrix.network.name }}.db
           key: ${{ runner.OS }}-ledger-cache-v6-${{ steps.timestamp.outputs.value }}
 
+      - name: Install Cardano CLI
+        if: github.event_name != 'pull_request'
+        run: |
+          set -eux
+
+          VERSION="10.11.0.0"
+
+          curl -L -o cardano-cli.tar.gz \
+            "https://github.com/IntersectMBO/cardano-cli/releases/download/cardano-cli-${VERSION}/cardano-cli-${VERSION}-x86_64-linux.tar.gz"
+
+          mkdir -p cardano-cli-bin
+          tar -xzf cardano-cli.tar.gz -C cardano-cli-bin
+          mv cardano-cli-bin/cardano-cli* cardano-cli-bin/cardano-cli
+          ls $PWD/cardano-cli-bin/
+          chmod +x cardano-cli-bin/cardano-cli
+
       - name: Run node
         timeout-minutes: 30
         shell: bash
         run: |
+          set -eux
+
+          if [[ "${{ github.event_name }}" != "pull_request" ]]; then
+            # If this is a push event, we want to test all revolut epochs
+            export DEMO_TARGET_EPOCH=$(sudo $PWD/cardano-cli-bin/cardano-cli query tip --socket-path ${{ runner.temp }}/ipc/node.socket --testnet-magic ${{ matrix.network.magic }} | jq '.epoch - 1')
+          fi
           make BUILD_PROFILE=test demo
 
       - name: Run tests
         run: |
-          make BUILD_PROFILE=test test-e2e
+          make BUILD_PROFILE=test NETWORK=${{ matrix.network.name }} test-e2e
 
       - name: Teardown haskell node
         shell: bash
@@ -266,8 +291,8 @@ jobs:
       - uses: actions/cache/save@v4
         if: github.event_name == 'push'
         with:
-          path: ${{ runner.temp }}/db-${{ matrix.network }}
-          key: cardano-node-ogmios-${{ matrix.network }}-${{ steps.timestamp.outputs.value }}
+          path: ${{ runner.temp }}/db-${{ matrix.network.name }}
+          key: cardano-node-ogmios-${{ matrix.network.name }}-${{ steps.timestamp.outputs.value }}
 
   examples:
     name: Examples

.github/workflows/NightlySynchronization.yml

@@ -15,7 +15,7 @@ jobs:
   sync_and_cache:
     strategy:
       matrix:
-        network: [ preprod ]
+        network: [ preprod, preview ]
         ogmios_version: [ v6.11.2 ]
         cardano_node_version: [ 10.1.4 ]
 
@@ -50,4 +50,12 @@ jobs:
       shell: bash
       working-directory: ${{ runner.temp }}/db-${{ matrix.network }}
       run: |
-        rm -f immutable/00*.chunk immutable/01*.chunk immutable/02*.chunk immutable/030*.chunk
+        if [ "$NETWORK" == "preprod" ]; then
+          rm -f immutable/00*.chunk immutable/01*.chunk immutable/02*.chunk immutable/030*.chunk
+        fi
+
+        if [ "$NETWORK" == "preview" ]; then
+          rm -f immutable/0*.chunk immutable/10*.chunk immutable/11*.chunk
+        fi
+      env:
+        NETWORK: ${{ matrix.network }}

.gitignore

@@ -14,10 +14,10 @@ target/
 *.pdb
 
 # Default database location for the ledger
-/*ledger.db/
+/*ledger.*.db/
 
 # Default database location for the consensus
-/*chain.db/
+/*chain.*.db/
 
 # Insta not-yet reviewed snapshots
 *.snap.new
@@ -40,4 +40,4 @@ coverage/
 /cardano-node-config/
 
 # llvm-cov generated file
-lcov.info
+lcov.info

Makefile

@@ -11,8 +11,8 @@ HASKELL_NODE_CONFIG_SOURCE := https://book.world.dev.cardano.org/environments
 COVERAGE_DIR ?= coverage
 COVERAGE_CRATES ?=
 LISTEN_ADDRESS ?= 0.0.0.0:0
-LEDGER_DIR ?= ./ledger.db
-CHAIN_DIR ?= ./chain.db
+LEDGER_DIR ?= ./ledger.$(NETWORK).db
+CHAIN_DIR ?= ./chain.$(NETWORK).db
 BUILD_PROFILE ?= release
 
 .PHONY: help bootstrap run import-snapshots import-headers import-nonces download-haskell-config coverage-html coverage-lconv check-llvm-cov
@@ -25,60 +25,68 @@ help:
 	@grep -E '^[a-zA-Z0-9_]+ \?= '  Makefile | sort | while read -r l; do printf "  \033[36m$$(echo $$l | cut -f 1 -d'=')\033[00m=$$(echo $$l | cut -f 2- -d'=')\n"; done
 
 snapshots/$(NETWORK): ## Download snapshots
-	@mkdir -p $@
-	@cat $(SNAPSHOTS_FILE) \
+	@if [ ! -f "${SNAPSHOTS_FILE}" ]; then echo "SNAPSHOTS_FILE not found: ${SNAPSHOTS_FILE}"; exit 1; fi;
+	mkdir -p $@
+	cat $(SNAPSHOTS_FILE) \
 		| jq -r '.[] | "\(.point) \(.url)"' \
 		| while read p u; do \
 			echo "Fetching $$p.cbor"; \
 			curl --progress-bar -o - $$u | gunzip > $@/$$p.cbor; \
 		done
 
-download-haskell-config: ## Download Cardano Haskell configuration for $NETWORK
-	mkdir -p $(HASKELL_NODE_CONFIG_DIR)
-	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/config.json
-	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/topology.json
-	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/byron-genesis.json
-	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/shelley-genesis.json
-	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/alonzo-genesis.json
-	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/conway-genesis.json
-
 import-snapshots: snapshots/$(NETWORK) ## Import snapshots for demo
-	@test -d $^ || (echo "Error: folder '$^' does not exist!" && exit 1)
-	SNAPSHOT_ARGS=""; \
+	@SNAPSHOT_ARGS=""; \
+	CBOR_FILES=$$(find "$^" -maxdepth 1 -name '*.cbor'); \
+	if [ -z "$$CBOR_FILES" ]; then echo "No .cbor files found in $^"; exit 1; fi; \
 	for SNAPSHOT in $(wildcard $^/*.cbor); do \
 		SNAPSHOT_ARGS="$$SNAPSHOT_ARGS --snapshot $$SNAPSHOT"; \
 	done; \
 	cargo run --profile $(BUILD_PROFILE) -- import-ledger-state \
+		--network $(NETWORK) \
 		--ledger-dir "$(LEDGER_DIR)" \
 		$$SNAPSHOT_ARGS
 
 import-headers: ## Import headers from $AMARU_PEER_ADDRESS for demo
-	@HEADERS=$$(jq -r '.[]' $(HEADERS_FILE)); \
+	@if [ ! -f "$(HEADERS_FILE)" ]; then echo "HEADERS_FILE not found: $(HEADERS_FILE)"; exit 1; fi; \
+	HEADERS=$$(jq -r '.[]' $(HEADERS_FILE)); \
 	for HEADER in $$HEADERS; do \
 		cargo run --profile $(BUILD_PROFILE) -- import-headers \
+			--network $(NETWORK) \
 			--chain-dir $(CHAIN_DIR) \
 			--peer-address $(AMARU_PEER_ADDRESS) \
 			--starting-point $$HEADER \
 			--count 2; \
 	done
 
 import-nonces: ## Import nonces for demo
+	@if [ ! -f "$(NONCES_FILE)" ]; then echo "NONCES_FILE not found: $(NONCES_FILE)"; exit 1; fi; \
 	cargo run --profile $(BUILD_PROFILE) -- import-nonces \
+		--network $(NETWORK) \
 		--chain-dir $(CHAIN_DIR) \
 		--at $$(jq -r .at $(NONCES_FILE)) \
 		--active $$(jq -r .active $(NONCES_FILE)) \
 		--candidate $$(jq -r .candidate $(NONCES_FILE)) \
 		--evolving $$(jq -r .evolving $(NONCES_FILE)) \
 		--tail $$(jq -r .tail $(NONCES_FILE))
 
+download-haskell-config: ## Download Cardano Haskell configuration for $NETWORK
+	mkdir -p $(HASKELL_NODE_CONFIG_DIR)
+	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/config.json
+	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/topology.json
+	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/byron-genesis.json
+	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/shelley-genesis.json
+	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/alonzo-genesis.json
+	curl -O --output-dir $(HASKELL_NODE_CONFIG_DIR) $(HASKELL_NODE_CONFIG_SOURCE)/$(NETWORK)/conway-genesis.json
+
 clear-dbs: ## Clear the databases
 	@rm -rf $(LEDGER_DIR) $(CHAIN_DIR)
 
 bootstrap: clear-dbs ## Bootstrap the node from scratch
 	cargo run --profile $(BUILD_PROFILE) -- bootstrap \
 		--peer-address $(AMARU_PEER_ADDRESS) \
 		--config-dir $(CONFIG_FOLDER) \
-		--target-dir '.' \
+		--ledger-dir $(LEDGER_DIR) \
+		--chain-dir $(CHAIN_DIR) \
 		--network $(NETWORK)
 
 dev: ## Compile and run for development with default options
@@ -117,7 +125,7 @@ coverage-lconv: ## Run test coverage for CI to upload to Codecov
 
 demo: ## Synchronize Amaru until a target epoch $DEMO_TARGET_EPOCH
 	LEDGER_DIR=$(LEDGER_DIR) CHAIN_DIR=$(CHAIN_DIR) \
-		./scripts/demo.sh $(BUILD_PROFILE) $(AMARU_PEER_ADDRESS) $(LISTEN_ADDRESS) $(DEMO_TARGET_EPOCH) $(NETWORK)
+		./scripts/demo $(BUILD_PROFILE) $(AMARU_PEER_ADDRESS) $(LISTEN_ADDRESS) $(DEMO_TARGET_EPOCH) $(NETWORK)
 
 build-examples: ## Build all examples
 	@for dir in $(wildcard examples/*/.); do \

conformance-tests/.gitignore

@@ -1,5 +1,4 @@
 node_modules
 yarn.lock
 package-lock.json
-snapshots/*
-!snapshots/.gitkeep
+generated/*

conformance-tests/README.md

@@ -2,51 +2,30 @@
 
 This folder contains scripts used to generate snapshots used for [conformance tests](../crates/amaru/tests/snapshots).
 
-## Pre-Requisites
+## Getting started
 
-### 3rd-party dependencies
-
-- [Ogmios](https://github.com/CardanoSolutions/ogmios) >= `v6.13.0`
-- [cardano-node](https://github.com/IntersectMBO/cardano-node) >= `10.1.0`
-
-### Node.js dependencies
-
-Install necessary Node.js dependencies via:
+Install the required dependencies by running:
 
 ```console
 yarn
 ```
 
-## Configuration
-
-Data will be fetched for each point listed in [`config.json`](./config.json). Typically, those points needs to be the _last point_ of target epochs. Currently, it contains points for the PreProd network.
-
-## Fetching Data
-
-The Haskell node only has a memory of up to few thousand blocks. So, in order to fetch states corresponding to historical data, fetching must be done while the cardano-node is synchronizing.
-
-For the current configuration, that means removing all immutable chunks after `03149`, starting your node from there will have it synchronize through the required points. As soon as a point gets available, data will be fetched and the script will move on to the next point. Once your Haskell node is syncing and Ogmios up-and-running as well, quick run:
-
-```console
-yarn fetch
-```
-
-This should terminate once all the points have been processed.
+If you haven't done so, you will also need [the necessary `data`](../data). For an epoch `n`, you need data corresponding to `n`, `n+1` and `n+3` in order to generate the test snapshots. So make sure that you have fetched all the necessary informations beforehand.
 
 ## Generating snapshots
 
 Once the data is available, it needs some post-processing to create proper JSON snapshots to be used as conformance tests. The post-processing ensures that data is properly formatted in a canonical way, and combine raw data fetched from the node in a more meaningful way.
 
-Simply run:
+Simply run (replacing `NETWORK` with the target network):
 
 ```console
-yarn generate-all
+yarn generate-all NETWORK
 ```
 
 > [!TIP]
 >
 > You can also generate snapshots for a single epoch:
 >
 > ```console
-> yarn generate 168
+> yarn generate NETWORK 168
 > ```

conformance-tests/scripts/generate-all

@@ -1,9 +1,20 @@
 #!/usr/bin/env bash
 
+NETWORK=${1:-}
+
+if [[ -z ${1:-} ]]; then
+  echo -e "\033[91mMissing argument 'NETWORK'\033[0m" >&2
+  exit 1
+fi
+
 scripts=$(dirname -- "${BASH_SOURCE[0]}")
 
-for e in $(seq 163 179); do
-  node $scripts/generate.mjs $e &
+lo=$(ls $scripts/../../data/$NETWORK/pools | sort | head -n 1 | sed "s/.json//")
+hi=$(ls $scripts/../../data/$NETWORK/pools | sort | tail -n 1 | sed "s/.json//")
+hi=$(($hi - 3))
+
+for EPOCH in $(seq $lo $hi); do
+  node $scripts/generate.mjs $NETWORK $EPOCH &
 done;
 
 wait