Merge pull request #460 from pragma-org/abailly/tesnet-cluster

feat: run a basic testnet cluster with docker

Author: abailly

crates/amaru-kernel/src/network.rs

@@ -460,7 +460,7 @@ impl std::fmt::Display for NetworkName {
             Self::Mainnet => write!(f, "mainnet"),
             Self::Preprod => write!(f, "preprod"),
             Self::Preview => write!(f, "preview"),
-            Self::Testnet(magic) => write!(f, "testnet:{}", magic),
+            Self::Testnet(magic) => write!(f, "testnet_{}", magic),
         }
     }
 }
@@ -475,7 +475,7 @@ impl std::str::FromStr for NetworkName {
             "preview" => Ok(Self::Preview),
             _ => {
                 let magic = s
-                    .strip_prefix("testnet:")
+                    .strip_prefix("testnet_")
                     .ok_or(format!("Invalid network name {}", s))?;
 
                 magic

crates/amaru-ledger/src/bootstrap.rs

@@ -24,8 +24,7 @@ use amaru_kernel::{
     DRep, DRepRegistration, DRepState, Epoch, EraHistory, Lovelace, MemoizedTransactionOutput,
     Point, PoolId, PoolParams, Proposal, ProposalId, ProposalPointer, ProposalState, Reward,
     ScriptHash, Set, Slot, StakeCredential, StrictMaybe, TransactionInput, TransactionPointer,
-    UnitInterval, Vote, Voter, cbor, heterogeneous_array, network::NetworkName,
-    protocol_parameters::ProtocolParameters,
+    UnitInterval, Vote, Voter, cbor, heterogeneous_array, protocol_parameters::ProtocolParameters,
 };
 use amaru_progress_bar::ProgressBar;
 use std::{
@@ -63,7 +62,7 @@ pub fn import_initial_snapshot(
     db: &(impl Store + 'static),
     bytes: &[u8],
     point: &Point,
-    network: NetworkName,
+    era_history: &EraHistory,
     // A way to notify progress while importing. The second argument is a template argument, which
     // follows the format described in:
     //
@@ -74,8 +73,6 @@ pub fn import_initial_snapshot(
     // Assumes the presence of fully computed rewards when set.
     has_rewards: bool,
 ) -> Result<Epoch, Box<dyn std::error::Error>> {
-    let era_history = <&EraHistory>::from(network);
-
     let mut d = cbor::Decoder::new(bytes);
 
     d.array()?;

crates/amaru-network/src/chain_sync_client.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::point::to_network_point;
+use crate::point::{from_network_point, to_network_point};
 use amaru_kernel::{Point, peer::Peer};
 use pallas_network::miniprotocols::chainsync::{Client, ClientError, HeaderContent, NextResponse};
 use pallas_traverse::MultiEraHeader;
@@ -64,7 +64,7 @@ impl ChainSyncClient {
             intersection.slot = %self.intersection.last().map(|p| p.slot_or_default()).unwrap_or_default(),
         ),
     )]
-    pub async fn find_intersection(&mut self) -> Result<(), ChainSyncClientError> {
+    pub async fn find_intersection(&mut self) -> Result<Point, ChainSyncClientError> {
         let client = &mut self.chain_sync;
         let (point, _) = client
             .find_intersect(
@@ -77,10 +77,10 @@ impl ChainSyncClient {
             .await
             .map_err(ChainSyncClientError::NetworkError)?;
 
-        point.ok_or(ChainSyncClientError::NoIntersectionFound {
+        let intersection = point.ok_or(ChainSyncClientError::NoIntersectionFound {
             points: self.intersection.clone(),
         })?;
-        Ok(())
+        Ok(from_network_point(&intersection))
     }
 
     pub fn intersection(&self) -> &[Point] {

crates/amaru-ouroboros/src/praos/header.rs

@@ -123,7 +123,8 @@ pub fn assert_all<'a>(
         }),
         Box::new(move || {
             AssertVrfProofError::new(
-                &vrf::Input::new(absolute_slot, epoch_nonce),
+                absolute_slot,
+                epoch_nonce,
                 &header.header_body.leader_vrf_output()[..],
                 &vrf::PublicKey::from(declared_vrf_key),
                 &header.header_body.vrf_result,
@@ -201,7 +202,7 @@ pub enum AssertVrfProofError {
     MalformedProof(#[from] vrf::ProofFromBytesError),
 
     #[error("Invalid VRF proof: {0}")]
-    InvalidProof(#[from] vrf::ProofVerifyError),
+    InvalidProof(vrf::ProofVerifyError, Slot, Hash<32>, Vec<u8>),
 
     #[error("could not convert slice to array")]
     TryFromSliceError,
@@ -239,7 +240,9 @@ impl PartialEq for AssertVrfProofError {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {
             (Self::MalformedProof(l0), Self::MalformedProof(r0)) => l0 == r0,
-            (Self::InvalidProof(l0), Self::InvalidProof(r0)) => l0 == r0,
+            (Self::InvalidProof(l0, l1, l2, l3), Self::InvalidProof(r0, r1, r2, r3)) => {
+                l0 == r0 && l1 == r1 && l2 == r2 && l3 == r3
+            }
             (Self::TryFromSliceError, Self::TryFromSliceError) => true,
             (
                 Self::ProofMismatch {
@@ -269,11 +272,13 @@ impl PartialEq for AssertVrfProofError {
 impl AssertVrfProofError {
     /// Assert that the VRF output from the block and its corresponding hash.
     pub fn new(
-        input: &vrf::Input,
+        absolute_slot: Slot,
+        epoch_nonce: &Hash<32>,
         output: &[u8],
         leader_public_key: &vrf::PublicKey,
         certificate: &VrfCert,
     ) -> Result<(), Self> {
+        let input = &vrf::Input::new(absolute_slot, epoch_nonce);
         // TODO: Pallas should have fixed size slices here.
         let block_proof_hash: [u8; vrf::Proof::HASH_SIZE] = {
             let bytes: &[u8] = certificate.0.as_ref();
@@ -288,7 +293,14 @@ impl AssertVrfProofError {
 
         // Verify the VRF proof
         let vrf_proof = vrf::Proof::try_from(&block_proof)?;
-        let proof_hash = vrf_proof.verify(leader_public_key, input)?;
+        let proof_hash = vrf_proof.verify(leader_public_key, input).map_err(|e| {
+            Self::InvalidProof(
+                e,
+                absolute_slot,
+                *epoch_nonce,
+                leader_public_key.as_ref().to_vec(),
+            )
+        })?;
         if proof_hash.as_slice() != block_proof_hash {
             return Err(Self::ProofMismatch {
                 declared: Box::new(block_proof_hash),

crates/amaru/src/bin/amaru/cmd/bootstrap.rs

@@ -13,11 +13,9 @@
 // limitations under the License.
 
 use super::{
-    import_headers::import_headers_for_network,
-    import_ledger_state::import_all_from_directory,
-    import_nonces::{InitialNonces, import_nonces},
+    import_headers::import_headers_for_network, import_ledger_state::import_all_from_directory,
 };
-use crate::cmd::DEFAULT_NETWORK;
+use crate::cmd::{DEFAULT_NETWORK, import_nonces::import_nonces_from_file};
 use amaru::snapshots_dir;
 use amaru_kernel::{default_chain_dir, default_ledger_dir, network::NetworkName};
 use async_compression::tokio::bufread::GzipDecoder;
@@ -49,7 +47,7 @@ pub struct Args {
 
     /// Network to bootstrap the node for.
     ///
-    /// Should be one of 'mainnet', 'preprod', 'preview' or 'testnet:<magic>' where
+    /// Should be one of 'mainnet', 'preprod', 'preview' or 'testnet_<magic>' where
     /// `magic` is a 32-bits unsigned value denoting a particular testnet.
     #[arg(
         long,
@@ -114,9 +112,7 @@ async fn import_nonces_for_network(
     chain_dir: &PathBuf,
 ) -> Result<(), Box<dyn Error>> {
     let nonces_file: PathBuf = config_dir.join("nonces.json");
-    let content = tokio::fs::read_to_string(nonces_file).await?;
-    let initial_nonces: InitialNonces = serde_json::from_str(&content)?;
-    import_nonces(network.into(), chain_dir, initial_nonces).await?;
+    import_nonces_from_file(network, &nonces_file, chain_dir).await?;
     Ok(())
 }
 

crates/amaru/src/bin/amaru/cmd/convert_ledger_state.rs

@@ -13,20 +13,22 @@
 // limitations under the License.
 
 use amaru_kernel::{
-    Bound, EraHistory, EraParams, HEADER_HASH_SIZE, Hash, Nonce, Summary, cbor,
+    Bound, EraHistory, EraParams, HEADER_HASH_SIZE, Hash, Nonce, Point, Summary, cbor,
     network::NetworkName,
 };
-use amaru_ouroboros_traits::Nonces;
 use clap::Parser;
 use std::path::{Path, PathBuf};
 use tokio::fs::{self};
+use tracing::{debug, info};
+
+use crate::cmd::import_nonces::InitialNonces;
 
 #[derive(Debug, Parser)]
 pub struct Args {
     /// Path to the CBOR encoded ledger state snapshot as serialised by Haskell
     /// node.
-    #[arg(long, value_name = "SNAPSHOT", verbatim_doc_comment, num_args(0..))]
-    snapshot: Vec<PathBuf>,
+    #[arg(long, value_name = "SNAPSHOT", verbatim_doc_comment)]
+    snapshot: PathBuf,
 
     /// Directory to store converted snapshots into.
     ///
@@ -55,10 +57,7 @@ pub enum Error {
 
 pub(crate) async fn run(args: Args) -> Result<(), Box<dyn std::error::Error>> {
     let target_dir = args.target_dir.unwrap_or(PathBuf::from("."));
-    for snapshot in args.snapshot {
-        convert_one_snapshot_file(&target_dir, &snapshot, &args.network).await?;
-    }
-
+    convert_one_snapshot_file(&target_dir, &args.snapshot, &args.network).await?;
     Ok(())
 }
 
@@ -76,7 +75,12 @@ async fn convert_one_snapshot_file(
 
     fs::create_dir_all(target_dir).await?;
 
-    convert_snapshot_to(snapshot, target_dir, network).await
+    let converted = convert_snapshot_to(snapshot, target_dir, network).await?;
+    info!(
+        "converted ledger state from {:?} to {:?}",
+        snapshot, converted
+    );
+    Ok(converted)
 }
 
 async fn convert_snapshot_to(
@@ -118,7 +122,8 @@ async fn convert_snapshot_to(
             slot_length: 1000,
         },
     });
-    let history = EraHistory::new(&eras, network.default_stability_window());
+
+    let era_history = EraHistory::new(&eras, network.default_stability_window());
 
     // ledger state
     // https://github.com/abailly/ouroboros-consensus/blob/1508638f832772d21874e18e48b908fcb791cd49/ouroboros-consensus-cardano/src/shelley/Ouroboros/Consensus/Shelley/Ledger/Ledger.hs#L736
@@ -160,7 +165,7 @@ async fn convert_snapshot_to(
     d.array()?;
     // NOTE: The encoding of an AnnTip is not consistent with the encoding of a Tip
     let tip_slot = d.u64()?;
-    let _tip_hash: Hash<HEADER_HASH_SIZE> = d.decode()?;
+    let tip_hash: Hash<HEADER_HASH_SIZE> = d.decode()?;
     let _tip_height = d.u64()?;
 
     // ChainDepState for Praos
@@ -210,26 +215,37 @@ async fn convert_snapshot_to(
     d.u8()?;
     let active: Nonce = d.decode()?;
 
-    d.array()?;
-    d.u8()?;
-    let tail = d.decode()?;
+    // lab nonce
+    d.skip()?;
 
-    // previous epoch nonce, can be defined or 0
-    // FIXME: do we use it?
+    // last epoch nonce
     d.skip()?;
 
-    let nonces = Nonces {
+    let nonces = InitialNonces {
+        at: Point::Specific(tip_slot, tip_hash.to_vec()),
         active,
         evolving,
         candidate,
-        epoch: history.slot_to_epoch(tip_slot.into(), tip_slot.into())?,
-        tail,
+        tail: Hash::new([0; 32]),
     };
 
     write_nonces(target_dir, slot, hash, nonces).await?;
+    write_era_history(target_dir, slot, hash, &era_history).await?;
     write_ledger_snapshot(target_dir, slot, hash, &bytes[begin..end]).await
 }
 
+async fn write_era_history(
+    target_dir: &Path,
+    slot: u64,
+    hash: Hash<32>,
+    era_history: &EraHistory,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let target_path = target_dir.join(format!("history.{}.{}.json", slot, hash));
+    fs::write(&target_path, serde_json::to_string(era_history)?).await?;
+    debug!("wrote era history {:?}", target_path);
+    Ok(())
+}
+
 /// This is the number of past eras before the current era in the "standard" Cardano history, e.g
 /// from Byron to Babbage. Bump this number when a hard fork happens.
 pub const PAST_ERAS_NUMBER: i32 = 6;
@@ -287,10 +303,11 @@ async fn write_nonces(
     target_dir: &Path,
     slot: u64,
     hash: Hash<HEADER_HASH_SIZE>,
-    nonces: Nonces,
+    nonces: InitialNonces,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let target_path = target_dir.join(format!("nonces.{}.{}.json", slot, hash));
     fs::write(&target_path, serde_json::to_string(&nonces)?).await?;
+    debug!("wrote nonces file {:?}", target_path);
     Ok(())
 }
 
@@ -302,6 +319,7 @@ async fn write_ledger_snapshot(
 ) -> Result<PathBuf, Box<dyn std::error::Error>> {
     let target_path = target_dir.join(format!("{}.{}.cbor", slot, hash));
     fs::write(&target_path, ledger_data).await?;
+    debug!("wrote ledger snapshot {:?}", target_path);
     Ok(target_path)
 }
 
@@ -326,7 +344,7 @@ mod test {
     }
 
     #[tokio::test]
-    async fn generates_converted_snapshots_in_given_target_dir() {
+    async fn generates_converted_snapshot_in_given_target_dir() {
         let network = NetworkName::Testnet(42);
         let tempdir = tempfile::tempdir().unwrap();
         let expected_paths = vec![
@@ -341,15 +359,19 @@ mod test {
             ),
         ];
 
-        let args = super::Args {
-            snapshot: dir_content(Path::new("tests/data/convert")).await.unwrap(),
-            target_dir: Some(tempdir.path().to_path_buf()),
-            network,
-        };
+        let snapshots = dir_content(Path::new("tests/data/convert")).await.unwrap();
 
-        run(args)
-            .await
-            .expect("unexpected error in conversion test");
+        for snapshot in snapshots {
+            let args = super::Args {
+                snapshot,
+                target_dir: Some(tempdir.path().to_path_buf()),
+                network,
+            };
+
+            run(args)
+                .await
+                .expect("unexpected error in conversion test");
+        }
 
         assert!(
             expected_paths.iter().all(|p| p.exists()),
@@ -389,15 +411,19 @@ mod test {
             ),
         ];
 
-        let args = super::Args {
-            snapshot: dir_content(Path::new("tests/data/convert")).await.unwrap(),
-            target_dir: Some(target_dir.clone()),
-            network,
-        };
+        let snapshots = dir_content(Path::new("tests/data/convert")).await.unwrap();
 
-        run(args)
-            .await
-            .expect("unexpected error in conversion test");
+        for snapshot in snapshots {
+            let args = super::Args {
+                snapshot,
+                target_dir: Some(target_dir.clone()),
+                network,
+            };
+
+            run(args)
+                .await
+                .expect("unexpected error in conversion test");
+        }
 
         assert!(
             expected_paths.iter().all(|p| p.exists()),

crates/amaru/src/bin/amaru/cmd/daemon.rs

@@ -33,7 +33,7 @@ pub struct Args {
 
     /// The target network to choose from.
     ///
-    /// Should be one of 'mainnet', 'preprod', 'preview' or 'testnet:<magic>' where
+    /// Should be one of 'mainnet', 'preprod', 'preview' or 'testnet_<magic>' where
     /// `magic` is a 32-bits unsigned value denoting a particular testnet.
     #[arg(
         long,

crates/amaru/src/bin/amaru/cmd/dump_chain_db.rs

@@ -26,7 +26,7 @@ use std::{error::Error, path::PathBuf};
 pub struct Args {
     /// Network for which we are importing headers.
     ///
-    /// Should be one of 'mainnet', 'preprod', 'preview' or 'testnet:<magic>' where
+    /// Should be one of 'mainnet', 'preprod', 'preview' or 'testnet_<magic>' where
     /// `magic` is a 32-bits unsigned value denoting a particular testnet.
     #[arg(
         long,