Merge branch 'payref' of https://github.com/fluffypony/tari into payref

Author: fluffypony

.github/workflows/build_binaries.yml

@@ -31,7 +31,7 @@ env:
   # TS_BUILD: "debug"
   TS_BUILD: "release"
   TARI_NETWORK_DIR: testnet
-  toolchain: nightly-2024-07-07
+  toolchain: nightly-2025-01-01
   matrix-json-file: ".github/workflows/build_binaries.json"
   CARGO_HTTP_MULTIPLEXING: false
   CARGO_UNSTABLE_SPARSE_REGISTRY: true

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ name: CI
 env:
   toolchain: stable
   nightly_toolchain: nightly-2025-01-01
-  nightly_test_toolchain: nightly-2024-08-03
+  nightly_test_toolchain: nightly-2025-01-01
   CARGO_HTTP_MULTIPLEXING: false
   CARGO_TERM_COLOR: always
   CARGO_UNSTABLE_SPARSE_REGISTRY: true

.github/workflows/integration_tests.yml

@@ -28,7 +28,7 @@ name: Integration tests
 
 env:
   toolchain: stable
-  nightly_toolchain: nightly-2024-07-07
+  nightly_toolchain: nightly-2025-01-01
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

base_layer/core/src/chain_storage/blockchain_database.rs

@@ -1805,7 +1805,7 @@ fn check_for_valid_height<T: BlockchainBackend>(db: &T, height: u64) -> Result<(
 /// Removes blocks from the db from current tip to specified height.
 /// Returns the blocks removed, ordered from tip to height.
 #[allow(clippy::too_many_lines)]
-fn rewind_to_height<T: BlockchainBackend>(
+pub(crate) fn rewind_to_height<T: BlockchainBackend>(
     db: &mut T,
     target_height: u64,
 ) -> Result<Vec<Arc<ChainBlock>>, ChainStorageError> {

base_layer/core/src/chain_storage/lmdb_db/lmdb_db.rs

@@ -78,6 +78,7 @@ use crate::{
         UpdateBlockAccumulatedData,
     },
     chain_storage::{
+        blockchain_database::rewind_to_height,
         db_transaction::{DbKey, DbTransaction, DbValue, WriteOperation},
         error::{ChainStorageError, OrNotFound},
         lmdb_db::{
@@ -311,7 +312,7 @@ impl LMDBDatabase {
         consensus_manager: ConsensusManager,
     ) -> Result<Self, ChainStorageError> {
         let env = store.env();
-        let db = Self {
+        let mut db = Self {
             metadata_db: get_database(store, LMDB_DB_METADATA)?,
             headers_db: get_database(store, LMDB_DB_HEADERS)?,
             header_accumulated_data_db: get_database(store, LMDB_DB_HEADER_ACCUMULATED_DATA)?,
@@ -350,7 +351,7 @@ impl LMDBDatabase {
             consensus_manager,
         };
 
-        run_migrations(&db)?;
+        run_migrations(&mut db)?;
 
         Ok(db)
     }
@@ -2954,8 +2955,8 @@ impl fmt::Display for MetadataValue {
 }
 
 #[allow(clippy::too_many_lines)]
-fn run_migrations(db: &LMDBDatabase) -> Result<(), ChainStorageError> {
-    const MIGRATION_VERSION: u64 = 2;
+fn run_migrations(db: &mut LMDBDatabase) -> Result<(), ChainStorageError> {
+    const MIGRATION_VERSION: u64 = 3;
     let txn = db.read_transaction()?;
     let k = MetadataKey::MigrationVersion;
     let val = lmdb_get::<_, MetadataValue>(&txn, &db.metadata_db, &k.as_u32())?;
@@ -3083,8 +3084,7 @@ fn run_migrations(db: &LMDBDatabase) -> Result<(), ChainStorageError> {
             }
             txn.commit()?;
         }
-        
-        if migrate_from_version == 1 {
+        if migrate_from_version == 2 {
             // Migration v1 -> v2: Rebuild PayRef index for existing nodes
             info!(target: LOG_TARGET, "Migrating to v2: Checking if PayRef index needs rebuilding");
 
@@ -3159,6 +3159,39 @@ fn run_migrations(db: &LMDBDatabase) -> Result<(), ChainStorageError> {
             }
 
             info!(target: LOG_TARGET, "PayRef index rebuild completed: {} outputs indexed", rebuild_count);
+
+        if migrate_from_version == 1 {
+            let known_good_difficulties = get_correct_accumulated_difficulty();
+            if known_good_difficulties.is_empty() {
+                info!(target: LOG_TARGET, "No migration to perform for version network");
+                continue;
+            }
+            let mut last_correct_height = 0;
+            for (height, correct_difficulty) in known_good_difficulties {
+                let txn = db.read_transaction()?;
+                let accum_data: Option<BlockHeaderAccumulatedData> =
+                    lmdb_get(&txn, &db.header_accumulated_data_db, &height)?;
+                if let Some(accum_data) = accum_data {
+                    if accum_data.total_accumulated_difficulty == correct_difficulty {
+                        info!(
+                            target: LOG_TARGET,
+                            "Block height {} already has correct accumulated difficulty",
+                            height
+                        );
+                        last_correct_height = height;
+                    }
+                } else {
+                    info!(target: LOG_TARGET, "No accumulated difficulty found for block height {}", height);
+                    break;
+                }
+            }
+            if last_correct_height == 0 {
+                // this will happen only happen if the db is below the fork height of the RxT fork
+                info!(target: LOG_TARGET, "No migration to perform for version network");
+                continue;
+            }
+            // lets rewind to last known good accumulated difficulty so the db can be correctly calculated again
+            rewind_to_height(db, last_correct_height)?;
         }
     }
     if last_migrated_version != MIGRATION_VERSION {
@@ -3216,3 +3249,54 @@ pub struct OldChainTipData {
     pub hash: HashOutput,
     pub total_accumulated_difficulty: U256,
 }
+
+fn get_correct_accumulated_difficulty() -> Vec<(u64, U512)> {
+    #[cfg(tari_target_network_mainnet)]
+    {
+        vec![
+            (
+                14999,
+                U512::from_dec_str("230963847231029670329787338266632060").expect("should not fail"),
+            ),
+            (
+                16000,
+                U512::from_dec_str("37870972808147006178902366165325544920691850526080").expect("should not fail"),
+            ),
+            (
+                17000,
+                U512::from_dec_str("123219722351554302645774736761840507999792186766920").expect("should not fail"),
+            ),
+            (
+                18000,
+                U512::from_dec_str("245169616636012105701848119083014332169855273375890").expect("should not fail"),
+            ),
+            (
+                19000,
+                U512::from_dec_str("428081108397470519627923902616128115025981546384670").expect("should not fail"),
+            ),
+            (
+                20000,
+                U512::from_dec_str("678404434598953994059276298108149917133080906779800").expect("should not fail"),
+            ),
+        ]
+    }
+    #[cfg(tari_target_network_nextnet)]
+    {
+        vec![
+            (
+                1499,
+                U512::from_dec_str("17340317256602964156796").expect("should not fail"),
+            ),
+            (
+                2000,
+                U512::from_dec_str("267045542397987769905169797604842").expect("should not fail"),
+            ),
+            (
+                3000,
+                U512::from_dec_str("2261524423095838119669981829692352").expect("should not fail"),
+            ),
+        ]
+    }
+    #[cfg(not(any(tari_target_network_mainnet, tari_target_network_nextnet)))]
+    vec![]
+}

base_layer/core/src/chain_storage/target_difficulties.rs

@@ -47,9 +47,8 @@ impl TargetDifficulties {
     }
 
     pub fn update_algos(&mut self, consensus_rules: &ConsensusManager, height: u64) -> Result<(), String> {
-        let permitted_algos = consensus_rules
-            .consensus_constants(height)
-            .current_permitted_pow_algos();
+        let consensus_constants = consensus_rules.consensus_constants(height);
+        let permitted_algos = consensus_constants.current_permitted_pow_algos();
         let current_keys: Vec<PowAlgorithm> = self.algos.keys().copied().collect();
         for algo in current_keys {
             if !permitted_algos.contains(&algo) {
@@ -60,6 +59,10 @@ impl TargetDifficulties {
             if let std::collections::hash_map::Entry::Vacant(e) = self.algos.entry(algo) {
                 let target_difficulty_window = consensus_rules.new_target_difficulty(algo, height)?;
                 e.insert(target_difficulty_window);
+            } else if let Some(target_diff) = self.algos.get_mut(&algo) {
+                target_diff.update_target_time(consensus_constants.pow_target_block_interval(algo))?
+            } else {
+                // clippy, this else should never be hit
             }
         }
         Ok(())

base_layer/core/src/proof_of_work/lwma_diff.rs

@@ -176,6 +176,26 @@ impl LinearWeightedMovingAverage {
         }
         self.target_difficulties.push_back((timestamp, target_difficulty));
     }
+
+    pub fn update_target_time(&mut self, target_time: u64) -> Result<(), String> {
+        if target_time == 0 {
+            return Err(
+                "LinearWeightedMovingAverage::update_target_time(...) expected `target_time` to be greater than 0, \
+                 but 0 was given"
+                    .into(),
+            );
+        }
+        if target_time.checked_mul(LWMA_MAX_BLOCK_TIME_RATIO).is_none() {
+            return Err(format!(
+                "LinearWeightedMovingAverage::update_target_time(...) expected `target_time` to be at least {} times \
+                 smaller than `u64::MAX`",
+                LWMA_MAX_BLOCK_TIME_RATIO,
+            ));
+        }
+        self.target_time = u128::from(target_time);
+        self.max_block_time = target_time * LWMA_MAX_BLOCK_TIME_RATIO;
+        Ok(())
+    }
 }
 
 impl DifficultyAdjustment for LinearWeightedMovingAverage {

base_layer/core/src/proof_of_work/target_difficulty_window.rs

@@ -76,6 +76,10 @@ impl TargetDifficultyWindow {
         let difficulty = self.lwma.get_difficulty().unwrap_or(min);
         cmp::max(min, cmp::min(max, difficulty))
     }
+
+    pub fn update_target_time(&mut self, target_time: u64) -> Result<(), String> {
+        self.lwma.update_target_time(target_time)
+    }
 }
 
 #[cfg(test)]

base_layer/core/tests/tests/node_state_machine.rs

@@ -284,6 +284,17 @@ async fn test_listening_initial_fallen_behind() {
 
 #[tokio::test]
 async fn test_event_channel() {
+    let network = Network::Esmeralda;
+    if std::env::var("TARI_NETWORK").is_err() {
+        std::env::set_var("TARI_NETWORK", network.as_key_str());
+    }
+    if Network::get_current_or_user_setting_or_default() != network {
+        let _ = Network::set_current(network);
+    }
+    let current_network = Network::get_current_or_user_setting_or_default();
+    if current_network != network {
+        panic!("could not set network");
+    }
     // env_logger::init(); // Set `$env:RUST_LOG = "trace"`
     let temp_dir = tempdir().unwrap();
     let (node, consensus_manager) = BaseNodeBuilder::new(Network::Esmeralda.into())

base_layer/tari_mining_helper_ffi/src/lib.rs

@@ -150,7 +150,7 @@ pub unsafe extern "C" fn byte_vector_get_at(ptr: *mut ByteVector, position: c_ui
         ptr::swap(error_out, &mut error as *mut c_int);
         return 0u8;
     }
-    (*ptr).0[position as usize]
+    (&(*ptr).0)[position as usize]
 }
 
 /// Gets the number of elements in a ByteVector