Ocean: block indexing (#2777)

* experimental_allow_proto3_optional

* ffi blockv2info

* update ocean block apis

* pindex->stakeModifier

* rm block.reward

* add missing column block_by_height

* rm unuse

Author: canonbrother

lib/ain-grpc/build.rs

@@ -266,6 +266,7 @@ fn compile_proto_and_generate_services(
         // There's no way to compile protos using custom generator in tonic,
         // so we're left with creating a prost config and using that for codegen.
         let mut config = Config::new();
+        config.protoc_arg("--experimental_allow_proto3_optional");
         config.out_dir(out_dir);
         config.service_generator(Box::new(gen));
         config

lib/ain-ocean/src/api/block.rs

@@ -3,40 +3,54 @@ use axum::{
     routing::get,
     Json, Router,
 };
-use serde::{Deserialize, Serialize};
-
-use crate::{api_paged_response::ApiPagedResponse, api_query::PaginationQuery, error::OceanResult};
-
-#[derive(Deserialize)]
-struct BlockId {
-    id: String,
-}
-
-#[derive(Deserialize)]
-struct BlockHash {
-    hash: String,
-}
+use bitcoin::BlockHash;
+
+use crate::{
+    api_paged_response::ApiPagedResponse,
+    api_query::PaginationQuery,
+    error::OceanResult,
+    SERVICES,
+    repository::RepositoryOps,
+    model::Block,
+};
 
 async fn list_blocks(
     Query(query): Query<PaginationQuery>,
 ) -> OceanResult<Json<ApiPagedResponse<Block>>> {
-    // TODO(): query from lvldb.. or maybe pull from index
-    let blocks = vec![
-        Block { id: "0".into() },
-        Block { id: "1".into() },
-        Block { id: "2".into() },
-    ];
-
-    Ok(Json(ApiPagedResponse::of(blocks, query.size, |block| {
-        block.clone().id
-    })))
+    let blocks = SERVICES
+        .block
+        .by_height
+        .list(None)?
+        .take(query.size)
+        .map(|item| {
+            let (_, id) = item?;
+            let b = SERVICES
+                .block
+                .by_id
+                .get(&id)?
+                .ok_or("Missing block index")?;
+
+            Ok(b)
+        })
+        .collect::<OceanResult<Vec<_>>>()?;
+
+    Ok(Json(ApiPagedResponse::of(
+        blocks,
+        query.size,
+        |block| block.clone().id,
+    )))
 }
 
-async fn get_block(Path(BlockId { id }): Path<BlockId>) -> OceanResult<Json<Block>> {
-    Ok(Json(Block { id }))
+async fn get_block(Path(id): Path<BlockHash>) -> OceanResult<Json<Option<Block>>> {
+    let block = SERVICES
+        .block
+        .by_id
+        .get(&id)?;
+
+    Ok(Json(block))
 }
 
-async fn get_transactions(Path(BlockHash { hash }): Path<BlockHash>) -> String {
+async fn get_transactions(Path(hash): Path<BlockHash>) -> String {
     format!("Transactions for block with hash {}", hash)
 }
 
@@ -46,33 +60,3 @@ pub fn router() -> Router {
         .route("/:id", get(get_block))
         .route("/:hash/transactions", get(get_transactions))
 }
-
-#[derive(Clone, Debug, Serialize)]
-#[serde(default)]
-pub struct Block {
-    id: String,
-    // TODO(): type mapping
-    // hash: H256,
-    // previous_hash: H256,
-
-    // height: u64,
-    // version: u64,
-    // time: u64, // ---------------| block time in seconds since epoch
-    // median_time: u64, // --------| meidan time of the past 11 block timestamps
-
-    // transaction_count: u64,
-
-    // difficulty: u64,
-
-    // masternode: H256,
-    // minter: H256,
-    // minter_block_count: u64,
-    // reward: f64
-
-    // state_modifier: H256,
-    // merkle_root: H256,
-
-    // size: u64,
-    // size_stripped: u64,
-    // weight: u64,
-}

lib/ain-ocean/src/indexer/mod.rs

@@ -13,21 +13,64 @@ pub(crate) trait Index {
 use dftx_rs::{deserialize, Block, DfTx};
 use log::debug;
 
-use crate::{model::BlockContext, Result};
+use crate::{
+    model::{BlockContext, Block as BlockMapper},
+    repository::RepositoryOps,
+    Result,
+    SERVICES,
+};
 
-pub fn index_block(block: String, block_height: u32) -> Result<()> {
+pub struct BlockV2Info {
+    pub height: u32,
+    pub difficulty: u32,
+    pub version: i32,
+    pub median_time: i64,
+    pub minter_block_count: u64,
+    pub size: usize,
+    pub size_stripped: usize,
+    pub weight: i64,
+    pub stake_modifier: String,
+    pub minter: String,
+    pub masternode: String,
+}
+
+pub fn index_block(encoded_block: String, info: &BlockV2Info) -> Result<()> {
     debug!("[index_block] Indexing block...");
 
-    let hex = hex::decode(block)?;
+    let hex = hex::decode(&encoded_block)?;
     debug!("got hex");
     let block = deserialize::<Block>(&hex)?;
     debug!("got block");
+    let block_hash = block.block_hash();
     let ctx = BlockContext {
-        height: block_height,
-        hash: block.block_hash(),
+        height: info.height,
+        hash: block_hash,
         time: 0,        // TODO
         median_time: 0, // TODO
     };
+    let block_mapper = BlockMapper {
+        id: block_hash.to_string(),
+        hash: block_hash.to_string(),
+        previous_hash: block.header.prev_blockhash.to_string(),
+        height: info.height,
+        version: info.version,
+        time: block.header.time,
+        median_time: info.median_time,
+        transaction_count: block.txdata.len(),
+        difficulty: info.difficulty,
+        masternode: info.masternode.to_owned(),
+        minter: info.minter.to_owned(),
+        minter_block_count: info.minter_block_count,
+        stake_modifier: info.stake_modifier.to_owned(),
+        merkleroot: block.header.merkle_root.to_string(),
+        size: info.size,
+        size_stripped: info.size_stripped,
+        weight: info.weight,
+    };
+
+    SERVICES.block.raw.put(&ctx.hash,  &encoded_block)?;
+    SERVICES.block.by_id.put(&ctx.hash, &block_mapper)?;
+    SERVICES.block.by_height.put(&ctx.height, &block_hash)?;
 
     for (idx, tx) in block.txdata.into_iter().enumerate() {
         let bytes = tx.output[0].script_pubkey.as_bytes();
@@ -60,6 +103,6 @@ pub fn index_block(block: String, block_height: u32) -> Result<()> {
     Ok(())
 }
 
-pub fn invalidate_block(block: String, block_height: u32) -> Result<()> {
+pub fn invalidate_block(block: String, info: &BlockV2Info) -> Result<()> {
     Ok(())
 }

lib/ain-ocean/src/lib.rs

@@ -6,7 +6,7 @@ mod indexer;
 use std::{path::PathBuf, sync::Arc};
 
 pub use api::ocean_router;
-pub use indexer::{index_block, invalidate_block};
+pub use indexer::{BlockV2Info, index_block, invalidate_block};
 use repository::{
     AuctionHistoryByHeightRepository, AuctionHistoryRepository, BlockByHeightRepository,
     BlockRepository, MasternodeByHeightRepository, MasternodeRepository, MasternodeStatsRepository,

lib/ain-ocean/src/model/block.rs

@@ -7,21 +7,20 @@ pub struct Block {
     pub id: String,
     pub hash: String,
     pub previous_hash: String,
-    pub height: i32,
+    pub height: u32,
     pub version: i32,
-    pub time: i32,
-    pub median_time: i32,
-    pub transaction_count: i32,
-    pub difficulty: i32,
+    pub time: u32,
+    pub median_time: i64,
+    pub transaction_count: usize,
+    pub difficulty: u32,
     pub masternode: String,
     pub minter: String,
-    pub minter_block_count: i32,
-    pub reward: String,
+    pub minter_block_count: u64,
     pub stake_modifier: String,
     pub merkleroot: String,
-    pub size: i32,
-    pub size_stripped: i32,
-    pub weight: i32,
+    pub size: usize,
+    pub size_stripped: usize,
+    pub weight: i64,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]

lib/ain-ocean/src/storage/columns/mod.rs

@@ -41,8 +41,9 @@ pub use transaction_vin::*;
 pub use transaction_vout::*;
 pub use vault_auction_history::*;
 
-pub const COLUMN_NAMES: [&'static str; 22] = [
+pub const COLUMN_NAMES: [&'static str; 23] = [
     block::Block::NAME,
+    block::BlockByHeight::NAME,
     masternode_stats::MasternodeStats::NAME,
     masternode::Masternode::NAME,
     masternode::MasternodeByHeight::NAME,

lib/ain-rs-exports/src/lib.rs

@@ -55,6 +55,21 @@ pub mod ffi {
         fn ain_rs_stop_network_services(result: &mut CrossBoundaryResult);
     }
 
+    #[derive(Default)]
+    pub struct BlockV2Info {
+        pub height: u32,
+        pub difficulty: u32,
+        pub version: i32,
+        pub median_time: i64,
+        pub minter_block_count: u64,
+        pub size: usize,
+        pub size_stripped: usize,
+        pub weight: i64,
+        pub stake_modifier: String,
+        pub minter: String,
+        pub masternode: String,
+    }
+
     // ========== Block ==========
     #[derive(Default)]
     pub struct EVMBlockHeader {
@@ -346,11 +361,11 @@ pub mod ffi {
 
         fn evm_try_flush_db(result: &mut CrossBoundaryResult);
 
-        fn ocean_index_block(result: &mut CrossBoundaryResult, block: String, block_height: u32);
+        fn ocean_index_block(result: &mut CrossBoundaryResult, block: String, info: &BlockV2Info);
         fn ocean_invalidate_block(
             result: &mut CrossBoundaryResult,
             block: String,
-            block_height: u32,
+            info: &BlockV2Info,
         );
     }
 

lib/ain-rs-exports/src/ocean.rs

@@ -1,7 +1,25 @@
-use crate::ffi::CrossBoundaryResult;
+use crate::ffi::{BlockV2Info as BlockV2InfoFFI, CrossBoundaryResult};
+use ain_ocean::BlockV2Info;
 
-pub fn ocean_index_block(result: &mut CrossBoundaryResult, block: String, block_height: u32) {
-    match ain_ocean::index_block(block, block_height) {
+// manually convert since BlockV2InfoFFI is belongs to CPP which can't impl From -> Into
+pub fn convert(b: &BlockV2InfoFFI) -> BlockV2Info {
+    BlockV2Info {
+        height: b.height,
+        difficulty: b.difficulty,
+        version: b.version,
+        median_time: b.median_time,
+        minter_block_count: b.minter_block_count,
+        size: b.size,
+        size_stripped: b.size_stripped,
+        weight: b.weight,
+        stake_modifier: b.stake_modifier.to_owned(),
+        minter: b.minter.to_owned(),
+        masternode: b.masternode.to_owned(),
+    }
+}
+
+pub fn ocean_index_block(result: &mut CrossBoundaryResult, block: String, b: &BlockV2InfoFFI) {
+    match ain_ocean::index_block(block, &convert(b)) {
         Ok(()) => result.ok = true,
         Err(e) => {
             result.ok = false;
@@ -10,8 +28,8 @@ pub fn ocean_index_block(result: &mut CrossBoundaryResult, block: String, block_
     }
 }
 
-pub fn ocean_invalidate_block(result: &mut CrossBoundaryResult, block: String, block_height: u32) {
-    match ain_ocean::invalidate_block(block, block_height) {
+pub fn ocean_invalidate_block(result: &mut CrossBoundaryResult, block: String, b: &BlockV2InfoFFI) {
+    match ain_ocean::invalidate_block(block, &convert(b)) {
         Ok(()) => result.ok = true,
         Err(e) => {
             result.ok = false;

src/dfi/validation.cpp

@@ -21,6 +21,7 @@
 #include <ffi/ffiexports.h>
 #include <ffi/ffihelpers.h>
 #include <validation.h>
+#include <consensus/validation.h>
 
 #include <boost/asio.hpp>
 
@@ -2798,7 +2799,19 @@ Res ProcessDeFiEventFallible(const CBlock &block,
         std::string serializedData = HexStr(ss.begin(), ss.end());
 
         CrossBoundaryResult result;
-        ocean_index_block(result, serializedData, pindex->nHeight);
+        BlockV2Info info;
+        info.height = pindex->nHeight;
+        info.difficulty = pindex->nBits;
+        info.version = pindex->nVersion;
+        info.median_time = (int64_t)pindex->GetMedianTimePast();
+        info.minter_block_count = pindex->mintedBlocks;
+        info.size = GetSerializeSize(block, PROTOCOL_VERSION);
+        info.size_stripped = GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS);
+        info.weight = GetBlockWeight(block);
+        info.stake_modifier = pindex->stakeModifier.ToString();
+        info.minter = "", // mn operator address
+        info.masternode = "", // mn owner address
+        ocean_index_block(result, serializedData, info);
         // if (!result.ok) {
         //     return Res::Err(result.reason.c_str());
         // }