Ocean: Initial Scan support (#2829)

* Fix next not being set

* Temporary fix of burn address

* Add blocks/:hash/transactions endpoint

* Add support for querying list Ascending

* handle ocean api network

* Don't handle including_start at the iter level

* Column implementation to fix ser/deser order issue on usize

* Index txs from 0

* Handle including_start through Paginate trait

---------

Co-authored-by: canonbrother <w.canonbrother@gmail.com>

Author: Jouzo

lib/Cargo.lock

@@ -9,8 +9,8 @@ use std::{
 use anyhow::format_err;
 use bincode;
 use rocksdb::{
-    BlockBasedOptions, Cache, ColumnFamily, ColumnFamilyDescriptor, DBIterator, IteratorMode,
-    Options, DB,
+    BlockBasedOptions, Cache, ColumnFamily, ColumnFamilyDescriptor, DBIterator, Direction,
+    IteratorMode, Options, DB,
 };
 use serde::{de::DeserializeOwned, Serialize};
 
@@ -181,16 +181,22 @@ where
     pub fn iter(
         &self,
         from: Option<C::Index>,
+        direction: Direction,
     ) -> Result<impl Iterator<Item = Result<(C::Index, C::Type)>> + '_> {
-        let skip = if from.as_ref().is_some() { 1 } else { 0 };
         let index = from
             .as_ref()
             .map(|i| C::key(i))
             .transpose()?
             .unwrap_or_default();
-        let iterator_mode = from.map_or(IteratorMode::End, |_| {
-            IteratorMode::From(&index, rocksdb::Direction::Reverse)
-        });
+
+        let iterator_mode = match direction {
+            Direction::Forward => from.map_or(IteratorMode::Start, |_| {
+                IteratorMode::From(&index, Direction::Forward)
+            }),
+            Direction::Reverse => from.map_or(IteratorMode::End, |_| {
+                IteratorMode::From(&index, Direction::Reverse)
+            }),
+        };
         Ok(self
             .backend
             .iterator_cf::<C>(self.handle()?, iterator_mode)
@@ -199,8 +205,7 @@ where
                 let value = bincode::deserialize(&value)?;
                 let key = C::get_key(key)?;
                 Ok((key, value))
-            })
-            .skip(skip))
+            }))
     }
 }
 

lib/ain-db/src/lib.rs

@@ -237,7 +237,10 @@ impl Rollback for BlockStore {
             let block_deployed_codes_cf = self.column::<columns::BlockDeployedCodeHashes>();
             let address_codes_cf = self.column::<columns::AddressCodeMap>();
 
-            for item in block_deployed_codes_cf.iter(Some((block.header.number, H160::zero())))? {
+            for item in block_deployed_codes_cf.iter(
+                Some((block.header.number, H160::zero())),
+                rocksdb::Direction::Reverse,
+            )? {
                 let ((block_number, address), hash) = item?;
 
                 if block_number == block.header.number {
@@ -325,7 +328,11 @@ impl BlockStore {
         let response_max_size = usize::try_from(ain_cpp_imports::get_max_response_byte_size())
             .map_err(|_| format_err!("failed to convert response size limit to usize"))?;
 
-        for item in self.column::<C>().iter(from)?.take(limit) {
+        for item in self
+            .column::<C>()
+            .iter(from, rocksdb::Direction::Reverse)?
+            .take(limit)
+        {
             let (k, v) = item?;
 
             if out.len() > response_max_size {

lib/ain-evm/src/storage/block_store.rs

@@ -113,9 +113,9 @@ pub fn repository_derive(input: TokenStream) -> TokenStream {
                 Ok(self.col.delete(id)?)
             }
 
-            fn list<'a>(&'a self, from: Option<#key_type_ident>) -> Result<Box<dyn Iterator<Item = std::result::Result<(#key_type_ident, #value_type_ident), ain_db::DBError>> + 'a>>
+            fn list<'a>(&'a self, from: Option<#key_type_ident>, dir: crate::storage::SortOrder) -> Result<Box<dyn Iterator<Item = std::result::Result<(#key_type_ident, #value_type_ident), ain_db::DBError>> + 'a>>
             {
-                let it = self.col.iter(from)?;
+                let it = self.col.iter(from, dir.into())?;
                 Ok(Box::new(it))
             }
         }

lib/ain-macros/src/lib.rs

@@ -12,9 +12,11 @@ use super::{
     AppContext,
 };
 use crate::{
+    api::common::Paginate,
     error::{ApiError, Error},
-    model::Block,
+    model::{Block, Transaction},
     repository::RepositoryOps,
+    storage::SortOrder,
     Result,
 };
 
@@ -46,6 +48,7 @@ async fn list_blocks(
 ) -> Result<ApiPagedResponse<Block>> {
     let next = query
         .next
+        .as_ref()
         .map(|q| {
             let height = q
                 .parse::<u32>()
@@ -58,8 +61,8 @@ async fn list_blocks(
         .services
         .block
         .by_height
-        .list(next)?
-        .take(query.size)
+        .list(next, SortOrder::Descending)?
+        .paginate(&query)
         .map(|item| {
             let (_, id) = item?;
             let b = ctx
@@ -95,11 +98,43 @@ async fn get_block(
     Ok(Response::new(block))
 }
 
+#[ocean_endpoint]
 async fn get_transactions(
     Path(hash): Path<BlockHash>,
+    Query(query): Query<PaginationQuery>,
     Extension(ctx): Extension<Arc<AppContext>>,
-) -> String {
-    format!("Transactions for block with hash {}", hash)
+) -> Result<ApiPagedResponse<Transaction>> {
+    let next = query.next.as_ref().map_or(Ok((hash, 0)), |q| {
+        let height = q
+            .parse::<usize>()
+            .map_err(|_| format_err!("Invalid height"))?;
+        Ok::<(BlockHash, usize), Error>((hash, height))
+    })?;
+
+    let txs = ctx
+        .services
+        .transaction
+        .by_block_hash
+        .list(Some(next), SortOrder::Ascending)?
+        .paginate(&query)
+        .take_while(|item| match item {
+            Ok(((h, _), _)) => h == &hash,
+            _ => true,
+        })
+        .map(|item| {
+            let (_, id) = item?;
+            let tx = ctx
+                .services
+                .transaction
+                .by_id
+                .get(&id)?
+                .ok_or("Missing tx index")?;
+
+            Ok(tx)
+        })
+        .collect::<Result<Vec<_>>>()?;
+
+    Ok(ApiPagedResponse::of(txs, query.size, |tx| tx.order))
 }
 
 // Get highest indexed block

lib/ain-ocean/src/api/block.rs

@@ -107,27 +107,32 @@ pub fn find_token_balance(tokens: Vec<String>, symbol: &str) -> Decimal {
 ///     .list_loan_schemes()
 ///     .await?
 ///     .into_iter()
-///     .paginate(&query, skip_while)
+///     .fake_paginate(&query, skip_while)
 ///     .collect();
 ///
 /// assert!(res.len() <= query.size, "The result should not contain more items than the specified limit");
 /// assert!(res[0].id > query.next.unwrap(), "The result should start after the requested start id");
 /// ```
 pub trait Paginate<'a, T>: Iterator<Item = T> + Sized {
-    fn paginate<F>(
+    fn fake_paginate<F>(
         self,
         query: &PaginationQuery,
         skip_while: F,
     ) -> Box<dyn Iterator<Item = T> + 'a>
     where
         F: FnMut(&T) -> bool + 'a;
+    fn paginate(self, query: &PaginationQuery) -> Box<dyn Iterator<Item = T> + 'a>;
 }
 
 impl<'a, T, I> Paginate<'a, T> for I
 where
     I: Iterator<Item = T> + 'a,
 {
-    fn paginate<F>(self, query: &PaginationQuery, skip_while: F) -> Box<dyn Iterator<Item = T> + 'a>
+    fn fake_paginate<F>(
+        self,
+        query: &PaginationQuery,
+        skip_while: F,
+    ) -> Box<dyn Iterator<Item = T> + 'a>
     where
         F: FnMut(&T) -> bool + 'a,
     {
@@ -137,4 +142,7 @@ where
                 .take(query.size),
         )
     }
+    fn paginate(self, query: &PaginationQuery) -> Box<dyn Iterator<Item = T> + 'a> {
+        Box::new(self.skip(query.next.is_some() as usize).take(query.size))
+    }
 }

lib/ain-ocean/src/api/common.rs

@@ -27,6 +27,7 @@ use crate::{
     error::{ApiError, Error, NotFoundKind},
     model::VaultAuctionBatchHistory,
     repository::RepositoryOps,
+    storage::SortOrder,
     Result,
 };
 
@@ -63,7 +64,7 @@ async fn list_scheme(
         .list_loan_schemes()
         .await?
         .into_iter()
-        .paginate(&query, skip_while)
+        .fake_paginate(&query, skip_while)
         .map(Into::into)
         .collect();
     Ok(ApiPagedResponse::of(res, query.size, |loan_scheme| {
@@ -76,7 +77,6 @@ async fn get_scheme(
     Path(scheme_id): Path<String>,
     Extension(ctx): Extension<Arc<AppContext>>,
 ) -> Result<Response<LoanSchemeData>> {
-    println!("[get_scheme]");
     Ok(Response::new(
         ctx.client.get_loan_scheme(scheme_id).await?.into(),
     ))
@@ -120,7 +120,7 @@ async fn list_collateral_token(
 
     let fut = tokens
         .into_iter()
-        .paginate(&query, skip_while)
+        .fake_paginate(&query, skip_while)
         .map(|v| async {
             let (id, info) = get_token_cached(&ctx, &v.token_id).await?;
             Ok::<CollateralToken, Error>(CollateralToken::from_with_id(id, v, info))
@@ -187,7 +187,7 @@ async fn list_loan_token(
                     interest: el.interest,
                 })
         })
-        .paginate(&query, |token| match &query.next {
+        .fake_paginate(&query, |token| match &query.next {
             None => false,
             Some(v) => v != &token.id,
         })
@@ -259,7 +259,10 @@ async fn list_vault_auction_history(
         .services
         .auction
         .by_height
-        .list(Some((vault_id, batch_index, next.0, next.1)))?
+        .list(
+            Some((vault_id, batch_index, next.0, next.1)),
+            SortOrder::Descending,
+        )?
         .take(size)
         .take_while(|item| match item {
             Ok((k, _)) => k.0 == vault_id && k.1 == batch_index,

lib/ain-ocean/src/api/loan.rs

@@ -16,9 +16,11 @@ use super::{
     AppContext,
 };
 use crate::{
+    api::common::Paginate,
     error::{ApiError, Error, NotFoundKind},
     model::Masternode,
     repository::RepositoryOps,
+    storage::SortOrder,
     Result,
 };
 
@@ -105,6 +107,7 @@ async fn list_masternodes(
 ) -> Result<ApiPagedResponse<MasternodeData>> {
     let next = query
         .next
+        .as_ref()
         .map(|q| {
             let height = q[0..8]
                 .parse::<u32>()
@@ -121,8 +124,8 @@ async fn list_masternodes(
         .services
         .masternode
         .by_height
-        .list(next)?
-        .take(query.size)
+        .list(next, SortOrder::Descending)?
+        .paginate(&query)
         .map(|item| {
             let ((_, id), _) = item?;
             let mn = ctx

lib/ain-ocean/src/api/masternode.rs

@@ -18,7 +18,7 @@ mod query;
 mod response;
 mod stats;
 mod tokens;
-// mod transactions;
+mod transactions;
 
 use defichain_rpc::Client;
 use serde::{Deserialize, Serialize};
@@ -59,19 +59,24 @@ pub async fn ocean_router(services: &Arc<Services>, client: Arc<Client>) -> Resu
         services: services.clone(),
     });
 
-    Ok(Router::new()
-        // .nest("/address", address::router(Arc::clone(&context)))
-        .nest("/governance", governance::router(Arc::clone(&context)))
-        .nest("/loans", loan::router(Arc::clone(&context)))
-        .nest("/fee", fee::router(Arc::clone(&context)))
-        .nest("/masternodes", masternode::router(Arc::clone(&context)))
-        // .nest("/oracles", oracle::router(Arc::clone(&context)))
-        // .nest("/poolpairs", poolpairs::router(Arc::clone(&context)))
-        // .nest("/prices", prices::router(Arc::clone(&context)))
-        // .nest("/rawtx", rawtx::router(Arc::clone(&context)))
-        .nest("/stats", stats::router(Arc::clone(&context)))
-        .nest("/tokens", tokens::router(Arc::clone(&context)))
-        // .nest("/transactions", transactions::router(Arc::clone(&context)))
-        .nest("/blocks", block::router(Arc::clone(&context)))
-        .fallback(not_found))
+    let network = ain_cpp_imports::get_network();
+
+    Ok(Router::new().nest(
+        format!("/v0/{network}").as_str(),
+        Router::new()
+            // .nest("/address/", address::router(Arc::clone(&context)))
+            .nest("/governance", governance::router(Arc::clone(&context)))
+            .nest("/loans", loan::router(Arc::clone(&context)))
+            .nest("/fee", fee::router(Arc::clone(&context)))
+            .nest("/masternodes", masternode::router(Arc::clone(&context)))
+            // .nest("/oracles", oracle::router(Arc::clone(&context)))
+            // .nest("/poolpairs", poolpairs::router(Arc::clone(&context)))
+            // .nest("/prices", prices::router(Arc::clone(&context)))
+            // .nest("/rawtx", rawtx::router(Arc::clone(&context)))
+            .nest("/stats", stats::router(Arc::clone(&context)))
+            .nest("/tokens", tokens::router(Arc::clone(&context)))
+            .nest("/transactions", transactions::router(Arc::clone(&context)))
+            .nest("/blocks", block::router(Arc::clone(&context)))
+            .fallback(not_found),
+    ))
 }

lib/ain-ocean/src/api/mod.rs

@@ -23,6 +23,7 @@ pub struct PaginationQuery {
     #[serde_as(as = "DisplayFromStr")]
     #[serde(default = "default_pagination_size")]
     pub size: usize,
+    #[serde(default)]
     #[serde(deserialize_with = "undefined_to_none")]
     pub next: Option<String>,
 }