feat: add query.GenericFilteredPaginated (backport #12253) (#12371)

mergify[bot] · web-flow · commit f2d94445c0f5 · 2022-06-28T12:42:01.000-04:00
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -14,7 +14,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v2.1.4
         with:
-          go-version: 1.17
+          go-version: 1.18
       - uses: technote-space/get-diff-action@v5
         id: git_diff
         with:
diff --git a/.github/workflows/release-sims.yml b/.github/workflows/release-sims.yml
@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: install runsim
         run: |
-          export GO111MODULE="on" && go get github.com/cosmos/tools/cmd/runsim@v1.0.0
+          export GO111MODULE="on" && go install github.com/cosmos/tools/cmd/runsim@v1.0.0
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/bin
diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml
@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Display go version
         run: go version
       - run: make build
@@ -26,11 +26,11 @@ jobs:
     steps:
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Display go version
         run: go version
       - name: Install runsim
-        run: export GO111MODULE="on" && go get github.com/cosmos/tools/cmd/runsim@v1.0.0
+        run: export GO111MODULE="on" && go install github.com/cosmos/tools/cmd/runsim@v1.0.0
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/bin
@@ -43,7 +43,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Display go version
         run: go version
       - uses: technote-space/get-diff-action@v4
@@ -69,7 +69,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Display go version
         run: go version
       - uses: technote-space/get-diff-action@v4
@@ -97,7 +97,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Display go version
         run: go version
       - uses: technote-space/get-diff-action@v4
@@ -125,7 +125,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Display go version
         run: go version
       - uses: technote-space/get-diff-action@v4
diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml
@@ -15,7 +15,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Unshallow
         run: git fetch --prune --unshallow
       - name: Create release
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Display go version
         run: go version
       - name: install tparse
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - uses: technote-space/get-diff-action@v4
         id: git_diff
         with:
@@ -49,7 +49,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Display go version
         run: go version
       - uses: technote-space/get-diff-action@v4
@@ -102,7 +102,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - uses: technote-space/get-diff-action@v4
         with:
           PATTERNS: |
@@ -180,7 +180,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - uses: technote-space/get-diff-action@v4
         with:
           PATTERNS: |
@@ -224,7 +224,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2.1.3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - uses: technote-space/get-diff-action@v4
         id: git_diff
         with:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -42,13 +42,18 @@ Ref: https://keepachangelog.com/en/1.0.0/
 ### Improvements
 
 * (simapp) [#12314](https://github.com/cosmos/cosmos-sdk/pull/12314) Increase `DefaultGenTxGas` from `1000000` to `10000000`
+* [#12371](https://github.com/cosmos/cosmos-sdk/pull/12371) Update min required Golang version to 1.18.
 
 ### Bug Fixes
 
 * [\#12317](https://github.com/cosmos/cosmos-sdk/pull/12317) Rename `edit-validator` command's `--moniker` flag to `--new-moniker`
 * (x/upgrade) [#12264](https://github.com/cosmos/cosmos-sdk/pull/12264) Fix `GetLastCompleteUpgrade` to properly return the latest upgrade.
 * (x/crisis) [#12208](https://github.com/cosmos/cosmos-sdk/pull/12208) Fix progress index of crisis invariant assertion logs.
 
+### Features
+
+* (query) [#12253](https://github.com/cosmos/cosmos-sdk/pull/12253) Add `GenericFilteredPaginate` to the `query` package to improve UX.
+
 ## [v0.45.5](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.45.5) - 2022-06-09
 
 ### Improvements
diff --git a/README.md b/README.md
@@ -45,7 +45,7 @@ The Cosmos SDK is a framework for building blockchain applications. [Tendermint
 **WARNING**: The Cosmos SDK has mostly stabilized, but we are still making some
 breaking changes.
 
-**Note**: Requires [Go 1.17+](https://golang.org/dl/)
+**Note**: Requires [Go 1.18+](https://golang.org/dl/)
 
 ## Quick Start
 
diff --git a/contrib/images/simd-dlv/Dockerfile b/contrib/images/simd-dlv/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.17-alpine AS build
+FROM golang:1.18-alpine AS build
 RUN apk add build-base git linux-headers libc-dev
 RUN go install github.com/go-delve/delve/cmd/dlv@latest
 WORKDIR /work
diff --git a/contrib/images/simd-env/Dockerfile b/contrib/images/simd-env/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.17-alpine AS build
+FROM golang:1.18-alpine AS build
 RUN apk add build-base git linux-headers
 WORKDIR /work
 COPY go.mod go.sum /work/
diff --git a/contrib/rosetta/node/Dockerfile b/contrib/rosetta/node/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.17-alpine as build
+FROM golang:1.18-alpine as build
 
 RUN apk add --no-cache tar
 
diff --git a/contrib/rosetta/rosetta-cli/Dockerfile b/contrib/rosetta/rosetta-cli/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.17-alpine as build
+FROM golang:1.18-alpine as build
 
 RUN apk add git gcc libc-dev --no-cache
 
diff --git a/go.mod b/go.mod
@@ -1,4 +1,4 @@
-go 1.17
+go 1.18
 
 module github.com/cosmos/cosmos-sdk
 
diff --git a/types/query/filtered_pagination.go b/types/query/filtered_pagination.go
@@ -3,6 +3,7 @@ package query
 import (
 	"fmt"
 
+	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/cosmos/cosmos-sdk/store/types"
 )
 
@@ -46,8 +47,10 @@ func FilteredPaginate(
 		iterator := getIterator(prefixStore, key, reverse)
 		defer iterator.Close()
 
-		var numHits uint64
-		var nextKey []byte
+		var (
+			numHits uint64
+			nextKey []byte
+		)
 
 		for ; iterator.Valid(); iterator.Next() {
 			if numHits == limit {
@@ -79,8 +82,10 @@ func FilteredPaginate(
 
 	end := offset + limit
 
-	var numHits uint64
-	var nextKey []byte
+	var (
+		numHits uint64
+		nextKey []byte
+	)
 
 	for ; iterator.Valid(); iterator.Next() {
 		if iterator.Error() != nil {
@@ -113,3 +118,137 @@ func FilteredPaginate(
 
 	return res, nil
 }
+
+// GenericFilteredPaginate does pagination of all the results in the PrefixStore based on the
+// provided PageRequest. `onResult` should be used to filter or transform the results.
+// `c` is a constructor function that needs to return a new instance of the type T (this is to
+// workaround some generic pitfalls in which we can't instantiate a T struct inside the function).
+// If key is provided, the pagination uses the optimized querying.
+// If offset is used, the pagination uses lazy filtering i.e., searches through all the records.
+// The resulting slice (of type F) can be of a different type than the one being iterated through
+// (type T), so it's possible to do any necessary transformation inside the onResult function.
+func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler](
+	cdc codec.BinaryCodec,
+	prefixStore types.KVStore,
+	pageRequest *PageRequest,
+	onResult func(key []byte, value T) (F, error),
+	constructor func() T,
+) ([]F, *PageResponse, error) {
+	// if the PageRequest is nil, use default PageRequest
+	if pageRequest == nil {
+		pageRequest = &PageRequest{}
+	}
+
+	offset := pageRequest.Offset
+	key := pageRequest.Key
+	limit := pageRequest.Limit
+	countTotal := pageRequest.CountTotal
+	reverse := pageRequest.Reverse
+	results := []F{}
+
+	if offset > 0 && key != nil {
+		return results, nil, fmt.Errorf("invalid request, either offset or key is expected, got both")
+	}
+
+	if limit == 0 {
+		limit = DefaultLimit
+
+		// count total results when the limit is zero/not supplied
+		countTotal = true
+	}
+
+	if len(key) != 0 {
+		iterator := getIterator(prefixStore, key, reverse)
+		defer iterator.Close()
+
+		var (
+			numHits uint64
+			nextKey []byte
+		)
+
+		for ; iterator.Valid(); iterator.Next() {
+			if numHits == limit {
+				nextKey = iterator.Key()
+				break
+			}
+
+			if iterator.Error() != nil {
+				return nil, nil, iterator.Error()
+			}
+
+			protoMsg := constructor()
+
+			err := cdc.Unmarshal(iterator.Value(), protoMsg)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			val, err := onResult(iterator.Key(), protoMsg)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			if val.Size() != 0 {
+				results = append(results, val)
+				numHits++
+			}
+		}
+
+		return results, &PageResponse{
+			NextKey: nextKey,
+		}, nil
+	}
+
+	iterator := getIterator(prefixStore, nil, reverse)
+	defer iterator.Close()
+
+	end := offset + limit
+
+	var (
+		numHits uint64
+		nextKey []byte
+	)
+
+	for ; iterator.Valid(); iterator.Next() {
+		if iterator.Error() != nil {
+			return nil, nil, iterator.Error()
+		}
+
+		protoMsg := constructor()
+
+		err := cdc.Unmarshal(iterator.Value(), protoMsg)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		val, err := onResult(iterator.Key(), protoMsg)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if val.Size() != 0 {
+			// Previously this was the "accumulate" flag
+			if numHits >= offset && numHits < end {
+				results = append(results, val)
+			}
+			numHits++
+		}
+
+		if numHits == end+1 {
+			if nextKey == nil {
+				nextKey = iterator.Key()
+			}
+
+			if !countTotal {
+				break
+			}
+		}
+	}
+
+	res := &PageResponse{NextKey: nextKey}
+	if countTotal {
+		res.Total = numHits
+	}
+
+	return results, res, nil
+}
diff --git a/x/authz/keeper/grpc_query.go b/x/authz/keeper/grpc_query.go
diff --git a/x/authz/keeper/grpc_query_test.go b/x/authz/keeper/grpc_query_test.go

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM golang:1.17-alpine AS build`
	`1`	`+FROM golang:1.18-alpine AS build`
`2`	`2`	`RUN apk add build-base git linux-headers libc-dev`
`3`	`3`	`RUN go install github.com/go-delve/delve/cmd/dlv@latest`
`4`	`4`	`WORKDIR /work`