feat(confix): add migration to v2 (backport #21052) (#21103)

Co-authored-by: Akhil Kumar P <36399231+akhilkumarpilli@users.noreply.github.com>
Co-authored-by: akhilkumarpilli <akhilkumar7947@gmail.com>

Author: mergify[bot]

CHANGELOG.md

@@ -42,6 +42,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
 
 ### Features
 
+* (tools/confix) [#21052](https://github.com/cosmos/cosmos-sdk/pull/21052) Add a migration to v2 config.
 * (tests) [#20013](https://github.com/cosmos/cosmos-sdk/pull/20013) Introduce system tests to run multi node local testnet in CI
 * (runtime) [#19953](https://github.com/cosmos/cosmos-sdk/pull/19953) Implement `core/transaction.Service` in runtime.
 * (client) [#19905](https://github.com/cosmos/cosmos-sdk/pull/19905) Add grpc client config to `client.toml`.

store/v2/commitment/iavl/config.go

@@ -2,8 +2,8 @@ package iavl
 
 // Config is the configuration for the IAVL tree.
 type Config struct {
-	CacheSize              int  `mapstructure:"cache_size"`
-	SkipFastStorageUpgrade bool `mapstructure:"skip_fast_storage_upgrade"`
+	CacheSize              int  `mapstructure:"cache-size" toml:"cache-size" comment:"CacheSize set the size of the iavl tree cache."`
+	SkipFastStorageUpgrade bool `mapstructure:"skip-fast-storage-upgrade" toml:"skip-fast-storage-upgrade" comment:"If true, the tree will work like no fast storage and always not upgrade fast storage."`
 }
 
 // DefaultConfig returns the default configuration for the IAVL tree.

tools/confix/cmd/migrate.go

@@ -32,15 +32,31 @@ In case of any error in updating the file, no output is written.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var configPath string
 			clientCtx := client.GetClientContextFromCmd(cmd)
+
+			configType := confix.AppConfigType
+			isClient, _ := cmd.Flags().GetBool(confix.ClientConfigType)
+
+			if isClient {
+				configType = confix.ClientConfigType
+			}
+
 			switch {
 			case len(args) > 1:
 				configPath = args[1]
 			case clientCtx.HomeDir != "":
-				configPath = filepath.Join(clientCtx.HomeDir, "config", "app.toml")
+				suffix := "app.toml"
+				if isClient {
+					suffix = "client.toml"
+				}
+				configPath = filepath.Join(clientCtx.HomeDir, "config", suffix)
 			default:
 				return errors.New("must provide a path to the app.toml or client.toml")
 			}
 
+			if strings.HasSuffix(configPath, "client.toml") && !isClient {
+				return errors.New("app.toml file expected, got client.toml, use --client flag to migrate client.toml")
+			}
+
 			targetVersion := args[0]
 			plan, ok := confix.Migrations[targetVersion]
 			if !ok {
@@ -62,15 +78,10 @@ In case of any error in updating the file, no output is written.`,
 				outputPath = ""
 			}
 
-			configType := confix.AppConfigType
-			if ok, _ := cmd.Flags().GetBool(confix.ClientConfigType); ok {
-				configPath = strings.ReplaceAll(configPath, "app.toml", "client.toml") // for the case we are using the home dir of client ctx
-				configType = confix.ClientConfigType
-			} else if strings.HasSuffix(configPath, "client.toml") {
-				return errors.New("app.toml file expected, got client.toml, use --client flag to migrate client.toml")
-			}
+			// get transformation steps and formatDoc in which plan need to be applied
+			steps, formatDoc := plan(rawFile, targetVersion, configType)
 
-			if err := confix.Upgrade(ctx, plan(rawFile, targetVersion, configType), configPath, outputPath, FlagSkipValidate); err != nil {
+			if err := confix.Upgrade(ctx, steps, formatDoc, configPath, outputPath, FlagSkipValidate); err != nil {
 				return fmt.Errorf("failed to migrate config: %w", err)
 			}
 

tools/confix/cmd/mutate.go

@@ -73,7 +73,12 @@ func SetCommand() *cobra.Command {
 				ctx = confix.WithLogWriter(ctx, cmd.ErrOrStderr())
 			}
 
-			return confix.Upgrade(ctx, plan, filename, outputPath, FlagSkipValidate)
+			doc, err := confix.LoadConfig(filename)
+			if err != nil {
+				return err
+			}
+
+			return confix.Upgrade(ctx, plan, doc, filename, outputPath, FlagSkipValidate)
 		},
 	}
 

tools/confix/data/v2-app.toml

@@ -0,0 +1,59 @@
+[comet]
+# min-retain-blocks defines the minimum block height offset from the current block being committed, such that all blocks past this offset are pruned from CometBFT. A value of 0 indicates that no blocks should be pruned.
+min-retain-blocks = 0
+# index-events defines the set of events in the form {eventType}.{attributeKey}, which informs CometBFT what to index. If empty, all events will be indexed.
+index-events = []
+# halt-height contains a non-zero block height at which a node will gracefully halt and shutdown that can be used to assist upgrades and testing.
+halt-height = 0
+# halt-time contains a non-zero minimum block time (in Unix seconds) at which a node will gracefully halt and shutdown that can be used to assist upgrades and testing.
+halt-time = 0
+# address defines the CometBFT RPC server address to bind to.
+address = 'tcp://127.0.0.1:26658'
+# transport defines the CometBFT RPC server transport protocol: socket, grpc
+transport = 'socket'
+# trace enables the CometBFT RPC server to output trace information about its internal operations.
+trace = false
+# standalone starts the application without the CometBFT node. The node should be started separately.
+standalone = false
+
+[grpc]
+# Enable defines if the gRPC server should be enabled.
+enable = true
+# Address defines the gRPC server address to bind to.
+address = 'localhost:9090'
+# MaxRecvMsgSize defines the max message size in bytes the server can receive.
+# The default value is 10MB.
+max-recv-msg-size = 10485760
+# MaxSendMsgSize defines the max message size in bytes the server can send.
+# The default value is math.MaxInt32.
+max-send-msg-size = 2147483647
+
+[store]
+# The type of database for application and snapshots databases.
+app-db-backend = 'goleveldb'
+
+[store.options]
+# State storage database type. Currently we support: 0 for SQLite, 1 for Pebble
+ss-type = 0
+# State commitment database type. Currently we support:0 for iavl, 1 for iavl v2
+sc-type = 0
+
+# Pruning options for state storage
+[store.options.ss-pruning-option]
+# Number of recent heights to keep on disk.
+keep-recent = 2
+# Height interval at which pruned heights are removed from disk.
+interval = 1
+
+# Pruning options for state commitment
+[store.options.sc-pruning-option]
+# Number of recent heights to keep on disk.
+keep-recent = 2
+# Height interval at which pruned heights are removed from disk.
+interval = 1
+
+[store.options.iavl-config]
+# CacheSize set the size of the iavl tree cache.
+cache-size = 100000
+# If true, the tree will work like no fast storage and always not upgrade fast storage.
+skip-fast-storage-upgrade = true

tools/confix/data/v2-client.toml

@@ -0,0 +1,27 @@
+# This is a TOML config file.
+# For more information, see https://github.com/toml-lang/toml
+
+###############################################################################
+###                           Client Configuration                          ###
+###############################################################################
+
+# The network chain ID
+chain-id = ""
+# The keyring's backend, where the keys are stored (os|file|kwallet|pass|test|memory)
+keyring-backend = "test"
+# Default key name, if set, defines the default key to use for signing transaction when the --from flag is not specified
+keyring-default-keyname = ""
+# CLI output format (text|json)
+output = "text"
+# <host>:<port> to CometBFT RPC interface for this chain
+node = "tcp://localhost:26657"
+# Transaction broadcasting mode (sync|async)
+broadcast-mode = "sync"
+
+# gRPC server endpoint to which the client will connect.
+# It can be overwritten by the --grpc-addr flag in each command.
+grpc-address = ""
+
+# Allow the gRPC client to connect over insecure channels.
+# It can be overwritten by the --grpc-insecure flag in each command.
+grpc-insecure = false

tools/confix/match.go

@@ -0,0 +1,63 @@
+package confix
+
+import (
+	"sort"
+
+	"github.com/creachadair/tomledit"
+	"github.com/creachadair/tomledit/transform"
+)
+
+// MatchKeys diffs the keyspaces of the TOML documents in files lhs and rhs.
+// Comments, order, and values are ignored for comparison purposes.
+// It will return in the format of map[oldKey]newKey
+func MatchKeys(lhs, rhs *tomledit.Document) map[string]string {
+	matches := matchDocs(map[string]string{}, allKVs(lhs.Global), allKVs(rhs.Global))
+
+	lsec, rsec := lhs.Sections, rhs.Sections
+	transform.SortSectionsByName(lsec)
+	transform.SortSectionsByName(rsec)
+
+	i, j := 0, 0
+	for i < len(lsec) && j < len(rsec) {
+		switch {
+		case lsec[i].Name.Before(rsec[j].Name):
+			i++
+		case rsec[j].Name.Before(lsec[i].Name):
+			j++
+		default:
+			matches = matchDocs(matches, allKVs(lsec[i]), allKVs(rsec[j]))
+			i++
+			j++
+		}
+	}
+
+	return matches
+}
+
+// matchDocs get all the keys matching in lhs and rhs.
+// value of keys are ignored
+func matchDocs(matchesMap map[string]string, lhs, rhs []KV) map[string]string {
+	sort.Slice(lhs, func(i, j int) bool {
+		return lhs[i].Key < lhs[j].Key
+	})
+	sort.Slice(rhs, func(i, j int) bool {
+		return rhs[i].Key < rhs[j].Key
+	})
+
+	i, j := 0, 0
+	for i < len(lhs) && j < len(rhs) {
+		switch {
+		case lhs[i].Key < rhs[j].Key:
+			i++
+		case lhs[i].Key > rhs[j].Key:
+			j++
+		default:
+			// key exists in both lhs and rhs
+			matchesMap[lhs[i].Key] = rhs[j].Key
+			i++
+			j++
+		}
+	}
+
+	return matchesMap
+}

tools/confix/migrations.go

@@ -19,19 +19,42 @@ const (
 )
 
 // MigrationMap defines a mapping from a version to a transformation plan.
-type MigrationMap map[string]func(from *tomledit.Document, to, planType string) transform.Plan
+type MigrationMap map[string]func(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document)
 
 var Migrations = MigrationMap{
 	"v0.45": NoPlan, // Confix supports only the current supported SDK version. So we do not support v0.44 -> v0.45.
 	"v0.46": PlanBuilder,
 	"v0.47": PlanBuilder,
 	"v0.50": PlanBuilder,
 	"v0.52": PlanBuilder,
+	"v2":    V2PlanBuilder,
 	// "v0.xx.x": PlanBuilder, // add specific migration in case of configuration changes in minor versions
 }
 
+type v2KeyChangesMap map[string][]string
+
+// list all the keys which are need to be modified in v2
+var v2KeyChanges = v2KeyChangesMap{
+	"min-retain-blocks": []string{"comet.min-retain-blocks"},
+	"index-events":      []string{"comet.index-events"},
+	"halt-height":       []string{"comet.halt-height"},
+	"halt-time":         []string{"comet.halt-time"},
+	"app-db-backend":    []string{"store.app-db-backend"},
+	"pruning-keep-recent": []string{
+		"store.options.ss-pruning-option.keep-recent",
+		"store.options.sc-pruning-option.keep-recent",
+	},
+	"pruning-interval": []string{
+		"store.options.ss-pruning-option.interval",
+		"store.options.sc-pruning-option.interval",
+	},
+	"iavl-cache-size":       []string{"store.options.iavl-config.cache-size"},
+	"iavl-disable-fastnode": []string{"store.options.iavl-config.skip-fast-storage-upgrade"},
+	// Add other key mappings as needed
+}
+
 // PlanBuilder is a function that returns a transformation plan for a given diff between two files.
-func PlanBuilder(from *tomledit.Document, to, planType string) transform.Plan {
+func PlanBuilder(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) {
 	plan := transform.Plan{}
 	deletedSections := map[string]bool{}
 
@@ -114,11 +137,105 @@ func PlanBuilder(from *tomledit.Document, to, planType string) transform.Plan {
 		plan = append(plan, step)
 	}
 
-	return plan
+	return plan, from
 }
 
 // NoPlan returns a no-op plan.
-func NoPlan(_ *tomledit.Document, to, planType string) transform.Plan {
+func NoPlan(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) {
 	fmt.Printf("no migration needed to %s\n", to)
-	return transform.Plan{}
+	return transform.Plan{}, from
+}
+
+// V2PlanBuilder is a function that returns a transformation plan to convert to v2 config
+func V2PlanBuilder(from *tomledit.Document, to, planType string) (transform.Plan, *tomledit.Document) {
+	target, err := LoadLocalConfig(to, planType)
+	if err != nil {
+		panic(fmt.Errorf("failed to parse file: %w. This file should have been valid", err))
+	}
+
+	plan := transform.Plan{}
+	plan = updateMatchedKeysPlan(from, target, plan)
+	plan = applyKeyChangesPlan(from, plan)
+
+	return plan, target
+}
+
+// updateMatchedKeysPlan updates all matched keys with old key values
+func updateMatchedKeysPlan(from, target *tomledit.Document, plan transform.Plan) transform.Plan {
+	matches := MatchKeys(from, target)
+	for oldKey, newKey := range matches {
+		oldEntry := getEntry(from, oldKey)
+		if oldEntry == nil {
+			continue
+		}
+
+		// check if the key "app-db-backend" exists and if its value is empty in the existing config
+		// If the value is empty, update the key value with the default value
+		// of v2 i.e., goleveldb  to prevent network failures.
+		if isAppDBBackend(newKey, oldEntry) {
+			continue // lets keep app-db-backend with v2 default value
+		}
+
+		// update newKey value with old entry value
+		step := createUpdateStep(oldKey, newKey, oldEntry)
+		plan = append(plan, step)
+	}
+	return plan
+}
+
+// applyKeyChangesPlan checks if key changes are needed with the "to" version and applies them
+func applyKeyChangesPlan(from *tomledit.Document, plan transform.Plan) transform.Plan {
+	changes := v2KeyChanges
+	for oldKey, newKeys := range changes {
+		oldEntry := getEntry(from, oldKey)
+		if oldEntry == nil {
+			continue
+		}
+
+		for _, newKey := range newKeys {
+			// check if the key "app-db-backend" exists and if its value is empty in the existing config
+			// If the value is empty, update the key value with the default value
+			// of v2 i.e., goleveldb  to prevent network failures.
+			if isAppDBBackend(newKey, oldEntry) {
+				continue // lets keep app-db-backend with v2 default value
+			}
+
+			// update newKey value with old entry value
+			step := createUpdateStep(oldKey, newKey, oldEntry)
+			plan = append(plan, step)
+		}
+	}
+	return plan
+}
+
+// getEntry retrieves the first entry for the given key from the document
+func getEntry(doc *tomledit.Document, key string) *parser.KeyValue {
+	splitKeys := strings.Split(key, ".")
+	entry := doc.First(splitKeys...)
+	if entry == nil || entry.KeyValue == nil {
+		return nil
+	}
+	return entry.KeyValue
+}
+
+// isAppDBBackend checks if the key is "store.app-db-backend" and the value is empty
+func isAppDBBackend(key string, entry *parser.KeyValue) bool {
+	return key == "store.app-db-backend" && entry.Value.String() == `""`
+}
+
+// createUpdateStep creates a transformation step to update a key with a new key value
+func createUpdateStep(oldKey, newKey string, oldEntry *parser.KeyValue) transform.Step {
+	return transform.Step{
+		Desc: fmt.Sprintf("updating %s key with %s key", oldKey, newKey),
+		T: transform.Func(func(_ context.Context, doc *tomledit.Document) error {
+			splitNewKeys := strings.Split(newKey, ".")
+			newEntry := doc.First(splitNewKeys...)
+			if newEntry == nil || newEntry.KeyValue == nil {
+				return nil
+			}
+
+			newEntry.KeyValue.Value = oldEntry.Value
+			return nil
+		}),
+	}
 }