[Persistence] TreeStore Refactor

persistence/context.go

@@ -31,7 +31,7 @@ type PostgresContext struct {
 	//                 Need to simply access them via the bus.
 	blockStore blockstore.BlockStore
 	txIndexer  indexer.TxIndexer
-	stateTrees *stateTrees
+	stateTrees modules.TreeStore
 
 	networkId string
 }
@@ -50,7 +50,7 @@ func (p *PostgresContext) RollbackToSavePoint(bytes []byte) error {
 // IMPROVE(#361): Guarantee the integrity of the state
 // Full details in the thread from the PR review: https://github.com/pokt-network/pocket/pull/285#discussion_r1018471719
 func (p *PostgresContext) ComputeStateHash() (string, error) {
-	stateHash, err := p.updateMerkleTrees()
+	stateHash, err := p.stateTrees.Update(p.tx, p.txIndexer, uint64(p.Height))
 	if err != nil {
 		return "", err
 	}

persistence/debug.go

@@ -1,12 +1,10 @@
 package persistence
 
 import (
-	"crypto/sha256"
 	"runtime/debug"
 
 	"github.com/pokt-network/pocket/persistence/types"
 	"github.com/pokt-network/pocket/shared/messaging"
-	"github.com/pokt-network/smt"
 )
 
 // A list of functions to clear data from the DB not associated with protocol actors
@@ -118,16 +116,5 @@ func (p *PostgresContext) clearAllSQLState() error {
 }
 
 func (p *PostgresContext) clearAllTreeState() error {
-	for treeType := merkleTree(0); treeType < numMerkleTrees; treeType++ {
-		nodeStore := p.stateTrees.nodeStores[treeType]
-
-		if err := nodeStore.ClearAll(); err != nil {
-			return err
-		}
-
-		// Needed in order to make sure the root is re-set correctly after clearing
-		p.stateTrees.merkleTrees[treeType] = smt.NewSparseMerkleTree(nodeStore, sha256.New())
-	}
-
-	return nil
+	return p.stateTrees.DebugClearAll()
 }

persistence/docs/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.0.58] - 2023-06-14
+
+- Refactors the stateTrees implementation off of the PersistenceContext and into its in module
+- Implements the new Update and Commit pattern with the SMT trees in the trees module
+
 ## [0.0.0.57] - 2023-06-06
 
 - Uses ":memory:" to signify when connecting to an in-memory database

persistence/gov.go

@@ -8,7 +8,6 @@ import (
 	"github.com/pokt-network/pocket/logger"
 	"github.com/pokt-network/pocket/persistence/types"
 	"github.com/pokt-network/pocket/runtime/genesis"
-	coreTypes "github.com/pokt-network/pocket/shared/core/types"
 	"github.com/pokt-network/pocket/shared/modules"
 )
 
@@ -156,48 +155,6 @@ func getParamOrFlag[T int | string | []byte](p *PostgresContext, tableName, para
 	return
 }
 
-func (p *PostgresContext) getParamsUpdated(height int64) ([]*coreTypes.Param, error) {
-	ctx, tx := p.getCtxAndTx()
-	// Get all parameters / flags at current height
-	rows, err := tx.Query(ctx, p.getParamsOrFlagsUpdateAtHeightQuery(types.ParamsTableName, height))
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var paramSlice []*coreTypes.Param // Store returned rows
-	// Loop over all rows returned and load them into the ParamOrFlag struct array
-	for rows.Next() {
-		param := new(coreTypes.Param)
-		if err := rows.Scan(&param.Name, &param.Value); err != nil {
-			return nil, err
-		}
-		param.Height = height
-		paramSlice = append(paramSlice, param)
-	}
-	return paramSlice, nil
-}
-
-func (p *PostgresContext) getFlagsUpdated(height int64) ([]*coreTypes.Flag, error) {
-	ctx, tx := p.getCtxAndTx()
-	// Get all parameters / flags at current height
-	rows, err := tx.Query(ctx, p.getParamsOrFlagsUpdateAtHeightQuery(types.FlagsTableName, height))
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var flagSlice []*coreTypes.Flag // Store returned rows
-	// Loop over all rows returned and load them into the ParamOrFlag struct array
-	for rows.Next() {
-		flag := new(coreTypes.Flag)
-		if err := rows.Scan(&flag.Name, &flag.Value, &flag.Enabled); err != nil {
-			return nil, err
-		}
-		flag.Height = height
-		flagSlice = append(flagSlice, flag)
-	}
-	return flagSlice, nil
-}
-
 // GetAllParams returns a map of the current latest updated values for all parameters
 // and their values in the form map[parameterName] = parameterValue
 func (p *PostgresContext) GetAllParams() ([][]string, error) {
@@ -219,15 +176,6 @@ func (p *PostgresContext) GetAllParams() ([][]string, error) {
 	return paramSlice, nil
 }
 
-func (p *PostgresContext) getParamsOrFlagsUpdateAtHeightQuery(tableName string, height int64) string {
-	fields := "name,value"
-	if tableName == types.FlagsTableName {
-		fields += ",enabled"
-	}
-	// Build correct query to get all Params/Flags at certain height ordered by their name values
-	return fmt.Sprintf("SELECT %s FROM %s WHERE height=%d ORDER BY name ASC", fields, tableName, height)
-}
-
 func (p *PostgresContext) getLatestParamsOrFlagsQuery(tableName string) string {
 	fields := "name,value"
 	if tableName == types.FlagsTableName {

persistence/module.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pokt-network/pocket/logger"
 	"github.com/pokt-network/pocket/persistence/blockstore"
 	"github.com/pokt-network/pocket/persistence/indexer"
+	"github.com/pokt-network/pocket/persistence/trees"
 	"github.com/pokt-network/pocket/runtime/configs"
 	"github.com/pokt-network/pocket/runtime/genesis"
 	"github.com/pokt-network/pocket/shared/modules"
@@ -36,13 +37,14 @@ type persistenceModule struct {
 	// A key-value store mapping heights to blocks. Needed for block synchronization.
 	blockStore blockstore.BlockStore
 
-	// A tx indexer (i.e. key-value store) mapping transaction hashes to transactions. Needed for
-	// avoiding tx replays attacks, and is also used as the backing database for the transaction
-	// tx merkle tree.
+	// txIndexer is a key-value store mapping transaction hashes to `IndexedTransaction` protos.
+	// It is needed to avoid tx replay attacks and for business logic around transaction validation.
+	// IMPORTANT: It doubles as the data store for the transaction tree in state tree set.
 	txIndexer indexer.TxIndexer
 
-	// A list of all the merkle trees maintained by the persistence module that roll up into the state commitment.
-	stateTrees *stateTrees
+	// stateTrees manages all of the merkle trees maintained by the
+	// persistence module that roll up into the state commitment.
+	stateTrees modules.TreeStore
 
 	// Only one write context is allowed at a time
 	writeContext *PostgresContext
@@ -102,7 +104,8 @@ func (*persistenceModule) Create(bus modules.Bus, options ...modules.ModuleOptio
 		return nil, err
 	}
 
-	stateTrees, err := newStateTrees(persistenceCfg.TreesStoreDir)
+	// TECHDEBT (#808): Make TreeStore into a full Module
+	stateTrees, err := trees.NewStateTrees(persistenceCfg.TreesStoreDir)
 	if err != nil {
 		return nil, err
 	}
@@ -234,6 +237,10 @@ func (m *persistenceModule) GetTxIndexer() indexer.TxIndexer {
 	return m.txIndexer
 }
 
+func (m *persistenceModule) GetTreeStore() modules.TreeStore {
+	return m.stateTrees
+}
+
 func (m *persistenceModule) GetNetworkID() string {
 	return m.networkId
 }

persistence/sql/README.md

@@ -0,0 +1,35 @@
+# SQL Package
+
+The SQL package is a library of functions that all take a `pgx.Tx` as an argument and execute SQL queries on it.
+
+- GetAccountsUpdated
+- GetActors
+- GetTransactions
+- GetPools
+- GetAccounts
+- GetFlags
+- GetParams
+
+These functions are meant to abstract away the SQL row and error handling for components of the persistence package.
+
+GetActors is used to get each of the actor types in the system: Applications, Validators, Watchers, and Servicers.
+
+## Why a whole package?
+
+If the SQL handling lives in the persistence package, the package submodules will run into import cycle errors when attempting to use the SQL handlers. Putting the SQL helpers into a package inside persistence avoids the import cycle.
+
+```mermaid
+flowchart TD
+    subgraph pers[persistence]
+        subgraph p[ without sql package - import cycle error ]
+            persistenceContext1[PersistenceContext]
+            treeStore1[TreeStore]
+            treeStore1 -- Tx --> persistenceContext1
+            persistenceContext1 -- Tx --> treeStore1
+        end 
+        subgraph p2[ with sql package - refactored ]
+            persistenceContext -- Tx --> sql
+            treeStore2 -- Tx --> sql
+        end 
+    end
+```