[WIP][Persistence] Adds save and load CLI functions

app/client/cli/node.go

@@ -1,18 +1,60 @@
 package cli
 
-import "github.com/spf13/cobra"
+import (
+	"fmt"
+
+	"github.com/pokt-network/pocket/app/client/cli/flags"
+	"github.com/pokt-network/pocket/rpc"
+	"github.com/spf13/cobra"
+)
 
 func init() {
 	nodeCmd := NewNodeCommand()
 	rootCmd.AddCommand(nodeCmd)
 }
 
+var (
+	dir string
+)
+
 func NewNodeCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "Node",
 		Short:   "Commands related to node management and operations",
 		Aliases: []string{"node", "n"},
 	}
 
+	cmd.AddCommand(nodeSaveCommands()...)
+
 	return cmd
 }
+
+func nodeSaveCommands() []*cobra.Command {
+	cmds := []*cobra.Command{
+		{
+			Use:     "Save",
+			Short:   "save a backup of node databases in the provided directory",
+			Example: "node save --dir /dir/path/here/",
+			RunE: func(cmd *cobra.Command, args []string) error {
+				client, err := rpc.NewClientWithResponses(flags.RemoteCLIURL)
+				if err != nil {
+					return err
+				}
+				resp, err := client.PostV1NodeBackup(cmd.Context(), rpc.NodeBackup{
+					Dir: &dir,
+				})
+				if err != nil {
+					return err
+				}
+				var dest []byte
+				_, err = resp.Body.Read(dest)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("%s", dest)
+				return nil
+			},
+		},
+	}
+	return cmds
+}

app/docs/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.0.9] - 2023-07-28
+
+- Adds the `Save` sub-command CLI
+- Adds `/v1/node/backup` to the OpenAPI spec
+
 ## [0.0.0.8] - 2023-06-06
 
 - Adds `query nodeRoles` sub-command the client CLI

e2e/tests/node.feature

@@ -0,0 +1,7 @@
+Feature: Node Namespace
+
+  Scenario: User Wants Help Using The Node Command 
+    Given the user has a validator
+    When the user runs the command "Node help"
+    Then the user should be able to see standard output containing "Available Commands"
+    And the validator should have exited without error

internal/testutil/trees.go

@@ -0,0 +1,124 @@
+package testutil
+
+import (
+	"log"
+	"testing"
+
+	"github.com/pokt-network/pocket/logger"
+	"github.com/pokt-network/pocket/persistence"
+	"github.com/pokt-network/pocket/persistence/trees"
+	"github.com/pokt-network/pocket/runtime"
+	"github.com/pokt-network/pocket/runtime/configs"
+	"github.com/pokt-network/pocket/runtime/test_artifacts"
+	"github.com/pokt-network/pocket/runtime/test_artifacts/keygen"
+	"github.com/pokt-network/pocket/shared/messaging"
+	"github.com/pokt-network/pocket/shared/modules"
+
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testSchema = "test_schema"
+
+	genesisStateNumValidators   = 5
+	genesisStateNumServicers    = 1
+	genesisStateNumApplications = 1
+)
+
+// creates a new tree store with a tmp directory for nodestore persistence
+// and then starts the tree store and returns its pointer.
+func NewTestTreeStoreSubmodule(t *testing.T, bus modules.Bus) modules.TreeStoreModule {
+	t.Helper()
+
+	tmpDir := t.TempDir()
+	ts, err := trees.Create(
+		bus,
+		trees.WithTreeStoreDirectory(tmpDir),
+		trees.WithLogger(logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName)))
+	require.NoError(t, err)
+
+	err = ts.Start()
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		err := ts.Stop()
+		require.NoError(t, err)
+	})
+
+	return ts
+}
+
+func SeedTestTreeStoreSubmodule(t *testing.T, mod modules.TreeStoreModule) modules.TreeStoreModule {
+	// TODO insert transaction data into postgres
+	// TODO trigger an update with a pgx connection
+	return mod
+}
+
+func NewTestPersistenceModule(t *testing.T, databaseUrl string) modules.PersistenceModule {
+	teardownDeterministicKeygen := keygen.GetInstance().SetSeed(42)
+	defer teardownDeterministicKeygen()
+
+	cfg := &configs.Config{
+		Persistence: &configs.PersistenceConfig{
+			PostgresUrl:       databaseUrl,
+			NodeSchema:        testSchema,
+			BlockStorePath:    ":memory:",
+			TxIndexerPath:     ":memory:",
+			TreesStoreDir:     ":memory:",
+			MaxConnsCount:     5,
+			MinConnsCount:     1,
+			MaxConnLifetime:   "5m",
+			MaxConnIdleTime:   "1m",
+			HealthCheckPeriod: "30s",
+		},
+	}
+
+	genesisState, _ := test_artifacts.NewGenesisState(
+		genesisStateNumValidators,
+		genesisStateNumServicers,
+		genesisStateNumApplications,
+		genesisStateNumServicers,
+	)
+
+	runtimeMgr := runtime.NewManager(cfg, genesisState)
+	bus, err := runtime.CreateBus(runtimeMgr)
+	require.NoError(t, err)
+
+	persistenceMod, err := persistence.Create(bus)
+	require.NoError(t, err)
+
+	return persistenceMod.(modules.PersistenceModule)
+}
+
+func NewTestPostgresContext(t testing.TB, pmod modules.PersistenceModule, height int64) *persistence.PostgresContext {
+	rwCtx, err := pmod.NewRWContext(height)
+	if err != nil {
+		log.Fatalf("Error creating new context: %v\n", err)
+	}
+
+	postgresCtx, ok := rwCtx.(*persistence.PostgresContext)
+	if !ok {
+		log.Fatalf("Error casting RW context to Postgres context")
+	}
+
+	// TECHDEBT: This should not be part of `NewTestPostgresContext`. It causes unnecessary resets
+	// if we call `NewTestPostgresContext` more than once in a single test.
+	t.Cleanup(func() {
+		resetStateToGenesis(pmod)
+	})
+
+	return postgresCtx
+}
+
+// This is necessary for unit tests that are dependant on a baseline genesis state
+func resetStateToGenesis(pmod modules.PersistenceModule) {
+	if err := pmod.ReleaseWriteContext(); err != nil {
+		log.Fatalf("Error releasing write context: %v\n", err)
+	}
+	if err := pmod.HandleDebugMessage(&messaging.DebugMessage{
+		Action:  messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS,
+		Message: nil,
+	}); err != nil {
+		log.Fatalf("Error clearing state: %v\n", err)
+	}
+}

persistence/blockstore/block_store.go

@@ -4,13 +4,17 @@ package blockstore
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/pokt-network/pocket/persistence/kvstore"
 	"github.com/pokt-network/pocket/shared/codec"
 	coreTypes "github.com/pokt-network/pocket/shared/core/types"
 	"github.com/pokt-network/pocket/shared/utils"
 )
 
+// backupName is the name of the archive file that is created when Backup is called for a BlockStore
+const backupName = "blockstore.bak"
+
 // BlockStore is a key-value store that maps block heights to serialized
 // block structures.
 // * It manages the atomic state transitions for applying a Unit of Work.
@@ -93,8 +97,8 @@ func (bs *blockStore) Stop() error {
 	return bs.kv.Stop()
 }
 
-func (bs *blockStore) Backup(path string) error {
-	return bs.kv.Backup(path)
+func (bs *blockStore) Backup(dir string) error {
+	return bs.kv.Backup(filepath.Join(dir, backupName))
 }
 
 ///////////////

persistence/kvstore/kvstore.go

@@ -26,7 +26,8 @@ type KVStore interface {
 	Exists(key []byte) (bool, error)
 	ClearAll() error
 
-	Backup(filepath string) error
+	// Backup takes a directory and makes a backup of the KVStore in that directory.
+	Backup(dir string) error
 }
 
 const (

persistence/trees/atomic_test.go

@@ -112,72 +112,55 @@ func TestTreeStore_SaveAndLoad(t *testing.T) {
 	t.Parallel()
 	t.Run("should save a backup in a directory", func(t *testing.T) {
 		ts := newTestTreeStore(t)
-		tmpdir := t.TempDir()
+		backupDir := t.TempDir()
 		// assert that the directory is empty before backup
-		ok, err := isEmpty(tmpdir)
+		ok, err := isEmpty(backupDir)
 		require.NoError(t, err)
 		require.True(t, ok)
 
 		// Trigger a backup
-		require.NoError(t, ts.Backup(tmpdir))
+		require.NoError(t, ts.Backup(backupDir))
 
 		// assert that the directory is not empty after Backup has returned
-		ok, err = isEmpty(tmpdir)
+		ok, err = isEmpty(backupDir)
 		require.NoError(t, err)
 		require.False(t, ok)
 	})
 	t.Run("should load a backup and maintain TreeStore hash integrity", func(t *testing.T) {
-		ctrl := gomock.NewController(t)
-		tmpDir := t.TempDir()
-
-		mockTxIndexer := mock_types.NewMockTxIndexer(ctrl)
-		mockBus := mock_modules.NewMockBus(ctrl)
-		mockPersistenceMod := mock_modules.NewMockPersistenceModule(ctrl)
-
-		mockBus.EXPECT().GetPersistenceModule().AnyTimes().Return(mockPersistenceMod)
-		mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer)
-
-		ts := &treeStore{
-			logger:       logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName),
-			treeStoreDir: tmpDir,
-		}
-		require.NoError(t, ts.Start())
-		require.NotNil(t, ts.rootTree.tree)
-
-		for _, treeName := range stateTreeNames {
-			err := ts.merkleTrees[treeName].tree.Update([]byte("foo"), []byte("bar"))
-			require.NoError(t, err)
-		}
+		// create a new tree store and save it's initial hash
+		ts := newTestTreeStore(t)
+		hash1 := ts.getStateHash()
 
-		err := ts.Commit()
+		// make a temp directory for the backup and assert it's empty
+		backupDir := t.TempDir()
+		empty, err := isEmpty(backupDir)
 		require.NoError(t, err)
+		require.True(t, empty)
 
-		hash1 := ts.getStateHash()
-		require.NotEmpty(t, hash1)
+		// make a backup
+		err = ts.Backup(backupDir)
+		require.NoError(t, err)
 
-		w, err := ts.save()
+		// assert directory is not empty after backup
+		empty2, err := isEmpty(backupDir)
 		require.NoError(t, err)
-		require.NotNil(t, w)
-		require.NotNil(t, w.rootHash)
-		require.NotNil(t, w.merkleRoots)
+		require.False(t, empty2)
 
-		// Stop the first tree store so that it's databases are no longer used
+		// stop the first tree store so that it's databases are released
 		require.NoError(t, ts.Stop())
 
 		// declare a second TreeStore with no trees then load the first worldstate into it
 		ts2 := &treeStore{
-			logger:       logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName),
-			treeStoreDir: tmpDir,
+			logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName),
 		}
 
-		// Load sets a tree store to the provided worldstate
-		err = ts2.Load(w)
+		// call load with the backup directory
+		err = ts2.Load(backupDir)
 		require.NoError(t, err)
 
+		// assert that hash is unchanged from save and load
 		hash2 := ts2.getStateHash()
-
-		// Assert that hash is unchanged from save and load
-		require.Equal(t, hash1, hash2)
+		require.Equal(t, hash1, hash2, "failed to maintain hash integrity")
 	})
 }