[GH-846] Added --bootstrap-url parameter for simple initialization of archive node

nil/client/client.go

@@ -14,6 +14,7 @@ import (
 	"github.com/NilFoundation/nil/nil/internal/contracts"
 	"github.com/NilFoundation/nil/nil/internal/types"
 	"github.com/NilFoundation/nil/nil/services/rpc/jsonrpc"
+	rpctypes "github.com/NilFoundation/nil/nil/services/rpc/types"
 	"github.com/NilFoundation/nil/nil/services/txnpool"
 )
 
@@ -132,6 +133,8 @@ type Client interface {
 
 	// GetDebugContract retrieves smart contract with its data, such as code, storage and proof
 	GetDebugContract(ctx context.Context, contractAddr types.Address, blockId any) (*jsonrpc.DebugRPCContract, error)
+
+	GetBootstrapConfig(ctx context.Context) (*rpctypes.BootstrapConfig, error)
 }
 
 func EstimateFeeExternal(

nil/client/direct_client.go

@@ -15,6 +15,7 @@ import (
 	"github.com/NilFoundation/nil/nil/services/rpc/jsonrpc"
 	"github.com/NilFoundation/nil/nil/services/rpc/rawapi"
 	"github.com/NilFoundation/nil/nil/services/rpc/transport"
+	rpctypes "github.com/NilFoundation/nil/nil/services/rpc/types"
 )
 
 // DirectClient is a client that interacts with the end api directly, without using the rpc server.
@@ -438,3 +439,7 @@ func (c *DirectClient) GetTxpoolStatus(ctx context.Context, shardId types.ShardI
 func (c *DirectClient) GetTxpoolContent(ctx context.Context, shardId types.ShardId) (jsonrpc.TxPoolContent, error) {
 	return c.txPoolApi.GetTxpoolContent(ctx, shardId)
 }
+
+func (c *DirectClient) GetBootstrapConfig(ctx context.Context) (*rpctypes.BootstrapConfig, error) {
+	return c.debugApi.GetBootstrapConfig(ctx)
+}

nil/client/rpc/client.go

@@ -25,6 +25,7 @@ import (
 	"github.com/NilFoundation/nil/nil/internal/types"
 	"github.com/NilFoundation/nil/nil/services/rpc/jsonrpc"
 	"github.com/NilFoundation/nil/nil/services/rpc/transport"
+	rpctypes "github.com/NilFoundation/nil/nil/services/rpc/types"
 )
 
 // CallError represents an error that occurs during a remote procedure call,
@@ -79,6 +80,7 @@ const (
 	Debug_getBlockByHash                 = "debug_getBlockByHash"
 	Debug_getBlockByNumber               = "debug_getBlockByNumber"
 	Debug_getContract                    = "debug_getContract"
+	Debug_getBootstrapConfig             = "debug_getBootstrapConfig"
 	Web3_clientVersion                   = "web3_clientVersion"
 	Dev_doPanicOnShard                   = "dev_doPanicOnShard"
 	Txpool_getTxpoolStatus               = "txpool_getTxpoolStatus"
@@ -951,6 +953,18 @@ func (c *Client) DoPanicOnShard(ctx context.Context, shardId types.ShardId) (uin
 	return 0, err
 }
 
+func (c *Client) GetTxpoolStatus(ctx context.Context, shardId types.ShardId) (jsonrpc.TxPoolStatus, error) {
+	return simpleCall[jsonrpc.TxPoolStatus](ctx, c, Txpool_getTxpoolStatus, shardId)
+}
+
+func (c *Client) GetTxpoolContent(ctx context.Context, shardId types.ShardId) (jsonrpc.TxPoolContent, error) {
+	return simpleCall[jsonrpc.TxPoolContent](ctx, c, Txpool_getTxpoolContent, shardId)
+}
+
+func (c *Client) GetBootstrapConfig(ctx context.Context) (*rpctypes.BootstrapConfig, error) {
+	return simpleCall[*rpctypes.BootstrapConfig](ctx, c, Debug_getBootstrapConfig)
+}
+
 func simpleCall[ReturnType any](ctx context.Context, c *Client, method string, params ...any) (ReturnType, error) {
 	res, err := c.call(ctx, method, params...)
 	var result ReturnType
@@ -977,11 +991,3 @@ func simpleCallUint64[ReturnType ~uint64](
 	}
 	return ReturnType(result), err
 }
-
-func (c *Client) GetTxpoolStatus(ctx context.Context, shardId types.ShardId) (jsonrpc.TxPoolStatus, error) {
-	return simpleCall[jsonrpc.TxPoolStatus](ctx, c, Txpool_getTxpoolStatus, shardId)
-}
-
-func (c *Client) GetTxpoolContent(ctx context.Context, shardId types.ShardId) (jsonrpc.TxPoolContent, error) {
-	return simpleCall[jsonrpc.TxPoolContent](ctx, c, Txpool_getTxpoolContent, shardId)
-}

nil/cmd/nild/devnet.go

@@ -32,31 +32,33 @@ type nodeSpec struct {
 }
 
 type clusterSpec struct {
-	NilServerName          string   `yaml:"nil_server_name"`
-	NilCertEmail           string   `yaml:"nil_cert_email"`
-	NildConfigDir          string   `yaml:"nild_config_dir"`
-	NildCredentialsDir     string   `yaml:"nild_credentials_dir"`
-	NildPromBasePort       int      `yaml:"nild_prom_base_port"`
-	NildP2PBaseTCPPort     int      `yaml:"nild_p2p_base_tcp_port"`
-	PprofBaseTCPPort       int      `yaml:"pprof_base_tcp_port"`
-	NilWipeOnUpdate        bool     `yaml:"nil_wipe_on_update"`
-	NShards                uint32   `yaml:"nShards"`
-	NilRPCHost             string   `yaml:"nil_rpc_host"`
-	NilRPCPort             int      `yaml:"nil_rpc_port"`
-	EnableRPCOnValidators  bool     `yaml:"nil_rpc_enable_on_validators"`
-	ClickhouseHost         string   `yaml:"clickhouse_host"`
-	ClickhousePort         int      `yaml:"clickhouse_port"`
-	ClickhouseLogin        string   `yaml:"clickhouse_login"`
-	ClickhouseDatabase     string   `yaml:"clickhouse_database"`
-	CometaRPCHost          string   `yaml:"cometa_rpc_host"`
-	CometaPort             int      `yaml:"cometa_port"`
-	FaucetRPCHost          string   `yaml:"faucet_rpc_host"`
-	FaucetPort             int      `yaml:"faucet_port"`
-	NilLoadgenHost         string   `yaml:"nil_loadgen_host"`
-	NilLoadgenPort         int      `yaml:"nil_loadgen_port"`
-	NilUpdateRetryInterval int      `yaml:"nil_update_retry_interval_sec"`
-	InstanceEnv            string   `yaml:"instance_env"`
-	SignozJournaldLogs     []string `yaml:"signoz_journald_logs"`
+	NilServerName          string                `yaml:"nil_server_name"`
+	NilCertEmail           string                `yaml:"nil_cert_email"`
+	NildConfigDir          string                `yaml:"nild_config_dir"`
+	NildCredentialsDir     string                `yaml:"nild_credentials_dir"`
+	NildPromBasePort       int                   `yaml:"nild_prom_base_port"`
+	NildP2PBaseTCPPort     int                   `yaml:"nild_p2p_base_tcp_port"`
+	PprofBaseTCPPort       int                   `yaml:"pprof_base_tcp_port"`
+	NilWipeOnUpdate        bool                  `yaml:"nil_wipe_on_update"`
+	NShards                uint32                `yaml:"nShards"`
+	NilRPCHost             string                `yaml:"nil_rpc_host"`
+	NilRPCPort             int                   `yaml:"nil_rpc_port"`
+	EnableRPCOnValidators  bool                  `yaml:"nil_rpc_enable_on_validators"`
+	Relays                 network.AddrInfoSlice `yaml:"nil_relays"`
+	RelayPublicAddress     network.AddrInfo      `yaml:"nil_relay_public_address"`
+	ClickhouseHost         string                `yaml:"clickhouse_host"`
+	ClickhousePort         int                   `yaml:"clickhouse_port"`
+	ClickhouseLogin        string                `yaml:"clickhouse_login"`
+	ClickhouseDatabase     string                `yaml:"clickhouse_database"`
+	CometaRPCHost          string                `yaml:"cometa_rpc_host"`
+	CometaPort             int                   `yaml:"cometa_port"`
+	FaucetRPCHost          string                `yaml:"faucet_rpc_host"`
+	FaucetPort             int                   `yaml:"faucet_port"`
+	NilLoadgenHost         string                `yaml:"nil_loadgen_host"`
+	NilLoadgenPort         int                   `yaml:"nil_loadgen_port"`
+	NilUpdateRetryInterval int                   `yaml:"nil_update_retry_interval_sec"`
+	InstanceEnv            string                `yaml:"instance_env"`
+	SignozJournaldLogs     []string              `yaml:"signoz_journald_logs"`
 
 	NilConfig        []nodeSpec `yaml:"nil_config"`
 	NilArchiveConfig []nodeSpec `yaml:"nil_archive_config"`
@@ -66,18 +68,20 @@ type clusterSpec struct {
 }
 
 type server struct {
-	service         string
-	name            string
-	identity        string
-	p2pPort         int
-	promPort        int
-	pprofPort       int
-	rpcPort         int
-	credsDir        string
-	workDir         string
-	nodeSpec        nodeSpec
-	logClientEvents bool
-	vkm             *keys.ValidatorKeysManager
+	service            string
+	name               string
+	identity           string
+	relays             network.AddrInfoSlice
+	relayPublicAddress network.AddrInfo
+	p2pPort            int
+	promPort           int
+	pprofPort          int
+	rpcPort            int
+	credsDir           string
+	workDir            string
+	nodeSpec           nodeSpec
+	logClientEvents    bool
+	vkm                *keys.ValidatorKeysManager
 }
 
 type cluster struct {
@@ -194,9 +198,17 @@ func genDevnet(cmd *cobra.Command, args []string) error {
 	if spec.EnableRPCOnValidators {
 		validatorRPCBasePort = spec.NilRPCPort + len(spec.NilRPCConfig)
 	}
-	validators, err := spec.makeServers(spec.NilConfig,
-		spec.NildP2PBaseTCPPort, spec.NildPromBasePort, spec.PprofBaseTCPPort, validatorRPCBasePort,
-		"nil", baseDir, false)
+	validators, err := spec.makeServers(
+		spec.NilConfig,
+		spec.NildP2PBaseTCPPort,
+		spec.NildPromBasePort,
+		spec.PprofBaseTCPPort,
+		validatorRPCBasePort,
+		spec.Relays,
+		spec.RelayPublicAddress,
+		"nil",
+		baseDir,
+		false)
 	if err != nil {
 		return fmt.Errorf("failed to setup validator nodes: %w", err)
 	}
@@ -216,18 +228,34 @@ func genDevnet(cmd *cobra.Command, args []string) error {
 		archiveBasePprof = spec.PprofBaseTCPPort + len(validators)
 	}
 
-	c.archivers, err = spec.makeServers(spec.NilArchiveConfig,
-		archiveBaseP2P, archiveBaseProm, archiveBasePprof, 0,
-		"nil-archive", baseDir, false)
+	c.archivers, err = spec.makeServers(
+		spec.NilArchiveConfig,
+		archiveBaseP2P,
+		archiveBaseProm,
+		archiveBasePprof,
+		0,
+		spec.Relays,
+		spec.RelayPublicAddress,
+		"nil-archive",
+		baseDir,
+		false)
 	if err != nil {
 		return fmt.Errorf("failed to setup archive nodes: %w", err)
 	}
 
 	rpcBasePprof := spec.PprofBaseTCPPort + len(validators) + len(c.archivers)
 
-	c.rpcNodes, err = spec.makeServers(spec.NilRPCConfig,
-		0, 0, rpcBasePprof, spec.NilRPCPort,
-		"nil-rpc", baseDir, true)
+	c.rpcNodes, err = spec.makeServers(
+		spec.NilRPCConfig,
+		0,
+		0,
+		rpcBasePprof,
+		spec.NilRPCPort,
+		nil,
+		network.AddrInfo{},
+		"nil-rpc",
+		baseDir,
+		true)
 	if err != nil {
 		return fmt.Errorf("failed to setup rpc nodes: %w", err)
 	}
@@ -277,6 +305,8 @@ func (spec *clusterSpec) makeServers(
 	basePromPort int,
 	basePprofPort int,
 	baseHTTPPort int,
+	relays network.AddrInfoSlice,
+	relayPublicAddress network.AddrInfo,
 	service string,
 	baseDir string,
 	logClientEvents bool,
@@ -286,6 +316,8 @@ func (spec *clusterSpec) makeServers(
 		servers[i].service = service
 		servers[i].name = fmt.Sprintf("%s-%d", service, i)
 		servers[i].nodeSpec = nodeSpec
+		servers[i].relays = relays
+		servers[i].relayPublicAddress = relayPublicAddress
 		servers[i].logClientEvents = logClientEvents
 		if baseP2pPort != 0 {
 			servers[i].p2pPort = baseP2pPort + i
@@ -333,7 +365,10 @@ func (spec *clusterSpec) EnsureIdentity(srv server) (string, error) {
 		return "", fmt.Errorf("failed to load or generate keys: %w", err)
 	}
 	_, _, identity, err := network.SerializeKeys(privKey)
-	return identity.String(), err
+	if err != nil {
+		return "", fmt.Errorf("failed to serialize keys: %w", err)
+	}
+	return identity.String(), nil
 }
 
 func (c *cluster) writeServerConfig(instanceId int, srv server, only string) error {
@@ -363,8 +398,10 @@ func (c *cluster) writeServerConfig(instanceId int, srv server, only string) err
 			Network: &network.Config{
 				TcpPort: srv.p2pPort,
 
-				DHTEnabled:        true,
-				DHTBootstrapPeers: getPeers(c.validators, inst.DHTBootstrapPeersIdx),
+				DHTEnabled:         true,
+				DHTBootstrapPeers:  getPeers(c.validators, inst.DHTBootstrapPeersIdx),
+				Relays:             srv.relays,
+				RelayPublicAddress: srv.relayPublicAddress,
 			},
 			Telemetry: &telemetry.Config{
 				ExportMetrics:  true,

nil/cmd/nild/main.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"os"
+	"time"
 
+	rpc_client "github.com/NilFoundation/nil/nil/client/rpc"
 	"github.com/NilFoundation/nil/nil/cmd/nild/nildconfig"
 	"github.com/NilFoundation/nil/nil/common/check"
 	"github.com/NilFoundation/nil/nil/common/concurrent"
@@ -32,7 +34,7 @@ var logFilter string
 func main() {
 	logger := logging.NewLogger("nild")
 
-	cfg := parseArgs()
+	cfg := parseArgs(logger)
 
 	logging.ApplyComponentsFilter(logFilter)
 
@@ -100,7 +102,43 @@ func addBasicFlags(fset *pflag.FlagSet, cfg *nildconfig.Config) {
 		&cfg.CollatorTickPeriodMs, "collator-tick-ms", cfg.CollatorTickPeriodMs, "collator tick period in milliseconds")
 }
 
-func parseArgs() *nildconfig.Config {
+func doBootstrapRequestAndPatchConfig(
+	ctx context.Context,
+	bootstrapUrl string,
+	cfg *nildconfig.Config,
+	logger logging.Logger,
+) error {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+
+	bootstrapConfig, err := rpc_client.NewClient(bootstrapUrl, logger).GetBootstrapConfig(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Patch config
+	cfg.NShards = bootstrapConfig.NShards
+
+	if cfg.ZeroState != nil {
+		logger.Warn().Msg("overriding zero state config")
+	}
+	cfg.ZeroState = bootstrapConfig.ZeroStateConfig
+
+	if cfg.Network.TcpPort == 0 {
+		tcpPort := 3000 // Some port to work through libp2p
+		logger.Info().Msgf("setting TCP port to %d", tcpPort)
+		cfg.Network.TcpPort = tcpPort
+	}
+
+	cfg.BootstrapPeers = append(cfg.BootstrapPeers, bootstrapConfig.BootstrapPeers...)
+
+	cfg.Network.DHTBootstrapPeers = append(cfg.Network.DHTBootstrapPeers, bootstrapConfig.DhtBootstrapPeers...)
+	cfg.Network.DHTEnabled = true
+
+	return nil
+}
+
+func parseArgs(logger logging.Logger) *nildconfig.Config {
 	cfg, err := loadConfig()
 	check.PanicIfErr(err)
 
@@ -179,17 +217,30 @@ func parseArgs() *nildconfig.Config {
 	replayCmd.Flags().Var(&cfg.Replay.BlockIdLast, "last-block", "last block id to replay")
 	replayCmd.Flags().Var(&cfg.Replay.ShardId, "shard-id", "shard id to replay block from")
 
+	var bootstrapUrl string
 	archiveCmd := &cobra.Command{
 		Use:   "archive",
 		Short: "Run nil archive node",
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg.RunMode = nilservice.ArchiveRunMode
+
+			if bootstrapUrl != "" {
+				return doBootstrapRequestAndPatchConfig(cmd.Context(), bootstrapUrl, cfg, logger)
+			}
+
+			return nil
 		},
 	}
 
 	addBasicFlags(archiveCmd.Flags(), cfg)
 	cmdflags.AddNetwork(archiveCmd.Flags(), cfg.Network)
 	cmdflags.AddTelemetry(archiveCmd.Flags(), cfg.Telemetry)
+	// N.B. Despite the fact that we override the config with the loaded configuration,
+	// we still handle this flag in the usual way rather than resorting to a hack like GetConfigNameFromArgs.
+	// Network-supplied values should win by default, but because we merge only a small, well-defined
+	// subset of settings, any rare conflicts can be resolved explicitly and safely.
+	archiveCmd.Flags().StringVar(
+		&bootstrapUrl, "bootstrap-url", "", "url to fetch initial configuration from (genesis config etc.")
 
 	rpcCmd := &cobra.Command{
 		Use:   "rpc",

nil/internal/cobrax/cmdflags/network.go

@@ -15,6 +15,7 @@ func AddNetwork(fset *pflag.FlagSet, cfg *network.Config) {
 
 	fset.BoolVar(&cfg.ServeRelay, "serve-relay", cfg.ServeRelay, "enable relay")
 	fset.Var(&cfg.Relays, "relays", "relay peers")
+	fset.Var(&cfg.RelayPublicAddress, "relay-public-address", "public address of relay")
 
 	fset.BoolVar(&cfg.DHTEnabled, "with-discovery", cfg.DHTEnabled, "enable discovery (with Kademlia DHT)")
 	fset.Var(&cfg.DHTBootstrapPeers, "discovery-bootstrap-peers", "bootstrap peers for discovery")