Add --runtime-state-path parameter to support test orchestration

By default state will not be written, and providing
--runtime-state-path will ensure runtime state (API URI, bootstrap
address, and pid) are written to the provided path on startup and
removed from the provided path on shutdown. The details in the written
file support orchestrating nodes for testing:

 - pid: check if process is running, stop the process
 - uri: connect to the node's API
 - bootstrap address: used by other nodes to bootstrap themselves

Author: maru-ava

api/server/mock_server.go

@@ -68,6 +68,10 @@ type Server interface {
 	Dispatch() error
 	// DispatchTLS starts the API server with the provided TLS certificate
 	DispatchTLS(certBytes, keyBytes []byte) error
+	// Return the uri used to access the api server. Set by the
+	// Dispatch and DispatchTLS methods after binding a listener, so
+	// may need to be called until a non-empty value is returned.
+	GetURI() string
 	// RegisterChain registers the API endpoints associated with this chain.
 	// That is, add <route, handler> pairs to server so that API calls can be
 	// made to the VM.
@@ -103,6 +107,12 @@ type server struct {
 	router *router
 
 	srv *http.Server
+
+	// Synchronizes access to uri across this type's Dispatch*() methods
+	// and another goroutine calling GetURI().
+	uriLock sync.RWMutex
+	// URI (https://host:port) to access the api server.
+	uri string
 }
 
 // New returns an instance of a Server.
@@ -170,13 +180,29 @@ func New(
 	}, nil
 }
 
+// Retrieve the uri used to access the server.
+func (s *server) GetURI() string {
+	s.uriLock.RLock()
+	defer s.uriLock.RUnlock()
+	return s.uri
+}
+
+// Set the uri used to access the server.
+func (s *server) setURI(uri string) {
+	s.uriLock.Lock()
+	defer s.uriLock.Unlock()
+	s.uri = uri
+}
+
 func (s *server) Dispatch() error {
 	listenAddress := net.JoinHostPort(s.listenHost, s.listenPort)
 	listener, err := net.Listen("tcp", listenAddress)
 	if err != nil {
 		return err
 	}
 
+	s.setURI("http://" + listener.Addr().String())
+
 	ipPort, err := ips.ToIPPort(listener.Addr().String())
 	if err != nil {
 		s.log.Info("HTTP API server listening",
@@ -208,6 +234,8 @@ func (s *server) DispatchTLS(certBytes, keyBytes []byte) error {
 		return err
 	}
 
+	s.setURI("https://" + listener.Addr().String())
+
 	ipPort, err := ips.ToIPPort(listener.Addr().String())
 	if err != nil {
 		s.log.Info("HTTPS API server listening",

api/server/server.go

@@ -1495,6 +1495,8 @@ func GetNodeConfig(v *viper.Viper) (node.Config, error) {
 
 	nodeConfig.ChainDataDir = GetExpandedArg(v, ChainDataDirKey)
 
+	nodeConfig.RuntimeStatePath = v.GetString(RuntimeStatePathKey)
+
 	nodeConfig.ProvidedFlags = providedFlags(v)
 	return nodeConfig, nil
 }

config/config.go

@@ -388,6 +388,8 @@ func addNodeFlags(fs *pflag.FlagSet) {
 	fs.Float64(TracingSampleRateKey, 0.1, "The fraction of traces to sample. If >= 1, always sample. If <= 0, never sample")
 	fs.StringToString(TracingHeadersKey, map[string]string{}, "The headers to provide the trace indexer")
 	// TODO add flag to take in headers to send from exporter
+
+	fs.String(RuntimeStatePathKey, "", "The path to write runtime state to (including uri, bootstrap address and pid). If empty, runtime state will not be written.")
 }
 
 // BuildFlagSet returns a complete set of flags for avalanchego

config/flags.go

@@ -219,4 +219,5 @@ const (
 	TracingSampleRateKey                               = "tracing-sample-rate"
 	TracingExporterTypeKey                             = "tracing-exporter-type"
 	TracingHeadersKey                                  = "tracing-headers"
+	RuntimeStatePathKey                                = "runtime-state-path"
 )

config/keys.go

@@ -233,4 +233,8 @@ type Config struct {
 	// ChainDataDir is the root path for per-chain directories where VMs can
 	// write arbitrary data.
 	ChainDataDir string `json:"chainDataDir"`
+
+	// Path to write runtime state to (including uri, bootstrap
+	// address and pid). If empty, runtime state will not be written.
+	RuntimeStatePath string `json:"runtimeStatePath"`
 }

node/config.go

@@ -6,6 +6,7 @@ package node
 import (
 	"context"
 	"crypto"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -144,6 +145,11 @@ type Node struct {
 	networkNamespace string
 	Net              network.Network
 
+	// The bootstrap address will optionally be written to a runtime state
+	// file to enable other nodes to be configured to use this node as a
+	// beacon.
+	bootstrapAddress string
+
 	// tlsKeyLogWriterCloser is a debug file handle that writes all the TLS
 	// session keys. This value should only be non-nil during debugging.
 	tlsKeyLogWriterCloser io.WriteCloser
@@ -254,6 +260,9 @@ func (n *Node) initNetworking(primaryNetVdrs validators.Set) error {
 		)
 	}
 
+	// Record the bound address to enable inclusion in runtime state file.
+	n.bootstrapAddress = listener.Addr().String()
+
 	tlsKey, ok := n.Config.StakingTLSCert.PrivateKey.(crypto.Signer)
 	if !ok {
 		return errInvalidTLSKey
@@ -374,6 +383,60 @@ func (n *Node) initNetworking(primaryNetVdrs validators.Set) error {
 	return err
 }
 
+type NodeRuntimeState struct {
+	// The process id of the node
+	PID int
+	// URI to access the node API
+	// Format: [https|http]://[host]:[port]
+	URI string
+	// Address other nodes can use for bootstrapping
+	// Format: [host]:[port]
+	BootstrapAddress string
+}
+
+// Write runtime state to the configured path. Supports the use of
+// dynamically chosen network ports with local network orchestration.
+func (n *Node) writeRuntimeState() {
+	n.Log.Info("attempting to write runtime state to configured path", zap.String("path", n.Config.RuntimeStatePath))
+
+	uri := ""
+
+	// Wait until the API Server URI is available.
+	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
+	defer cancel()
+	for {
+		select {
+		case <-ctx.Done():
+			n.Log.Error("failed to retrieve API Server URI before timeout")
+			return
+		default:
+			uri = n.APIServer.GetURI()
+		}
+		if len(uri) > 0 {
+			break
+		}
+		time.Sleep(time.Millisecond * 200)
+	}
+
+	// Write the runtime state to disk
+	runtimeState := &NodeRuntimeState{
+		PID:              os.Getpid(),
+		URI:              uri,
+		BootstrapAddress: n.bootstrapAddress, // Set by network initialization
+	}
+	bytes, err := json.MarshalIndent(runtimeState, "", "  ")
+	if err != nil {
+		n.Log.Error("failed to marshal runtime state", zap.Error(err))
+		return
+	}
+	if err := os.WriteFile(n.Config.RuntimeStatePath, bytes, perms.ReadWrite); err != nil {
+		n.Log.Error("failed to write runtime state", zap.Error(err))
+		return
+	}
+
+	n.Log.Info("wrote runtime state")
+}
+
 // Dispatch starts the node's servers.
 // Returns when the node exits.
 func (n *Node) Dispatch() error {
@@ -400,6 +463,10 @@ func (n *Node) Dispatch() error {
 		n.Shutdown(1)
 	})
 
+	if len(n.Config.RuntimeStatePath) > 0 {
+		go n.writeRuntimeState()
+	}
+
 	// Add state sync nodes to the peer network
 	for i, peerIP := range n.Config.StateSyncIPs {
 		n.Net.ManuallyTrack(n.Config.StateSyncIDs[i], peerIP)
@@ -429,6 +496,17 @@ func (n *Node) Dispatch() error {
 
 	// Wait until the node is done shutting down before returning
 	n.DoneShuttingDown.Wait()
+
+	if len(n.Config.RuntimeStatePath) > 0 {
+		// Attempt to remove the runtime state path
+		if err := os.Remove(n.Config.RuntimeStatePath); err != nil && !os.IsNotExist(err) {
+			n.Log.Error("removal of runtime state file failed",
+				zap.String("path", n.Config.RuntimeStatePath),
+				zap.Error(err),
+			)
+		}
+	}
+
 	return err
 }