Rewrite program cache

- Reduce funk key size to 32 bytes (from 40 bytes)
- Move program cache to separate funk instance and wksp
- Introduce program cache eviction policy
- Restructure program cache logic / general cleanup
  - Reduce verbosity in API documentation
  - Clean up API naming
  - Split up API into a few smaller headers
- Reduce program cache fill allocations and memory copies
  (directly populate calldests map, skip calldests map alloc for
  newer sBPF versions)
- Remove broken/unused concurrency model in previous program cache
  implementation
- Remove strict instruction logging requirements
- Simplify cache invalidation logic ("queue program for re-
  verification"): now just inserts a tombstone

Author: riptl

src/discof/bank/fd_bank_tile.c

@@ -242,7 +242,7 @@ handle_microblock( fd_bank_ctx_t *     ctx,
        if that happens.  We cannot reject the transaction here as there
        would be no way to undo the partially applied changes to the bank
        in finalize anyway. */
-    fd_runtime_finalize_txn( ctx->txn_ctx->funk, txn_ctx->status_cache, txn_ctx->xid, txn_ctx, bank, NULL );
+    fd_runtime_finalize_txn( ctx->txn_ctx->funk, ctx->txn_ctx->progcache, txn_ctx->status_cache, txn_ctx->xid, txn_ctx, bank, NULL );
     FD_TEST( txn->flags );
   }
 

src/discof/exec/fd_exec_tile.c

@@ -62,6 +62,7 @@ typedef struct fd_exec_tile_ctx {
      TODO: These should probably be made read-only handles. */
   fd_banks_t *          banks;
   fd_funk_t             funk[ 1 ];
+  fd_progcache_t        progcache[1];
 
   fd_txncache_t *       txncache;
 
@@ -174,6 +175,7 @@ after_frag( fd_exec_tile_ctx_t * ctx,
       if( FD_LIKELY( ctx->txn_ctx->flags & FD_TXN_P_FLAGS_EXECUTE_SUCCESS ) ) {
           fd_runtime_finalize_txn(
             ctx->funk,
+            ctx->progcache,
             ctx->txncache,
             &xid,
             ctx->txn_ctx,

src/discof/replay/fd_replay_tile.c

@@ -127,6 +127,7 @@ struct fd_replay_tile {
   int tx_metadata_storage;
 
   fd_funk_t funk[1];
+  fd_progcache_t progcache[1];
 
   fd_txncache_t * txncache;
   fd_store_t *    store;
@@ -1390,7 +1391,7 @@ replay( fd_replay_tile_t *  ctx,
 
     fd_funk_txn_xid_t xid = { .ul = { ready_txn->slot, ready_txn->slot } };
 
-    fd_runtime_update_program_cache( bank, ctx->funk, &xid, txn_p, ctx->runtime_spad );
+    fd_runtime_update_program_cache( bank, ctx->progcache, ctx->funk, &xid, txn_p, ctx->runtime_spad );
 
     /* At this point, we are going to send the txn down the execution
         pipeline.  Increment the refcnt so we don't prematurely prune a

src/flamenco/fd_flamenco_base.h

@@ -9,9 +9,6 @@
 #define FD_SLOT_NULL                 ( ULONG_MAX )
 #define FD_SHRED_IDX_NULL            ( UINT_MAX )
 
-#define FD_FUNK_KEY_TYPE_ACC       ((uchar)1)
-#define FD_FUNK_KEY_TYPE_ELF_CACHE ((uchar)2)
-
 /* CLUSTER_VERSION is the default value for the cluster version
    in the epoch context. This value will foll forward to the
    latest version.
@@ -96,6 +93,12 @@ typedef struct fd_borrowed_account fd_borrowed_account_t;
 struct fd_txn_account;
 typedef struct fd_txn_account fd_txn_account_t;
 
+union fd_features;
+typedef union fd_features fd_features_t;
+
+struct fd_progcache;
+typedef struct fd_progcache fd_progcache_t;
+
 struct fd_account_meta {
   uchar owner[32];
   ulong lamports;

src/flamenco/progcache/Local.mk

@@ -0,0 +1,6 @@
+$(call add-hdrs,fd_prog_load.h)
+$(call add-objs,fd_prog_load,fd_flamenco)
+$(call add-hdrs,fd_progcache.h fd_progcache_rec.h)
+$(call add-objs,fd_progcache fd_progcache_rec fd_progcache_evict,fd_flamenco)
+$(call make-unit-test,test_progcache,test_progcache,fd_flamenco fd_funk fd_ballet fd_util)
+$(call run-unit-test,test_progcache)

src/flamenco/progcache/fd_prog_load.c

@@ -0,0 +1,171 @@
+#include "fd_prog_load.h"
+#include "../runtime/program/fd_bpf_loader_program.h"
+#include "../runtime/program/fd_loader_v4_program.h"
+
+/* Similar to the below function, but gets the executable program content for the v4 loader.
+   Unlike the v3 loader, the programdata is stored in a single program account. The program must
+   NOT be retracted to be added to the cache. Returns a pointer to the programdata on success,
+   and NULL on failure.
+
+   Reasons for failure include:
+   - The program state cannot be read from the account data or is in the `retracted` state. */
+static uchar const *
+fd_get_executable_program_content_for_v4_loader( fd_txn_account_t const * program_acc,
+                                                 ulong *                  program_data_len ) {
+  int err;
+
+  /* Get the current loader v4 state. This implicitly also checks the dlen. */
+  fd_loader_v4_state_t const * state = fd_loader_v4_get_state( program_acc, &err );
+  if( FD_UNLIKELY( err ) ) {
+    return NULL;
+  }
+
+  /* The program must be deployed or finalized. */
+  if( FD_UNLIKELY( fd_loader_v4_status_is_retracted( state ) ) ) {
+    return NULL;
+  }
+
+  /* This subtraction is safe because get_state() implicitly checks the
+     dlen. */
+  *program_data_len = fd_txn_account_get_data_len( program_acc )-LOADER_V4_PROGRAM_DATA_OFFSET;
+  return fd_txn_account_get_data( program_acc )+LOADER_V4_PROGRAM_DATA_OFFSET;
+}
+
+/* Gets the programdata for a v3 loader-owned account by decoding the account data
+   as well as the programdata account. Returns a pointer to the programdata on success,
+   and NULL on failure.
+
+   Reasons for failure include:
+   - The program account data cannot be decoded or is not in the `program` state.
+   - The programdata account is not large enough to hold at least `PROGRAMDATA_METADATA_SIZE` bytes. */
+static uchar const *
+fd_get_executable_program_content_for_upgradeable_loader( fd_funk_t const *         funk,
+                                                          fd_funk_txn_xid_t const * xid,
+                                                          fd_txn_account_t const *  program_acc,
+                                                          ulong *                   program_data_len,
+                                                          fd_funk_txn_xid_t *       out_xid ) {
+  fd_bpf_upgradeable_loader_state_t program_account_state[1];
+  if( FD_UNLIKELY( !fd_bincode_decode_static(
+      bpf_upgradeable_loader_state,
+      program_account_state,
+      fd_txn_account_get_data( program_acc ),
+      fd_txn_account_get_data_len( program_acc ),
+      NULL ) ) ) {
+    return NULL;
+  }
+  if( !fd_bpf_upgradeable_loader_state_is_program( program_account_state ) ) {
+    return NULL;
+  }
+
+  fd_pubkey_t * programdata_address = &program_account_state->inner.program.programdata_address;
+
+  fd_funk_txn_t const * acc_txn = NULL;
+  fd_account_meta_t const * meta = fd_funk_get_acc_meta_readonly(
+      funk, xid, programdata_address, NULL, NULL, &acc_txn );
+  if( FD_UNLIKELY( !meta ) ) return NULL;
+  *out_xid = acc_txn->xid;
+  fd_txn_account_t _rec[1];
+  fd_txn_account_t * programdata_acc = fd_txn_account_join( fd_txn_account_new( _rec, programdata_address, (void *)meta, 0 ), funk->wksp );
+  if( FD_UNLIKELY( !programdata_acc ) ) FD_LOG_CRIT(( "fd_txn_account_new failed" ));
+
+  /* We don't actually need to decode here, just make sure that the account
+     can be decoded successfully. */
+  fd_bincode_decode_ctx_t ctx_programdata = {
+    .data    = fd_txn_account_get_data( programdata_acc ),
+    .dataend = fd_txn_account_get_data( programdata_acc ) + fd_txn_account_get_data_len( programdata_acc ),
+  };
+
+  ulong total_sz = 0UL;
+  if( FD_UNLIKELY( fd_bpf_upgradeable_loader_state_decode_footprint( &ctx_programdata, &total_sz ) ) ) {
+    return NULL;
+  }
+
+  if( FD_UNLIKELY( fd_txn_account_get_data_len( programdata_acc )<PROGRAMDATA_METADATA_SIZE ) ) {
+    return NULL;
+  }
+
+  *program_data_len = fd_txn_account_get_data_len( programdata_acc ) - PROGRAMDATA_METADATA_SIZE;
+  return fd_txn_account_get_data( programdata_acc ) + PROGRAMDATA_METADATA_SIZE;
+}
+
+/* Gets the programdata for a v1/v2 loader-owned account by returning a
+   pointer to the account data. Returns a pointer to the programdata on
+   success. Given the txn account API always returns a handle to the
+   account data, this function should NEVER return NULL (since the
+   programdata of v1 and v2 loader) accounts start at the beginning of
+   the data. */
+static uchar const *
+fd_get_executable_program_content_for_v1_v2_loaders( fd_txn_account_t const * program_acc,
+                                                     ulong *                  program_data_len ) {
+  *program_data_len = fd_txn_account_get_data_len( program_acc );
+  return fd_txn_account_get_data( program_acc );
+}
+
+uchar const *
+fd_prog_load_elf( fd_funk_t const *         accdb,
+                  fd_funk_txn_xid_t const * xid,
+                  void const *              _prog_addr,
+                  ulong *                   out_sz,
+                  fd_funk_txn_xid_t *       out_xid ) {
+  fd_pubkey_t prog_addr = FD_LOAD( fd_pubkey_t, _prog_addr );
+
+  fd_funk_txn_t const * acc_txn = NULL;
+  fd_account_meta_t const * meta = fd_funk_get_acc_meta_readonly(
+      accdb, xid, &prog_addr, NULL, NULL, &acc_txn );
+  if( FD_UNLIKELY( !meta ) ) return NULL;
+  fd_funk_txn_xid_t rec_xid = acc_txn->xid;
+  fd_txn_account_t _rec[1];
+  fd_txn_account_t * rec = fd_txn_account_join( fd_txn_account_new( _rec, &prog_addr, (void *)meta, 0 ), accdb->wksp );
+  if( FD_UNLIKELY( !rec ) ) FD_LOG_CRIT(( "fd_txn_account_new failed" ));
+
+  /* v1/v2 loaders: Programdata is just the account data.
+     v3 loader: Programdata lives in a separate account. Deserialize the
+                program account and lookup the programdata account.
+                 Deserialize the programdata account.
+     v4 loader: Programdata lives in the program account, offset by
+                LOADER_V4_PROGRAM_DATA_OFFSET. */
+  fd_pubkey_t const * owner = fd_txn_account_get_owner( rec );
+  uchar const * elf = NULL;
+  fd_funk_txn_xid_t sub_xid = { .ul={ 0UL, 0UL } };
+  if( !memcmp( owner, fd_solana_bpf_loader_upgradeable_program_id.key, sizeof(fd_pubkey_t) ) ) {
+    elf = fd_get_executable_program_content_for_upgradeable_loader( accdb, xid, rec, out_sz, &sub_xid );
+  } else if( !memcmp( owner, fd_solana_bpf_loader_v4_program_id.key, sizeof(fd_pubkey_t) ) ) {
+    elf = fd_get_executable_program_content_for_v4_loader( rec, out_sz );
+  } else if( !memcmp( owner, fd_solana_bpf_loader_program_id.key, sizeof(fd_pubkey_t) ) ||
+             !memcmp( owner, fd_solana_bpf_loader_deprecated_program_id.key, sizeof(fd_pubkey_t) ) ) {
+    elf = fd_get_executable_program_content_for_v1_v2_loaders( rec, out_sz );
+  }
+
+  if( FD_LIKELY( elf ) ) {
+    /* Return the XID with the last modification slow (unclear if they can differ) */
+    *out_xid = sub_xid.ul[0]>rec_xid.ul[0] ? sub_xid : rec_xid;
+  } else {
+    fd_funk_txn_xid_set_root( out_xid );
+  }
+
+  return elf;
+}
+
+fd_prog_versions_t
+fd_prog_versions( fd_features_t const * features,
+                  ulong                 slot ) {
+  int disable_v0  = FD_FEATURE_ACTIVE( slot, features, disable_sbpf_v0_execution );
+  int reenable_v0 = FD_FEATURE_ACTIVE( slot, features, reenable_sbpf_v0_execution );
+  int enable_v0   = !disable_v0 || reenable_v0;
+  int enable_v1   = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v1_deployment_and_execution );
+  int enable_v2   = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v2_deployment_and_execution );
+  int enable_v3   = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v3_deployment_and_execution );
+
+  fd_prog_versions_t v = {0};
+  v.min_sbpf_version = enable_v0 ? FD_SBPF_V0 : FD_SBPF_V3;
+  if( enable_v3 ) {
+    v.max_sbpf_version = FD_SBPF_V3;
+  } else if( enable_v2 ) {
+    v.max_sbpf_version = FD_SBPF_V2;
+  } else if( enable_v1 ) {
+    v.max_sbpf_version = FD_SBPF_V1;
+  } else {
+    v.max_sbpf_version = FD_SBPF_V0;
+  }
+  return v;
+}

src/flamenco/progcache/fd_prog_load.h

@@ -0,0 +1,45 @@
+#ifndef HEADER_fd_src_flamenco_fd_progcache_fd_prog_load_h
+#define HEADER_fd_src_flamenco_fd_progcache_fd_prog_load_h
+
+/* fd_prog_load.h provides high-level APIs for loading Solana programs
+   from the account database. */
+
+#include "../../funk/fd_funk_rec.h"
+#include "../fd_flamenco_base.h"
+
+FD_PROTOTYPES_BEGIN
+
+/* fd_prog_load_elf loads a reference to program data from a funk-backed
+   account database.  *prog_addr gives the account address of the
+   program account (NOT the program data account).  Returns a pointer to
+   the first byte of the ELF binary on success.  *out_sz is set to the
+   size of the program data account.  *out_xid is set to the the txn
+   XID the program data account was written to.  On failure, returns
+   NULL.  Reasons for failure include: program account not found,
+   program not deployed, program data account not found, etc */
+
+uchar const *
+fd_prog_load_elf( fd_funk_t const *         accdb,
+                  fd_funk_txn_xid_t const * xid,
+                  void const *              prog_addr,
+                  ulong *                   out_sz,
+                  fd_funk_txn_xid_t *       out_xid );
+/* FIXME provide an API to detect data race */
+/* FIXME clarify edge case where program account and program data
+         account were modified in different funk txns */
+
+/* fd_prog_versions derives sBPF versions from the current feature set. */
+
+struct fd_prog_versions {
+  uint min_sbpf_version;
+  uint max_sbpf_version;
+};
+typedef struct fd_prog_versions fd_prog_versions_t;
+
+fd_prog_versions_t
+fd_prog_versions( fd_features_t const * features,
+                  ulong                 slot );
+
+FD_PROTOTYPES_END
+
+#endif /* HEADER_fd_src_flamenco_fd_progcache_fd_prog_load_h */