Rewrite and separate program cache

- Reduce funk key size to 32 bytes (from 40 bytes)
- Move program cache to separate funk instance and wksp
- Restructure program cache logic / general cleanup
  - Reduce verbosity in API documentation
  - Clean up API naming
  - Split up API into a few smaller headers
- Reduce program cache fill allocations and memory copies
  (directly populate calldests map, skip calldests map alloc for
  newer sBPF versions)
- Remove broken/unused concurrency model in previous program cache
  implementation
- Remove strict instruction logging requirements
  (no program cache tombstone entries for obviously invalid
  programs, such as invalid section table)
- Simplify cache invalidation logic ("queue program for re-
  verification"): now just inserts a tombstone
- Reuse cache entries across forks (cache entries tracked at fork
  graph nodes determined by program modify slot, instead of program
  load slot)
- Remove program pre-loading
- Make program cache fills shared memory concurrent (in exec tiles)

Author: riptl

book/api/metrics-generated.md

@@ -1057,3 +1057,19 @@
 | <span class="metrics-name">backt_&#8203;start_&#8203;slot</span> | gauge | The slot at which the backtest started |
 
 </div>
+
+## Exec Tile
+
+<div class="metrics">
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| <span class="metrics-name">exec_&#8203;progcache_&#8203;misses</span> | counter | Number of program cache misses |
+| <span class="metrics-name">exec_&#8203;progcache_&#8203;hits</span> | counter | Number of program cache hits |
+| <span class="metrics-name">exec_&#8203;progcache_&#8203;fills</span> | counter | Number of program cache insertions |
+| <span class="metrics-name">exec_&#8203;progcache_&#8203;fill_&#8203;tot_&#8203;sz</span> | counter | Total number of bytes inserted into program cache |
+| <span class="metrics-name">exec_&#8203;progcache_&#8203;fill_&#8203;fails</span> | counter | Number of program cache load fails (tombstones inserted) |
+| <span class="metrics-name">exec_&#8203;progcache_&#8203;dup_&#8203;inserts</span> | counter | Number of time two tiles raced to insert the same cache entry |
+| <span class="metrics-name">exec_&#8203;progcache_&#8203;invalidations</span> | counter | Number of program cache invalidations |
+
+</div>

src/app/firedancer-dev/commands/backtest.c

@@ -28,6 +28,7 @@
 #include "../../../ballet/lthash/fd_lthash.h"
 #include "../../../flamenco/runtime/context/fd_capture_ctx.h"
 #include "../../../disco/pack/fd_pack_cost.h"
+#include "../../../flamenco/progcache/fd_progcache_admin.h"
 
 #include "../main.h"
 
@@ -77,9 +78,16 @@ backtest_topo( config_t * config ) {
       config->firedancer.funk.max_database_transactions,
       config->firedancer.funk.heap_size_gib,
       config->firedancer.funk.lock_pages );
-
   fd_topob_tile_uses( topo, replay_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
 
+  fd_topob_wksp( topo, "progcache" );
+  fd_topo_obj_t * progcache_obj = setup_topo_progcache( topo, "progcache",
+      fd_progcache_est_rec_max( config->firedancer.runtime.program_cache.heap_size_mib<<20,
+                                config->firedancer.runtime.program_cache.mean_cache_entry_size ),
+      config->firedancer.funk.max_database_transactions,
+      config->firedancer.runtime.program_cache.heap_size_mib<<20 );
+  fd_topob_tile_uses( topo, replay_tile, progcache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+
   /**********************************************************************/
   /* Add the executor tiles to topo                                     */
   /**********************************************************************/

src/app/firedancer/config/default.toml

@@ -533,6 +533,17 @@ user = ""
     # max_fork_width, the client will crash.
     max_fork_width = 32
 
+    # The program cache pre-loads frequently executed programs for
+    # faster transaction execution.
+    [runtime.program_cache]
+        # The size of the loaded program cache in MiB.
+        heap_size_mib = 2048
+
+        # The mean expected heap utilization of a cache entry.  Controls
+        # the size of metadata structures (e.g. cache entry table).  It
+        # is not recommended to change this setting.
+        mean_cache_entry_size = 131072
+
 # This section configures the "groove" persistent account database.
 # [groove]
 # ...

src/app/firedancer/topology.c

@@ -21,7 +21,7 @@
 #include "../../discof/restore/utils/fd_ssmsg.h"
 #include "../../flamenco/gossip/fd_gossip.h"
 #include "../../flamenco/runtime/context/fd_capture_ctx.h"
-#include "../../funk/fd_funk.h" /* funk_footprint() */
+#include "../../flamenco/progcache/fd_progcache_admin.h"
 
 #include <sys/random.h>
 #include <sys/types.h>
@@ -99,6 +99,35 @@ setup_topo_funk( fd_topo_t *  topo,
   return obj;
 }
 
+fd_topo_obj_t *
+setup_topo_progcache( fd_topo_t *  topo,
+                      char const * wksp_name,
+                      ulong        max_cache_entries,
+                      ulong        max_database_transactions,
+                      ulong        heap_size ) {
+  fd_topo_obj_t * obj = fd_topob_obj( topo, "funk", wksp_name );
+  FD_TEST( fd_pod_insert_ulong(  topo->props, "progcache", obj->id ) );
+  FD_TEST( fd_pod_insertf_ulong( topo->props, max_cache_entries,         "obj.%lu.rec_max",  obj->id ) );
+  FD_TEST( fd_pod_insertf_ulong( topo->props, max_database_transactions, "obj.%lu.txn_max",  obj->id ) );
+  FD_TEST( fd_pod_insertf_ulong( topo->props, heap_size,                 "obj.%lu.heap_max", obj->id ) );
+  ulong funk_footprint = fd_funk_footprint( max_database_transactions, max_cache_entries );
+  if( FD_UNLIKELY( !funk_footprint ) ) FD_LOG_ERR(( "Invalid [runtime.program_cache] parameters" ));
+  if( FD_UNLIKELY( heap_size<(2*funk_footprint) ) ) {
+    FD_LOG_ERR(( "Invalid [runtime.program_cache] parameters: heap_size_mib should be at least %lu",
+                 ( 4*funk_footprint )>>20 ));
+  }
+
+  /* Increase workspace partition count */
+  ulong wksp_idx = fd_topo_find_wksp( topo, wksp_name );
+  FD_TEST( wksp_idx!=ULONG_MAX );
+  fd_topo_wksp_t * wksp = &topo->workspaces[ wksp_idx ];
+  ulong part_max = fd_wksp_part_max_est( heap_size, 1U<<14U );
+  if( FD_UNLIKELY( !part_max ) ) FD_LOG_ERR(( "fd_wksp_part_max_est(%lu,16KiB) failed", funk_footprint ));
+  wksp->part_max += part_max;
+
+  return obj;
+}
+
 fd_topo_obj_t *
 setup_topo_store( fd_topo_t *  topo,
                   char const * wksp_name,
@@ -276,8 +305,8 @@ fd_topo_initialize( config_t * config ) {
   fd_topob_wksp( topo, "poh_shred"    );
   fd_topob_wksp( topo, "poh_replay"   );
 
-  /* TODO: WTF are these for? */
   fd_topob_wksp( topo, "funk"         );
+  fd_topob_wksp( topo, "progcache"    );
   fd_topob_wksp( topo, "bh_cmp"       );
   fd_topob_wksp( topo, "fec_sets"     );
   fd_topob_wksp( topo, "txncache"     );
@@ -708,6 +737,15 @@ fd_topo_initialize( config_t * config ) {
   FOR(resolv_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "resolv", i   ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_ONLY  );
   FD_TEST( fd_pod_insertf_ulong( topo->props, banks_obj->id, "banks" ) );
 
+  fd_topo_obj_t * progcache_obj = setup_topo_progcache( topo, "progcache",
+      fd_progcache_est_rec_max( config->firedancer.runtime.program_cache.heap_size_mib<<20,
+                                config->firedancer.runtime.program_cache.mean_cache_entry_size ),
+      config->firedancer.funk.max_database_transactions,
+      config->firedancer.runtime.program_cache.heap_size_mib<<20 );
+  /**/                 fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "replay", 0UL ) ], progcache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+  FOR(exec_tile_cnt)   fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec",   i   ) ], progcache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+  FOR(bank_tile_cnt)   fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "bank",   i   ) ], progcache_obj, FD_SHMEM_JOIN_MODE_READ_ONLY  );
+
   /* TODO: This should not exist in production */
   fd_topo_obj_t * bank_hash_cmp_obj = setup_topo_bank_hash_cmp( topo, "bh_cmp" );
   /**/               fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "replay", 0UL ) ], bank_hash_cmp_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
@@ -901,8 +939,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
 
     tile->replay.tx_metadata_storage = config->rpc.extended_tx_metadata_storage;
 
-    tile->replay.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX );
-    tile->replay.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
+    tile->replay.txncache_obj_id  = fd_pod_query_ulong( config->topo.props, "txncache",  ULONG_MAX ); FD_TEST( tile->replay.txncache_obj_id !=ULONG_MAX );
+    tile->replay.funk_obj_id      = fd_pod_query_ulong( config->topo.props, "funk",      ULONG_MAX ); FD_TEST( tile->replay.funk_obj_id     !=ULONG_MAX );
+    tile->replay.progcache_obj_id = fd_pod_query_ulong( config->topo.props, "progcache", ULONG_MAX ); FD_TEST( tile->replay.progcache_obj_id!=ULONG_MAX );
 
     strncpy( tile->replay.cluster_version, config->tiles.replay.cluster_version, sizeof(tile->replay.cluster_version) );
 
@@ -925,8 +964,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "exec" ) ) ) {
 
-    tile->exec.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
-    tile->exec.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX );
+    tile->exec.funk_obj_id      = fd_pod_query_ulong( config->topo.props, "funk",      ULONG_MAX ); FD_TEST( tile->exec.funk_obj_id     !=ULONG_MAX );
+    tile->exec.txncache_obj_id  = fd_pod_query_ulong( config->topo.props, "txncache",  ULONG_MAX ); FD_TEST( tile->exec.txncache_obj_id !=ULONG_MAX );
+    tile->exec.progcache_obj_id = fd_pod_query_ulong( config->topo.props, "progcache", ULONG_MAX ); FD_TEST( tile->exec.progcache_obj_id!=ULONG_MAX );
 
     tile->exec.max_live_slots = config->firedancer.runtime.max_live_slots;
 
@@ -999,8 +1039,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
     }
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "bank" ) ) ) {
-    tile->bank.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX );
-    tile->bank.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
+    tile->bank.txncache_obj_id  = fd_pod_query_ulong( config->topo.props, "txncache",  ULONG_MAX );
+    tile->bank.funk_obj_id      = fd_pod_query_ulong( config->topo.props, "funk",      ULONG_MAX );
+    tile->bank.progcache_obj_id = fd_pod_query_ulong( config->topo.props, "progcache", ULONG_MAX );
 
     tile->bank.max_live_slots = config->firedancer.runtime.max_live_slots;
 

src/app/firedancer/topology.h

@@ -30,6 +30,13 @@ setup_topo_funk( fd_topo_t *  topo,
                  ulong        heap_size_gib,
                  int          lock_pages );
 
+fd_topo_obj_t *
+setup_topo_progcache( fd_topo_t *  topo,
+                      char const * wksp_name,
+                      ulong        max_cache_entries,
+                      ulong        max_database_transactions,
+                      ulong        heap_size_gib );
+
 fd_topo_obj_t *
 setup_topo_runtime_pub( fd_topo_t *  topo,
                         char const * wksp_name,

src/app/shared/fd_config.h

@@ -112,6 +112,11 @@ struct fd_configf {
     ulong max_live_slots;
     ulong max_vote_accounts;
     ulong max_fork_width;
+
+    struct {
+      ulong heap_size_mib;
+      ulong mean_cache_entry_size;
+    } program_cache;
   } runtime;
 
   struct {

src/app/shared/fd_config_parse.c

@@ -91,6 +91,9 @@ fd_config_extract_podf( uchar *        pod,
   CFG_POP      ( ulong,  runtime.max_vote_accounts                           );
   CFG_POP      ( ulong,  runtime.max_fork_width                              );
 
+  CFG_POP      ( ulong,  runtime.program_cache.heap_size_mib                 );
+  CFG_POP      ( ulong,  runtime.program_cache.mean_cache_entry_size         );
+
   CFG_POP      ( ulong,  store.max_completed_shred_sets                      );
 
   CFG_POP      ( bool,   snapshots.incremental_snapshots                     );

src/ballet/sbpf/fixtures/malformed_bytecode.so

@@ -35,6 +35,7 @@ class Tile(Enum):
     BANKF = 29
     RESOLF = 30
     BACKT = 31
+    EXEC = 32
 
 class MetricType(Enum):
     COUNTER = 0

src/disco/metrics/generate/types.py

@@ -64,6 +64,7 @@ const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT] = {
     "bankf",
     "resolf",
     "backt",
+    "exec",
 };
 
 const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
@@ -95,6 +96,7 @@ const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
     FD_METRICS_BANKF_TOTAL,
     FD_METRICS_RESOLF_TOTAL,
     FD_METRICS_BACKT_TOTAL,
+    FD_METRICS_EXEC_TOTAL,
 };
 const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT] = {
     FD_METRICS_NET,
@@ -125,4 +127,5 @@ const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT]
     FD_METRICS_BANKF,
     FD_METRICS_RESOLF,
     FD_METRICS_BACKT,
+    FD_METRICS_EXEC,
 };