diff --git a/book/api/metrics-generated.md b/book/api/metrics-generated.md
index 938f9afea98..336f641c3cf 100644
--- a/book/api/metrics-generated.md
+++ b/book/api/metrics-generated.md
@@ -1057,3 +1057,19 @@
| backt_start_slot | gauge | The slot at which the backtest started |
+
+## Exec Tile
+
+
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| exec_progcache_misses | counter | Number of program cache misses |
+| exec_progcache_hits | counter | Number of program cache hits |
+| exec_progcache_fills | counter | Number of program cache insertions |
+| exec_progcache_fill_tot_sz | counter | Total number of bytes inserted into program cache |
+| exec_progcache_fill_fails | counter | Number of program cache load fails (tombstones inserted) |
+| exec_progcache_dup_inserts | counter | Number of time two tiles raced to insert the same cache entry |
+| exec_progcache_invalidations | counter | Number of program cache invalidations |
+
+
diff --git a/src/app/firedancer-dev/commands/backtest.c b/src/app/firedancer-dev/commands/backtest.c
index 8076d3991ea..d95851a589e 100644
--- a/src/app/firedancer-dev/commands/backtest.c
+++ b/src/app/firedancer-dev/commands/backtest.c
@@ -28,6 +28,7 @@
#include "../../../ballet/lthash/fd_lthash.h"
#include "../../../flamenco/runtime/context/fd_capture_ctx.h"
#include "../../../disco/pack/fd_pack_cost.h"
+#include "../../../flamenco/progcache/fd_progcache_admin.h"
#include "../main.h"
@@ -77,9 +78,16 @@ backtest_topo( config_t * config ) {
config->firedancer.funk.max_database_transactions,
config->firedancer.funk.heap_size_gib,
config->firedancer.funk.lock_pages );
-
fd_topob_tile_uses( topo, replay_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+ fd_topob_wksp( topo, "progcache" );
+ fd_topo_obj_t * progcache_obj = setup_topo_progcache( topo, "progcache",
+ fd_progcache_est_rec_max( config->firedancer.runtime.program_cache.heap_size_mib<<20,
+ config->firedancer.runtime.program_cache.mean_cache_entry_size ),
+ config->firedancer.funk.max_database_transactions,
+ config->firedancer.runtime.program_cache.heap_size_mib<<20 );
+ fd_topob_tile_uses( topo, replay_tile, progcache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+
/**********************************************************************/
/* Add the executor tiles to topo */
/**********************************************************************/
diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml
index 13b5c805b2f..f0f49de7abb 100644
--- a/src/app/firedancer/config/default.toml
+++ b/src/app/firedancer/config/default.toml
@@ -533,6 +533,17 @@ user = ""
# max_fork_width, the client will crash.
max_fork_width = 32
+ # The program cache pre-loads frequently executed programs for
+ # faster transaction execution.
+ [runtime.program_cache]
+ # The size of the loaded program cache in MiB.
+ heap_size_mib = 2048
+
+ # The mean expected heap utilization of a cache entry. Controls
+ # the size of metadata structures (e.g. cache entry table). It
+ # is not recommended to change this setting.
+ mean_cache_entry_size = 131072
+
# This section configures the "groove" persistent account database.
# [groove]
# ...
diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c
index c3df3c6ab85..3b50930dc95 100644
--- a/src/app/firedancer/topology.c
+++ b/src/app/firedancer/topology.c
@@ -21,7 +21,7 @@
#include "../../discof/restore/utils/fd_ssmsg.h"
#include "../../flamenco/gossip/fd_gossip.h"
#include "../../flamenco/runtime/context/fd_capture_ctx.h"
-#include "../../funk/fd_funk.h" /* funk_footprint() */
+#include "../../flamenco/progcache/fd_progcache_admin.h"
#include
#include
@@ -99,6 +99,35 @@ setup_topo_funk( fd_topo_t * topo,
return obj;
}
+fd_topo_obj_t *
+setup_topo_progcache( fd_topo_t * topo,
+ char const * wksp_name,
+ ulong max_cache_entries,
+ ulong max_database_transactions,
+ ulong heap_size ) {
+ fd_topo_obj_t * obj = fd_topob_obj( topo, "funk", wksp_name );
+ FD_TEST( fd_pod_insert_ulong( topo->props, "progcache", obj->id ) );
+ FD_TEST( fd_pod_insertf_ulong( topo->props, max_cache_entries, "obj.%lu.rec_max", obj->id ) );
+ FD_TEST( fd_pod_insertf_ulong( topo->props, max_database_transactions, "obj.%lu.txn_max", obj->id ) );
+ FD_TEST( fd_pod_insertf_ulong( topo->props, heap_size, "obj.%lu.heap_max", obj->id ) );
+ ulong funk_footprint = fd_funk_footprint( max_database_transactions, max_cache_entries );
+ if( FD_UNLIKELY( !funk_footprint ) ) FD_LOG_ERR(( "Invalid [runtime.program_cache] parameters" ));
+ if( FD_UNLIKELY( heap_size<(2*funk_footprint) ) ) {
+ FD_LOG_ERR(( "Invalid [runtime.program_cache] parameters: heap_size_mib should be at least %lu",
+ ( 4*funk_footprint )>>20 ));
+ }
+
+ /* Increase workspace partition count */
+ ulong wksp_idx = fd_topo_find_wksp( topo, wksp_name );
+ FD_TEST( wksp_idx!=ULONG_MAX );
+ fd_topo_wksp_t * wksp = &topo->workspaces[ wksp_idx ];
+ ulong part_max = fd_wksp_part_max_est( heap_size, 1U<<14U );
+ if( FD_UNLIKELY( !part_max ) ) FD_LOG_ERR(( "fd_wksp_part_max_est(%lu,16KiB) failed", funk_footprint ));
+ wksp->part_max += part_max;
+
+ return obj;
+}
+
fd_topo_obj_t *
setup_topo_store( fd_topo_t * topo,
char const * wksp_name,
@@ -276,8 +305,8 @@ fd_topo_initialize( config_t * config ) {
fd_topob_wksp( topo, "poh_shred" );
fd_topob_wksp( topo, "poh_replay" );
- /* TODO: WTF are these for? */
fd_topob_wksp( topo, "funk" );
+ fd_topob_wksp( topo, "progcache" );
fd_topob_wksp( topo, "bh_cmp" );
fd_topob_wksp( topo, "fec_sets" );
fd_topob_wksp( topo, "txncache" );
@@ -708,6 +737,15 @@ fd_topo_initialize( config_t * config ) {
FOR(resolv_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "resolv", i ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
FD_TEST( fd_pod_insertf_ulong( topo->props, banks_obj->id, "banks" ) );
+ fd_topo_obj_t * progcache_obj = setup_topo_progcache( topo, "progcache",
+ fd_progcache_est_rec_max( config->firedancer.runtime.program_cache.heap_size_mib<<20,
+ config->firedancer.runtime.program_cache.mean_cache_entry_size ),
+ config->firedancer.funk.max_database_transactions,
+ config->firedancer.runtime.program_cache.heap_size_mib<<20 );
+ /**/ fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "replay", 0UL ) ], progcache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+ FOR(exec_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec", i ) ], progcache_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
+ FOR(bank_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "bank", i ) ], progcache_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
+
/* TODO: This should not exist in production */
fd_topo_obj_t * bank_hash_cmp_obj = setup_topo_bank_hash_cmp( topo, "bh_cmp" );
/**/ fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "replay", 0UL ) ], bank_hash_cmp_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
@@ -901,8 +939,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
tile->replay.tx_metadata_storage = config->rpc.extended_tx_metadata_storage;
- tile->replay.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX );
- tile->replay.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
+ tile->replay.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX ); FD_TEST( tile->replay.txncache_obj_id !=ULONG_MAX );
+ tile->replay.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX ); FD_TEST( tile->replay.funk_obj_id !=ULONG_MAX );
+ tile->replay.progcache_obj_id = fd_pod_query_ulong( config->topo.props, "progcache", ULONG_MAX ); FD_TEST( tile->replay.progcache_obj_id!=ULONG_MAX );
strncpy( tile->replay.cluster_version, config->tiles.replay.cluster_version, sizeof(tile->replay.cluster_version) );
@@ -925,8 +964,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
} else if( FD_UNLIKELY( !strcmp( tile->name, "exec" ) ) ) {
- tile->exec.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
- tile->exec.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX );
+ tile->exec.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX ); FD_TEST( tile->exec.funk_obj_id !=ULONG_MAX );
+ tile->exec.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX ); FD_TEST( tile->exec.txncache_obj_id !=ULONG_MAX );
+ tile->exec.progcache_obj_id = fd_pod_query_ulong( config->topo.props, "progcache", ULONG_MAX ); FD_TEST( tile->exec.progcache_obj_id!=ULONG_MAX );
tile->exec.max_live_slots = config->firedancer.runtime.max_live_slots;
@@ -999,8 +1039,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
}
} else if( FD_UNLIKELY( !strcmp( tile->name, "bank" ) ) ) {
- tile->bank.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX );
- tile->bank.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
+ tile->bank.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX );
+ tile->bank.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
+ tile->bank.progcache_obj_id = fd_pod_query_ulong( config->topo.props, "progcache", ULONG_MAX );
tile->bank.max_live_slots = config->firedancer.runtime.max_live_slots;
diff --git a/src/app/firedancer/topology.h b/src/app/firedancer/topology.h
index e108973f0a5..7b912dd5ec1 100644
--- a/src/app/firedancer/topology.h
+++ b/src/app/firedancer/topology.h
@@ -30,6 +30,13 @@ setup_topo_funk( fd_topo_t * topo,
ulong heap_size_gib,
int lock_pages );
+fd_topo_obj_t *
+setup_topo_progcache( fd_topo_t * topo,
+ char const * wksp_name,
+ ulong max_cache_entries,
+ ulong max_database_transactions,
+ ulong heap_size_gib );
+
fd_topo_obj_t *
setup_topo_runtime_pub( fd_topo_t * topo,
char const * wksp_name,
diff --git a/src/app/shared/fd_config.h b/src/app/shared/fd_config.h
index f5011c9fd74..210e5924d1c 100644
--- a/src/app/shared/fd_config.h
+++ b/src/app/shared/fd_config.h
@@ -112,6 +112,11 @@ struct fd_configf {
ulong max_live_slots;
ulong max_vote_accounts;
ulong max_fork_width;
+
+ struct {
+ ulong heap_size_mib;
+ ulong mean_cache_entry_size;
+ } program_cache;
} runtime;
struct {
diff --git a/src/app/shared/fd_config_parse.c b/src/app/shared/fd_config_parse.c
index 04c94388f70..b0f11e84c87 100644
--- a/src/app/shared/fd_config_parse.c
+++ b/src/app/shared/fd_config_parse.c
@@ -91,6 +91,9 @@ fd_config_extract_podf( uchar * pod,
CFG_POP ( ulong, runtime.max_vote_accounts );
CFG_POP ( ulong, runtime.max_fork_width );
+ CFG_POP ( ulong, runtime.program_cache.heap_size_mib );
+ CFG_POP ( ulong, runtime.program_cache.mean_cache_entry_size );
+
CFG_POP ( ulong, store.max_completed_shred_sets );
CFG_POP ( bool, snapshots.incremental_snapshots );
diff --git a/src/ballet/sbpf/fixtures/malformed_bytecode.so b/src/ballet/sbpf/fixtures/malformed_bytecode.so
new file mode 100644
index 00000000000..c359c0887bc
Binary files /dev/null and b/src/ballet/sbpf/fixtures/malformed_bytecode.so differ
diff --git a/src/disco/metrics/generate/types.py b/src/disco/metrics/generate/types.py
index 8e3ac1f6995..26fedf2b643 100644
--- a/src/disco/metrics/generate/types.py
+++ b/src/disco/metrics/generate/types.py
@@ -35,6 +35,7 @@ class Tile(Enum):
BANKF = 29
RESOLF = 30
BACKT = 31
+ EXEC = 32
class MetricType(Enum):
COUNTER = 0
diff --git a/src/disco/metrics/generated/fd_metrics_all.c b/src/disco/metrics/generated/fd_metrics_all.c
index 1dd3fda6c08..7d78226110d 100644
--- a/src/disco/metrics/generated/fd_metrics_all.c
+++ b/src/disco/metrics/generated/fd_metrics_all.c
@@ -64,6 +64,7 @@ const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT] = {
"bankf",
"resolf",
"backt",
+ "exec",
};
const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
@@ -95,6 +96,7 @@ const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
FD_METRICS_BANKF_TOTAL,
FD_METRICS_RESOLF_TOTAL,
FD_METRICS_BACKT_TOTAL,
+ FD_METRICS_EXEC_TOTAL,
};
const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT] = {
FD_METRICS_NET,
@@ -125,4 +127,5 @@ const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT]
FD_METRICS_BANKF,
FD_METRICS_RESOLF,
FD_METRICS_BACKT,
+ FD_METRICS_EXEC,
};
diff --git a/src/disco/metrics/generated/fd_metrics_all.h b/src/disco/metrics/generated/fd_metrics_all.h
index 69a9040b6a0..caf56c5eeeb 100644
--- a/src/disco/metrics/generated/fd_metrics_all.h
+++ b/src/disco/metrics/generated/fd_metrics_all.h
@@ -30,6 +30,7 @@
#include "fd_metrics_metric.h"
#include "fd_metrics_ipecho.h"
#include "fd_metrics_backt.h"
+#include "fd_metrics_exec.h"
/* Start of LINK OUT metrics */
#define FD_METRICS_COUNTER_LINK_SLOW_COUNT_OFF (0UL)
@@ -166,7 +167,7 @@ extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_OUT[FD_METRICS_ALL_LINK_OUT_T
#define FD_METRICS_TOTAL_SZ (8UL*253UL)
-#define FD_METRICS_TILE_KIND_CNT 28
+#define FD_METRICS_TILE_KIND_CNT 29
extern const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT];
extern const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT];
extern const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT];
diff --git a/src/disco/metrics/generated/fd_metrics_exec.c b/src/disco/metrics/generated/fd_metrics_exec.c
new file mode 100644
index 00000000000..1aa26fbd183
--- /dev/null
+++ b/src/disco/metrics/generated/fd_metrics_exec.c
@@ -0,0 +1,12 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+#include "fd_metrics_exec.h"
+
+const fd_metrics_meta_t FD_METRICS_EXEC[FD_METRICS_EXEC_TOTAL] = {
+ DECLARE_METRIC( EXEC_PROGCACHE_MISSES, COUNTER ),
+ DECLARE_METRIC( EXEC_PROGCACHE_HITS, COUNTER ),
+ DECLARE_METRIC( EXEC_PROGCACHE_FILLS, COUNTER ),
+ DECLARE_METRIC( EXEC_PROGCACHE_FILL_TOT_SZ, COUNTER ),
+ DECLARE_METRIC( EXEC_PROGCACHE_FILL_FAILS, COUNTER ),
+ DECLARE_METRIC( EXEC_PROGCACHE_DUP_INSERTS, COUNTER ),
+ DECLARE_METRIC( EXEC_PROGCACHE_INVALIDATIONS, COUNTER ),
+};
diff --git a/src/disco/metrics/generated/fd_metrics_exec.h b/src/disco/metrics/generated/fd_metrics_exec.h
new file mode 100644
index 00000000000..060c39f6af3
--- /dev/null
+++ b/src/disco/metrics/generated/fd_metrics_exec.h
@@ -0,0 +1,49 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+
+#include "../fd_metrics_base.h"
+#include "fd_metrics_enums.h"
+
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_MISSES_OFF (16UL)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_MISSES_NAME "exec_progcache_misses"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_MISSES_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_MISSES_DESC "Number of program cache misses"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_MISSES_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_HITS_OFF (17UL)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_HITS_NAME "exec_progcache_hits"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_HITS_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_HITS_DESC "Number of program cache hits"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_HITS_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILLS_OFF (18UL)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILLS_NAME "exec_progcache_fills"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILLS_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILLS_DESC "Number of program cache insertions"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILLS_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_TOT_SZ_OFF (19UL)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_TOT_SZ_NAME "exec_progcache_fill_tot_sz"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_TOT_SZ_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_TOT_SZ_DESC "Total number of bytes inserted into program cache"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_TOT_SZ_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_FAILS_OFF (20UL)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_FAILS_NAME "exec_progcache_fill_fails"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_FAILS_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_FAILS_DESC "Number of program cache load fails (tombstones inserted)"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_FILL_FAILS_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_DUP_INSERTS_OFF (21UL)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_DUP_INSERTS_NAME "exec_progcache_dup_inserts"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_DUP_INSERTS_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_DUP_INSERTS_DESC "Number of time two tiles raced to insert the same cache entry"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_DUP_INSERTS_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_INVALIDATIONS_OFF (22UL)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_INVALIDATIONS_NAME "exec_progcache_invalidations"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_INVALIDATIONS_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_INVALIDATIONS_DESC "Number of program cache invalidations"
+#define FD_METRICS_COUNTER_EXEC_PROGCACHE_INVALIDATIONS_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_EXEC_TOTAL (7UL)
+extern const fd_metrics_meta_t FD_METRICS_EXEC[FD_METRICS_EXEC_TOTAL];
diff --git a/src/disco/metrics/metrics.xml b/src/disco/metrics/metrics.xml
index 44a49e0fc8a..396fbbf53a1 100644
--- a/src/disco/metrics/metrics.xml
+++ b/src/disco/metrics/metrics.xml
@@ -1062,4 +1062,14 @@ metric introduced.
+
+
+
+
+
+
+
+
+
+
diff --git a/src/disco/topo/fd_topo.c b/src/disco/topo/fd_topo.c
index b5705257cd7..20164a19856 100644
--- a/src/disco/topo/fd_topo.c
+++ b/src/disco/topo/fd_topo.c
@@ -10,6 +10,17 @@
#include
#include
+void *
+fd_topo_obj_laddr( fd_topo_t const * topo,
+ ulong obj_id ) {
+ fd_topo_obj_t const * obj = &topo->objs[ obj_id ];
+ if( FD_UNLIKELY( obj_id==ULONG_MAX ) ) FD_LOG_CRIT(( "invalid obj_id ULONG_MAX" ));
+ if( FD_UNLIKELY( obj_id>=FD_TOPO_MAX_OBJS ) ) FD_LOG_CRIT(( "invalid obj_id %lu", obj_id ));
+ FD_TEST( obj->id == obj_id );
+ FD_TEST( obj->offset );
+ return (void *)((ulong)topo->workspaces[ obj->wksp_id ].wksp + obj->offset);
+}
+
void
fd_topo_join_workspace( fd_topo_t * topo,
fd_topo_wksp_t * wksp,
diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h
index 052598fd46e..dbe733af13e 100644
--- a/src/disco/topo/fd_topo.h
+++ b/src/disco/topo/fd_topo.h
@@ -353,6 +353,7 @@ struct fd_topo_tile {
int tx_metadata_storage;
ulong funk_obj_id;
ulong txncache_obj_id;
+ ulong progcache_obj_id;
char shred_cap[ PATH_MAX ];
char cluster_version[ 32 ];
@@ -381,6 +382,7 @@ struct fd_topo_tile {
struct {
ulong funk_obj_id;
ulong txncache_obj_id;
+ ulong progcache_obj_id;
ulong max_live_slots;
@@ -543,6 +545,7 @@ struct fd_topo_tile {
ulong txncache_obj_id;
ulong funk_obj_id;
+ ulong progcache_obj_id;
} bank;
struct {
@@ -643,15 +646,9 @@ fd_topo_workspace_align( void ) {
return 4096UL;
}
-static inline void *
+void *
fd_topo_obj_laddr( fd_topo_t const * topo,
- ulong obj_id ) {
- fd_topo_obj_t const * obj = &topo->objs[ obj_id ];
- FD_TEST( obj_idid == obj_id );
- FD_TEST( obj->offset );
- return (void *)((ulong)topo->workspaces[ obj->wksp_id ].wksp + obj->offset);
-}
+ ulong obj_id );
/* Returns a pointer in the local address space to the base address of
the workspace out of which the given object was allocated. */
diff --git a/src/discof/bank/fd_bank_tile.c b/src/discof/bank/fd_bank_tile.c
index 7b3dcdf3e6d..786065c5dc3 100644
--- a/src/discof/bank/fd_bank_tile.c
+++ b/src/discof/bank/fd_bank_tile.c
@@ -242,7 +242,7 @@ handle_microblock( fd_bank_ctx_t * ctx,
if that happens. We cannot reject the transaction here as there
would be no way to undo the partially applied changes to the bank
in finalize anyway. */
- fd_runtime_finalize_txn( ctx->txn_ctx->funk, txn_ctx->status_cache, txn_ctx->xid, txn_ctx, bank, NULL );
+ fd_runtime_finalize_txn( ctx->txn_ctx->funk, ctx->txn_ctx->progcache, txn_ctx->status_cache, txn_ctx->xid, txn_ctx, bank, NULL );
FD_TEST( txn->flags );
}
diff --git a/src/discof/exec/fd_exec_tile.c b/src/discof/exec/fd_exec_tile.c
index 2fdee0af1dc..1432cfc6789 100644
--- a/src/discof/exec/fd_exec_tile.c
+++ b/src/discof/exec/fd_exec_tile.c
@@ -60,7 +60,10 @@ typedef struct fd_exec_tile_ctx {
fd_funk_t.
TODO: These should probably be made read-only handles. */
fd_banks_t * banks;
- fd_funk_t funk[ 1 ];
+ void * shfunk;
+ void * shprogcache;
+ fd_funk_t funk[1];
+ fd_progcache_t progcache[1];
fd_txncache_t * txncache;
@@ -81,9 +84,21 @@ scratch_footprint( fd_topo_tile_t const * tile ) {
l = FD_LAYOUT_APPEND( l, alignof(fd_exec_tile_ctx_t), sizeof(fd_exec_tile_ctx_t) );
l = FD_LAYOUT_APPEND( l, fd_capture_ctx_align(), fd_capture_ctx_footprint() );
l = FD_LAYOUT_APPEND( l, fd_txncache_align(), fd_txncache_footprint( tile->exec.max_live_slots ) );
+ l = FD_LAYOUT_APPEND( l, FD_PROGCACHE_SCRATCH_ALIGN, FD_PROGCACHE_SCRATCH_FOOTPRINT );
return FD_LAYOUT_FINI( l, scratch_align() );
}
+static void
+metrics_write( fd_exec_tile_ctx_t * ctx ) {
+ fd_progcache_t * progcache = ctx->progcache;
+ FD_MCNT_SET( EXEC, PROGCACHE_MISSES, progcache->metrics->miss_cnt );
+ FD_MCNT_SET( EXEC, PROGCACHE_HITS, progcache->metrics->hit_cnt );
+ FD_MCNT_SET( EXEC, PROGCACHE_FILLS, progcache->metrics->fill_cnt );
+ FD_MCNT_SET( EXEC, PROGCACHE_FILL_TOT_SZ, progcache->metrics->fill_tot_sz );
+ FD_MCNT_SET( EXEC, PROGCACHE_INVALIDATIONS, progcache->metrics->invalidate_cnt );
+ FD_MCNT_SET( EXEC, PROGCACHE_DUP_INSERTS, progcache->metrics->dup_insert_cnt );
+}
+
static inline int
returnable_frag( fd_exec_tile_ctx_t * ctx,
ulong in_idx,
@@ -117,7 +132,7 @@ returnable_frag( fd_exec_tile_ctx_t * ctx,
fd_bank_t * bank = fd_banks_bank_query( ctx->banks, msg->bank_idx );
if( FD_LIKELY( ctx->txn_ctx->flags & FD_TXN_P_FLAGS_EXECUTE_SUCCESS ) ) {
fd_funk_txn_xid_t xid = (fd_funk_txn_xid_t){ .ul = { fd_bank_slot_get( bank ), fd_bank_slot_get( bank ) } };
- fd_runtime_finalize_txn( ctx->funk, ctx->txncache, &xid, ctx->txn_ctx, bank, ctx->capture_ctx );
+ fd_runtime_finalize_txn( ctx->funk, ctx->progcache, ctx->txncache, &xid, ctx->txn_ctx, bank, ctx->capture_ctx );
}
/* Notify replay. */
@@ -157,6 +172,7 @@ unprivileged_init( fd_topo_t * topo,
fd_exec_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_exec_tile_ctx_t), sizeof(fd_exec_tile_ctx_t) );
void * capture_ctx_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_capture_ctx_align(), fd_capture_ctx_footprint() );
void * _txncache = FD_SCRATCH_ALLOC_APPEND( l, fd_txncache_align(), fd_txncache_footprint( tile->exec.max_live_slots ) );
+ uchar * pc_scratch = FD_SCRATCH_ALLOC_APPEND( l, FD_PROGCACHE_SCRATCH_ALIGN, FD_PROGCACHE_SCRATCH_FOOTPRINT );
ulong scratch_alloc_mem = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
if( FD_UNLIKELY( scratch_alloc_mem - (ulong)scratch - scratch_footprint( tile ) ) ) {
@@ -239,15 +255,15 @@ unprivileged_init( fd_topo_t * topo,
FD_LOG_ERR(( "Failed to join bank hash cmp" ));
}
- /********************************************************************/
- /* funk-specific setup */
- /********************************************************************/
-
- FD_TEST( fd_funk_join( ctx->funk, fd_topo_obj_laddr( topo, tile->exec.funk_obj_id ) ) );
+ void * shfunk = fd_topo_obj_laddr( topo, tile->exec.funk_obj_id );
+ if( FD_UNLIKELY( !fd_funk_join( ctx->funk, shfunk ) ) ) {
+ FD_LOG_CRIT(( "fd_funk_join(accdb) failed" ));
+ }
- /********************************************************************/
- /* setup txncache */
- /********************************************************************/
+ void * shprogcache = fd_topo_obj_laddr( topo, tile->exec.progcache_obj_id );
+ if( FD_UNLIKELY( !fd_progcache_join( ctx->progcache, shprogcache, pc_scratch, FD_PROGCACHE_SCRATCH_FOOTPRINT ) ) ) {
+ FD_LOG_CRIT(( "fd_progcache_join() failed" ));
+ }
void * _txncache_shmem = fd_topo_obj_laddr( topo, tile->exec.txncache_obj_id );
fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( _txncache_shmem );
@@ -262,7 +278,13 @@ unprivileged_init( fd_topo_t * topo,
fd_spad_push( ctx->exec_spad );
uchar * txn_ctx_mem = fd_spad_alloc_check( ctx->exec_spad, FD_EXEC_TXN_CTX_ALIGN, FD_EXEC_TXN_CTX_FOOTPRINT );
ctx->txn_ctx = fd_exec_txn_ctx_join( fd_exec_txn_ctx_new( txn_ctx_mem ), ctx->exec_spad, ctx->exec_spad_wksp );
- ctx->txn_ctx->funk[0] = *ctx->funk;
+ if( FD_UNLIKELY( !fd_funk_join( ctx->txn_ctx->funk, shfunk ) ) ) {
+ FD_LOG_CRIT(( "fd_funk_join(accdb) failed" ));
+ }
+ ctx->txn_ctx->progcache = fd_progcache_join( ctx->txn_ctx->_progcache, shprogcache, pc_scratch, FD_PROGCACHE_SCRATCH_FOOTPRINT );
+ if( FD_UNLIKELY( !ctx->txn_ctx->progcache ) ) {
+ FD_LOG_CRIT(( "fd_progcache_join() failed" ));
+ }
ctx->txn_ctx->status_cache = ctx->txncache;
ctx->txn_ctx->bank_hash_cmp = ctx->bank_hash_cmp;
ctx->txn_ctx->spad = ctx->exec_spad;
@@ -413,16 +435,17 @@ populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED,
#define STEM_CALLBACK_AFTER_CREDIT after_credit
#define STEM_CALLBACK_RETURNABLE_FRAG returnable_frag
+#define STEM_CALLBACK_METRICS_WRITE metrics_write
#include "../../disco/stem/fd_stem.c"
fd_topo_run_tile_t fd_tile_execor = {
- .name = "exec",
- .loose_footprint = 0UL,
- .populate_allowed_seccomp = populate_allowed_seccomp,
- .populate_allowed_fds = populate_allowed_fds,
- .scratch_align = scratch_align,
- .scratch_footprint = scratch_footprint,
- .unprivileged_init = unprivileged_init,
- .run = stem_run,
+ .name = "exec",
+ .loose_footprint = 0UL,
+ .populate_allowed_seccomp = populate_allowed_seccomp,
+ .populate_allowed_fds = populate_allowed_fds,
+ .scratch_align = scratch_align,
+ .scratch_footprint = scratch_footprint,
+ .unprivileged_init = unprivileged_init,
+ .run = stem_run,
};
diff --git a/src/discof/replay/fd_replay_tile.c b/src/discof/replay/fd_replay_tile.c
index 0b48a19a89e..f74596ebe13 100644
--- a/src/discof/replay/fd_replay_tile.c
+++ b/src/discof/replay/fd_replay_tile.c
@@ -18,6 +18,7 @@
#include "../../util/pod/fd_pod.h"
#include "../../flamenco/rewards/fd_rewards.h"
#include "../../flamenco/leaders/fd_multi_epoch_leaders.h"
+#include "../../flamenco/progcache/fd_progcache_admin.h"
#include "../../disco/metrics/fd_metrics.h"
#include "../../flamenco/runtime/fd_runtime.h"
@@ -132,6 +133,7 @@ struct fd_replay_tile {
int tx_metadata_storage;
fd_funk_t funk[1];
+ fd_progcache_admin_t progcache_admin[1];
fd_txncache_t * txncache;
fd_store_t * store;
@@ -667,7 +669,8 @@ replay_block_start( fd_replay_tile_t * ctx,
fd_funk_txn_xid_t xid = { .ul = { slot, slot } };
fd_funk_txn_xid_t parent_xid = { .ul = { parent_slot, parent_slot } };
- fd_funk_txn_prepare( ctx->funk, &parent_xid, &xid );
+ fd_funk_txn_prepare ( ctx->funk, &parent_xid, &xid );
+ fd_progcache_txn_prepare( ctx->progcache_admin, &parent_xid, &xid );
/* Update any required runtime state and handle any potential epoch
boundary change. */
@@ -905,7 +908,8 @@ prepare_leader_bank( fd_replay_tile_t * ctx,
/* prepare the funk transaction for the leader bank */
fd_funk_txn_xid_t xid = { .ul = { slot, slot } };
fd_funk_txn_xid_t parent_xid = { .ul = { parent_slot, parent_slot } };
- fd_funk_txn_prepare( ctx->funk, &parent_xid, &xid );
+ fd_funk_txn_prepare ( ctx->funk, &parent_xid, &xid );
+ fd_progcache_txn_prepare( ctx->progcache_admin, &parent_xid, &xid );
fd_bank_execution_fees_set( ctx->leader_bank, 0UL );
fd_bank_priority_fees_set( ctx->leader_bank, 0UL );
@@ -1030,6 +1034,43 @@ publish_root_advanced( fd_replay_tile_t * ctx,
ctx->replay_out->chunk = fd_dcache_compact_next( ctx->replay_out->chunk, sizeof(fd_replay_root_advanced_t), ctx->replay_out->chunk0, ctx->replay_out->wmark );
}
+/* init_funk performs pre-flight checks for the account database and
+ program cache. Ensures that the account database was set up
+ correctly by bootstrap components (e.g. genesis or snapshot loader).
+ Mirrors the account database's fork tree down to the program cache. */
+
+static void
+init_funk( fd_replay_tile_t * ctx,
+ ulong bank_slot ) {
+ /* Ensure that the loaded bank root corresponds to the account
+ database's root. */
+ if( FD_UNLIKELY( !ctx->funk->shmem ) ) {
+ FD_LOG_CRIT(( "failed to initialize account database: replay tile is not joined to database shared memory objects" ));
+ }
+ fd_funk_txn_xid_t const * accdb_pub = fd_funk_last_publish( ctx->funk );
+ if( FD_UNLIKELY( accdb_pub->ul[0]!=bank_slot ) ) {
+ FD_LOG_CRIT(( "failed to initialize account database: accdb is at slot %lu, but chain state is at slot %lu\n"
+ "This is a bug in startup components.",
+ accdb_pub->ul[0], bank_slot ));
+ }
+ if( FD_UNLIKELY( fd_funk_last_publish_is_frozen( ctx->funk ) ) ) {
+ FD_LOG_CRIT(( "failed to initialize account database: accdb fork graph is not clean.\n"
+ "The account database should only contain state for the root slot at this point,\n"
+ "but there are incomplete database transactions leftover.\n"
+ "This is a bug in startup components." ));
+ }
+
+ /* The program cache tracks the account database's fork graph at all
+ times. Perform initial synchronization: pivot from funk 'root' (a
+ sentinel XID) to 'last publish' (the bootstrap root slot). */
+ if( FD_UNLIKELY( !ctx->progcache_admin->funk->shmem ) ) {
+ FD_LOG_CRIT(( "failed to initialize account database: replay tile is not joined to program cache" ));
+ }
+ fd_progcache_clear( ctx->progcache_admin );
+ fd_progcache_txn_prepare( ctx->progcache_admin, fd_funk_root( ctx->progcache_admin->funk ), fd_funk_last_publish( ctx->funk ) );
+ fd_progcache_txn_publish( ctx->progcache_admin, fd_funk_last_publish( ctx->funk ) );
+}
+
static void
init_after_snapshot( fd_replay_tile_t * ctx ) {
/* Now that the snapshot has been loaded in, we have to refresh the
@@ -1042,9 +1083,10 @@ init_after_snapshot( fd_replay_tile_t * ctx ) {
FD_LOG_CRIT(( "invariant violation: replay bank is NULL at bank index %lu", FD_REPLAY_BOOT_BANK_IDX ));
}
- fd_stake_delegations_t * root_delegations = fd_banks_stake_delegations_root_query( ctx->banks );
-
fd_funk_txn_xid_t xid = { .ul = { fd_bank_slot_get( bank ), fd_bank_slot_get( bank ) } };
+ init_funk( ctx, fd_bank_slot_get( bank ) );
+
+ fd_stake_delegations_t * root_delegations = fd_banks_stake_delegations_root_query( ctx->banks );
fd_stake_delegations_refresh( root_delegations, ctx->funk, &xid );
@@ -1445,8 +1487,6 @@ dispatch_task( fd_replay_tile_t * ctx,
have to loop over all accounts a second time. */
/* Insert or reverify invoked programs for this epoch, if needed. */
fd_bank_t * bank = fd_banks_bank_query( ctx->banks, task->txn_exec->bank_idx );
- fd_funk_txn_xid_t xid = { .ul = { task->txn_exec->slot, task->txn_exec->slot } };
- fd_runtime_update_program_cache( bank, ctx->funk, &xid, txn_p, ctx->runtime_spad );
/* Add the transaction to the block dumper if necessary. This
logic doesn't need to be fork-aware since it's only meant to
@@ -1662,7 +1702,8 @@ funk_publish( fd_replay_tile_t * ctx,
/* This is the standard case. Publish all transactions up to and
including the watermark. This will publish any in-prep ancestors
of root_txn as well. */
- if( FD_UNLIKELY( !fd_funk_txn_publish( ctx->funk, &xid ) ) ) FD_LOG_CRIT(( "failed to funk publish slot %lu", slot ));
+ if( FD_UNLIKELY( !fd_funk_txn_publish( ctx->funk, &xid ) ) ) FD_LOG_CRIT(( "failed to root slot %lu: fd_funk_txn_publish(accdb) failed", slot ));
+ fd_progcache_txn_publish( ctx->progcache_admin, &xid );
}
static int
@@ -2172,6 +2213,9 @@ unprivileged_init( fd_topo_t * topo,
fd_features_enable_one_offs( features, one_off_features, (uint)tile->replay.enable_features_cnt, 0UL );
FD_TEST( fd_funk_join( ctx->funk, fd_topo_obj_laddr( topo, tile->replay.funk_obj_id ) ) );
+ FD_TEST( fd_progcache_admin_join(
+ ctx->progcache_admin,
+ fd_topo_obj_laddr( topo, tile->replay.progcache_obj_id ) ) );
void * _txncache_shmem = fd_topo_obj_laddr( topo, tile->replay.txncache_obj_id );
fd_txncache_shmem_t * txncache_shmem = fd_txncache_shmem_join( _txncache_shmem );
diff --git a/src/flamenco/fd_flamenco_base.h b/src/flamenco/fd_flamenco_base.h
index 1b27f18474e..2af1f054358 100644
--- a/src/flamenco/fd_flamenco_base.h
+++ b/src/flamenco/fd_flamenco_base.h
@@ -9,9 +9,6 @@
#define FD_SLOT_NULL ( ULONG_MAX )
#define FD_SHRED_IDX_NULL ( UINT_MAX )
-#define FD_FUNK_KEY_TYPE_ACC ((uchar)1)
-#define FD_FUNK_KEY_TYPE_ELF_CACHE ((uchar)2)
-
/* CLUSTER_VERSION is the default value for the cluster version
in the epoch context. This value will foll forward to the
latest version.
@@ -96,6 +93,12 @@ typedef struct fd_borrowed_account fd_borrowed_account_t;
struct fd_txn_account;
typedef struct fd_txn_account fd_txn_account_t;
+union fd_features;
+typedef union fd_features fd_features_t;
+
+struct fd_progcache;
+typedef struct fd_progcache fd_progcache_t;
+
struct fd_account_meta {
uchar owner[32];
ulong lamports;
diff --git a/src/flamenco/fd_rwlock.h b/src/flamenco/fd_rwlock.h
index e5caba686bf..7a6eb72a9ed 100644
--- a/src/flamenco/fd_rwlock.h
+++ b/src/flamenco/fd_rwlock.h
@@ -15,6 +15,12 @@ struct fd_rwlock {
typedef struct fd_rwlock fd_rwlock_t;
+static inline fd_rwlock_t *
+fd_rwlock_new( fd_rwlock_t * lock ) {
+ lock->value = 0;
+ return 0;
+}
+
static inline void
fd_rwlock_write( fd_rwlock_t * lock ) {
# if FD_HAS_THREADS
diff --git a/src/flamenco/progcache/Local.mk b/src/flamenco/progcache/Local.mk
new file mode 100644
index 00000000000..e315e9597ad
--- /dev/null
+++ b/src/flamenco/progcache/Local.mk
@@ -0,0 +1,21 @@
+ifdef FD_HAS_INT128
+
+# Public APIs
+
+$(call add-hdrs,fd_prog_load.h)
+$(call add-objs,fd_prog_load,fd_flamenco)
+
+$(call add-hdrs,fd_progcache_admin.h)
+$(call add-objs,fd_progcache_admin,fd_flamenco)
+
+$(call add-hdrs,fd_progcache_user.h)
+$(call add-objs,fd_progcache_user,fd_flamenco)
+
+$(call make-unit-test,test_progcache,test_progcache,fd_flamenco fd_funk fd_ballet fd_util)
+$(call run-unit-test,test_progcache)
+
+# Internals
+
+$(call add-objs,fd_progcache_rec,fd_flamenco)
+
+endif
diff --git a/src/flamenco/progcache/fd_prog_load.c b/src/flamenco/progcache/fd_prog_load.c
new file mode 100644
index 00000000000..4d3a8bfda5b
--- /dev/null
+++ b/src/flamenco/progcache/fd_prog_load.c
@@ -0,0 +1,182 @@
+#include "fd_prog_load.h"
+#include "../runtime/program/fd_bpf_loader_program.h"
+#include "../runtime/program/fd_loader_v4_program.h"
+#include "../runtime/sysvar/fd_sysvar_epoch_schedule.h"
+
+/* Similar to the below function, but gets the executable program content for the v4 loader.
+ Unlike the v3 loader, the programdata is stored in a single program account. The program must
+ NOT be retracted to be added to the cache. Returns a pointer to the programdata on success,
+ and NULL on failure.
+
+ Reasons for failure include:
+ - The program state cannot be read from the account data or is in the `retracted` state. */
+static uchar const *
+fd_get_executable_program_content_for_v4_loader( fd_txn_account_t const * program_acc,
+ ulong * program_data_len ) {
+ int err;
+
+ /* Get the current loader v4 state. This implicitly also checks the dlen. */
+ fd_loader_v4_state_t const * state = fd_loader_v4_get_state( program_acc, &err );
+ if( FD_UNLIKELY( err ) ) {
+ return NULL;
+ }
+
+ /* The program must be deployed or finalized. */
+ if( FD_UNLIKELY( fd_loader_v4_status_is_retracted( state ) ) ) {
+ return NULL;
+ }
+
+ /* This subtraction is safe because get_state() implicitly checks the
+ dlen. */
+ *program_data_len = fd_txn_account_get_data_len( program_acc )-LOADER_V4_PROGRAM_DATA_OFFSET;
+ return fd_txn_account_get_data( program_acc )+LOADER_V4_PROGRAM_DATA_OFFSET;
+}
+
+/* Gets the programdata for a v3 loader-owned account by decoding the account data
+ as well as the programdata account. Returns a pointer to the programdata on success,
+ and NULL on failure.
+
+ Reasons for failure include:
+ - The program account data cannot be decoded or is not in the `program` state.
+ - The programdata account is not large enough to hold at least `PROGRAMDATA_METADATA_SIZE` bytes. */
+static uchar const *
+fd_get_executable_program_content_for_upgradeable_loader( fd_funk_t const * funk,
+ fd_funk_txn_xid_t const * xid,
+ fd_txn_account_t const * program_acc,
+ ulong * program_data_len,
+ fd_funk_txn_xid_t * out_xid ) {
+ fd_bpf_upgradeable_loader_state_t program_account_state[1];
+ if( FD_UNLIKELY( !fd_bincode_decode_static(
+ bpf_upgradeable_loader_state,
+ program_account_state,
+ fd_txn_account_get_data( program_acc ),
+ fd_txn_account_get_data_len( program_acc ),
+ NULL ) ) ) {
+ return NULL;
+ }
+ if( !fd_bpf_upgradeable_loader_state_is_program( program_account_state ) ) {
+ return NULL;
+ }
+
+ fd_pubkey_t * programdata_address = &program_account_state->inner.program.programdata_address;
+
+ fd_account_meta_t const * meta = fd_funk_get_acc_meta_readonly(
+ funk, xid, programdata_address, NULL, NULL, out_xid );
+ if( FD_UNLIKELY( !meta ) ) return NULL;
+ fd_txn_account_t _rec[1];
+ fd_txn_account_t * programdata_acc = fd_txn_account_join( fd_txn_account_new( _rec, programdata_address, (void *)meta, 0 ), funk->wksp );
+ if( FD_UNLIKELY( !programdata_acc ) ) FD_LOG_CRIT(( "fd_txn_account_new failed" ));
+
+ /* We don't actually need to decode here, just make sure that the account
+ can be decoded successfully. */
+ fd_bincode_decode_ctx_t ctx_programdata = {
+ .data = fd_txn_account_get_data( programdata_acc ),
+ .dataend = fd_txn_account_get_data( programdata_acc ) + fd_txn_account_get_data_len( programdata_acc ),
+ };
+
+ ulong total_sz = 0UL;
+ if( FD_UNLIKELY( fd_bpf_upgradeable_loader_state_decode_footprint( &ctx_programdata, &total_sz ) ) ) {
+ return NULL;
+ }
+
+ if( FD_UNLIKELY( fd_txn_account_get_data_len( programdata_acc )wksp );
+ if( FD_UNLIKELY( !rec ) ) FD_LOG_CRIT(( "fd_txn_account_new failed" ));
+
+ /* v1/v2 loaders: Programdata is just the account data.
+ v3 loader: Programdata lives in a separate account. Deserialize the
+ program account and lookup the programdata account.
+ Deserialize the programdata account.
+ v4 loader: Programdata lives in the program account, offset by
+ LOADER_V4_PROGRAM_DATA_OFFSET. */
+ fd_pubkey_t const * owner = fd_txn_account_get_owner( rec );
+ uchar const * elf = NULL;
+ if( !memcmp( owner, fd_solana_bpf_loader_upgradeable_program_id.key, sizeof(fd_pubkey_t) ) ) {
+ /* When a loader v3 program is redeployed, the programdata account
+ is always updated. Therefore, use the programdata account's
+ 'last update XID' instead of the program account's. */
+ elf = fd_get_executable_program_content_for_upgradeable_loader( accdb, xid, rec, out_sz, out_xid );
+ } else if( !memcmp( owner, fd_solana_bpf_loader_v4_program_id.key, sizeof(fd_pubkey_t) ) ) {
+ elf = fd_get_executable_program_content_for_v4_loader( rec, out_sz );
+ } else if( !memcmp( owner, fd_solana_bpf_loader_program_id.key, sizeof(fd_pubkey_t) ) ||
+ !memcmp( owner, fd_solana_bpf_loader_deprecated_program_id.key, sizeof(fd_pubkey_t) ) ) {
+ elf = fd_get_executable_program_content_for_v1_v2_loaders( rec, out_sz );
+ }
+
+ if( FD_LIKELY( !elf ) ) {
+ fd_funk_txn_xid_set_root( out_xid );
+ }
+
+ return elf;
+}
+
+FD_FN_PURE fd_prog_versions_t
+fd_prog_versions( fd_features_t const * features,
+ ulong slot ) {
+ int disable_v0 = FD_FEATURE_ACTIVE( slot, features, disable_sbpf_v0_execution );
+ int reenable_v0 = FD_FEATURE_ACTIVE( slot, features, reenable_sbpf_v0_execution );
+ int enable_v0 = !disable_v0 || reenable_v0;
+ int enable_v1 = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v1_deployment_and_execution );
+ int enable_v2 = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v2_deployment_and_execution );
+ int enable_v3 = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v3_deployment_and_execution );
+
+ fd_prog_versions_t v = {0};
+ v.min_sbpf_version = enable_v0 ? FD_SBPF_V0 : FD_SBPF_V3;
+ if( enable_v3 ) {
+ v.max_sbpf_version = FD_SBPF_V3;
+ } else if( enable_v2 ) {
+ v.max_sbpf_version = FD_SBPF_V2;
+ } else if( enable_v1 ) {
+ v.max_sbpf_version = FD_SBPF_V1;
+ } else {
+ v.max_sbpf_version = FD_SBPF_V0;
+ }
+ return v;
+}
+
+
+fd_prog_load_env_t *
+fd_prog_load_env_from_bank( fd_prog_load_env_t * env,
+ fd_bank_t const * bank ) {
+ *env = (fd_prog_load_env_t) {
+ .features = fd_bank_features_query( bank ),
+ .slot = fd_bank_slot_get ( bank ),
+ .epoch = fd_bank_epoch_get ( bank ),
+ .epoch_slot0 = fd_epoch_slot0( fd_bank_epoch_schedule_query( bank ), fd_bank_epoch_get( bank ) )
+ };
+ return env;
+}
diff --git a/src/flamenco/progcache/fd_prog_load.h b/src/flamenco/progcache/fd_prog_load.h
new file mode 100644
index 00000000000..5e1d5113946
--- /dev/null
+++ b/src/flamenco/progcache/fd_prog_load.h
@@ -0,0 +1,63 @@
+#ifndef HEADER_fd_src_flamenco_fd_progcache_fd_prog_load_h
+#define HEADER_fd_src_flamenco_fd_progcache_fd_prog_load_h
+
+/* fd_prog_load.h provides high-level APIs for loading Solana programs
+ from the account database. */
+
+#include "../../funk/fd_funk_rec.h"
+#include "../fd_flamenco_base.h"
+
+FD_PROTOTYPES_BEGIN
+
+/* fd_prog_load_elf loads a reference to program data from a funk-backed
+ account database. *prog_addr gives the account address of the
+ program account (NOT the program data account). Returns a pointer to
+ the first byte of the ELF binary on success. *out_sz is set to the
+ size of the program data account. *out_xid is set to the the txn
+ XID the program data account was written to. On failure, returns
+ NULL. Reasons for failure include: program account not found,
+ program not deployed, program data account not found, etc */
+
+uchar const *
+fd_prog_load_elf( fd_funk_t const * accdb,
+ fd_funk_txn_xid_t const * xid,
+ void const * prog_addr,
+ ulong * out_sz,
+ fd_funk_txn_xid_t * out_xid );
+/* FIXME provide an API to detect data race */
+/* FIXME clarify edge case where program account and program data
+ account were modified in different funk txns */
+
+/* fd_prog_versions derives sBPF versions from the current feature set. */
+
+struct fd_prog_versions {
+ uint min_sbpf_version;
+ uint max_sbpf_version;
+};
+typedef struct fd_prog_versions fd_prog_versions_t;
+
+FD_FN_PURE fd_prog_versions_t
+fd_prog_versions( fd_features_t const * features,
+ ulong slot );
+
+FD_PROTOTYPES_END
+
+struct fd_prog_load_env {
+ fd_features_t const * features;
+
+ ulong slot; /* current slot */
+ ulong epoch; /* current epoch */
+ ulong epoch_slot0; /* slot0 of current epoch */
+};
+
+typedef struct fd_prog_load_env fd_prog_load_env_t;
+
+FD_PROTOTYPES_BEGIN
+
+fd_prog_load_env_t *
+fd_prog_load_env_from_bank( fd_prog_load_env_t * env,
+ fd_bank_t const * bank );
+
+FD_PROTOTYPES_END
+
+#endif /* HEADER_fd_src_flamenco_fd_progcache_fd_prog_load_h */
diff --git a/src/flamenco/progcache/fd_progcache_admin.c b/src/flamenco/progcache/fd_progcache_admin.c
new file mode 100644
index 00000000000..87df8c25f1f
--- /dev/null
+++ b/src/flamenco/progcache/fd_progcache_admin.c
@@ -0,0 +1,405 @@
+#include "fd_progcache_admin.h"
+
+/* Algorithm to estimate size of cache metadata structures (rec_pool
+ object pool and rec_map hashchain table).
+
+ FIXME Carefully balance this */
+
+static ulong
+fd_progcache_est_rec_max1( ulong wksp_footprint,
+ ulong mean_cache_entry_size ) {
+ return wksp_footprint / mean_cache_entry_size;
+}
+
+ulong
+fd_progcache_est_rec_max( ulong wksp_footprint,
+ ulong mean_cache_entry_size ) {
+ ulong est = fd_progcache_est_rec_max1( wksp_footprint, mean_cache_entry_size );
+ if( FD_UNLIKELY( est>(1UL<<31) ) ) FD_LOG_ERR(( "fd_progcache_est_rec_max(wksp_footprint=%lu,mean_cache_entry_size=%lu) failed: invalid parameters", wksp_footprint, mean_cache_entry_size ));
+ return fd_ulong_max( est, 2048UL );
+}
+
+fd_progcache_admin_t *
+fd_progcache_admin_join( fd_progcache_admin_t * ljoin,
+ void * shfunk ) {
+ if( FD_UNLIKELY( !ljoin ) ) {
+ FD_LOG_WARNING(( "NULL ljoin" ));
+ return NULL;
+ }
+ if( FD_UNLIKELY( !shfunk ) ) {
+ FD_LOG_WARNING(( "NULL shfunk" ));
+ return NULL;
+ }
+
+ memset( ljoin, 0, sizeof(fd_progcache_admin_t) );
+ if( FD_UNLIKELY( !fd_funk_join( ljoin->funk, shfunk ) ) ) {
+ FD_LOG_CRIT(( "fd_funk_join failed" ));
+ }
+
+ return ljoin;
+}
+
+void *
+fd_progcache_admin_leave( fd_progcache_admin_t * ljoin,
+ void ** opt_shfunk ) {
+ if( FD_UNLIKELY( !ljoin ) ) FD_LOG_CRIT(( "NULL ljoin" ));
+
+ if( FD_UNLIKELY( !fd_funk_leave( ljoin->funk, opt_shfunk ) ) ) FD_LOG_CRIT(( "fd_funk_leave failed" ));
+
+ return ljoin;
+}
+
+/* Begin transaction-level operations. It is assumed that funk_txn data
+ structures are not concurrently modified. This includes txn_pool and
+ txn_map. */
+
+void
+fd_progcache_txn_prepare( fd_progcache_admin_t * cache,
+ fd_funk_txn_xid_t const * xid_parent,
+ fd_funk_txn_xid_t const * xid_new ) {
+ fd_funk_txn_prepare( cache->funk, xid_parent, xid_new );
+}
+
+static void
+fd_progcache_txn_cancel_one( fd_progcache_admin_t * cache,
+ fd_funk_txn_t * txn ) {
+ FD_LOG_DEBUG(( "progcache txn laddr=%p xid %lu:%lu: cancel", (void *)txn, txn->xid.ul[0], txn->xid.ul[1] ));
+
+ fd_funk_t * funk = cache->funk;
+ if( FD_UNLIKELY( !fd_funk_txn_idx_is_null( txn->child_head_cidx ) ||
+ !fd_funk_txn_idx_is_null( txn->child_tail_cidx ) ) ) {
+ FD_LOG_CRIT(( "fd_progcache_txn_cancel failed: txn at %p with xid %lu:%lu has children (data corruption?)",
+ (void *)txn, txn->xid.ul[0], txn->xid.ul[1] ));
+ }
+
+ /* Phase 1: Drain users from transaction */
+
+ fd_rwlock_write( txn->lock );
+ FD_VOLATILE( txn->state ) = FD_FUNK_TXN_STATE_CANCEL;
+
+ /* Phase 2: Remove records */
+
+ while( !fd_funk_rec_idx_is_null( txn->rec_head_idx ) ) {
+ fd_funk_rec_t * rec = &funk->rec_pool->ele[ txn->rec_head_idx ];
+ uint next_idx = rec->next_idx;
+ rec->next_idx = FD_FUNK_REC_IDX_NULL;
+ if( FD_LIKELY( !fd_funk_rec_idx_is_null( next_idx ) ) ) {
+ funk->rec_pool->ele[ next_idx ].prev_idx = FD_FUNK_REC_IDX_NULL;
+ }
+
+ fd_funk_val_flush( rec, funk->alloc, funk->wksp );
+
+ fd_funk_rec_query_t query[1];
+ int remove_err = fd_funk_rec_map_remove( funk->rec_map, &rec->pair, NULL, query, FD_MAP_FLAG_BLOCKING );
+ if( FD_UNLIKELY( remove_err ) ) FD_LOG_CRIT(( "fd_funk_rec_map_remove failed: %i-%s", remove_err, fd_map_strerror( remove_err ) ));
+
+ fd_funk_rec_pool_release( funk->rec_pool, rec, 1 );
+
+ txn->rec_head_idx = next_idx;
+ if( fd_funk_rec_idx_is_null( next_idx ) ) txn->rec_tail_idx = FD_FUNK_REC_IDX_NULL;
+ }
+
+ /* Phase 3: Remove transaction from fork graph */
+
+ uint self_cidx = fd_funk_txn_cidx( (ulong)( txn-funk->txn_pool->ele ) );
+ uint prev_cidx = txn->sibling_prev_cidx; ulong prev_idx = fd_funk_txn_idx( prev_cidx );
+ uint next_cidx = txn->sibling_next_cidx; ulong next_idx = fd_funk_txn_idx( next_cidx );
+ if( !fd_funk_txn_idx_is_null( next_idx ) ) {
+ funk->txn_pool->ele[ next_idx ].sibling_prev_cidx = prev_cidx;
+ }
+ if( !fd_funk_txn_idx_is_null( prev_idx ) ) {
+ funk->txn_pool->ele[ prev_idx ].sibling_next_cidx = next_cidx;
+ }
+ if( !fd_funk_txn_idx_is_null( fd_funk_txn_idx( txn->parent_cidx ) ) ) {
+ fd_funk_txn_t * parent = &funk->txn_pool->ele[ fd_funk_txn_idx( txn->parent_cidx ) ];
+ if( parent->child_head_cidx==self_cidx ) parent->child_head_cidx = next_cidx;
+ if( parent->child_tail_cidx==self_cidx ) parent->child_tail_cidx = prev_cidx;
+ } else {
+ if( funk->shmem->child_head_cidx==self_cidx ) funk->shmem->child_head_cidx = next_cidx;
+ if( funk->shmem->child_tail_cidx==self_cidx ) funk->shmem->child_tail_cidx = prev_cidx;
+ }
+
+ /* Phase 4: Remove transcation from index */
+
+ fd_funk_txn_map_query_t query[1];
+ int remove_err = fd_funk_txn_map_remove( funk->txn_map, &txn->xid, NULL, query, FD_MAP_FLAG_BLOCKING );
+ if( FD_UNLIKELY( remove_err!=FD_MAP_SUCCESS ) ) {
+ FD_LOG_CRIT(( "fd_progcache_txn_cancel failed: fd_funk_txn_map_remove(%lu:%lu) failed: %i-%s",
+ txn->xid.ul[0], txn->xid.ul[1], remove_err, fd_map_strerror( remove_err ) ));
+ }
+
+ /* Phase 5: Free transaction object */
+
+ fd_rwlock_unwrite( txn->lock );
+ FD_VOLATILE( txn->state ) = FD_FUNK_TXN_STATE_FREE;
+ fd_funk_txn_pool_release( funk->txn_pool, txn, 1 );
+}
+
+/* Cancels txn and all children */
+
+static void
+fd_progcache_txn_cancel_tree( fd_progcache_admin_t * cache,
+ fd_funk_txn_t * txn ) {
+ for(;;) {
+ ulong child_idx = fd_funk_txn_idx( txn->child_head_cidx );
+ if( fd_funk_txn_idx_is_null( child_idx ) ) break;
+ fd_funk_txn_t * child = &cache->funk->txn_pool->ele[ child_idx ];
+ fd_progcache_txn_cancel_tree( cache, child );
+ }
+ fd_progcache_txn_cancel_one( cache, txn );
+}
+
+/* Cancels all left/right siblings */
+
+static void
+fd_progcache_txn_cancel_prev_list( fd_progcache_admin_t * cache,
+ fd_funk_txn_t * txn ) {
+ ulong self_idx = (ulong)( txn - cache->funk->txn_pool->ele );
+ for(;;) {
+ ulong prev_idx = fd_funk_txn_idx( txn->sibling_prev_cidx );
+ if( FD_UNLIKELY( prev_idx==self_idx ) ) FD_LOG_CRIT(( "detected cycle in fork graph" ));
+ if( fd_funk_txn_idx_is_null( prev_idx ) ) break;
+ fd_funk_txn_t * sibling = &cache->funk->txn_pool->ele[ prev_idx ];
+ fd_progcache_txn_cancel_tree( cache, sibling );
+ }
+}
+
+static void
+fd_progcache_txn_cancel_next_list( fd_progcache_admin_t * cache,
+ fd_funk_txn_t * txn ) {
+ ulong self_idx = (ulong)( txn - cache->funk->txn_pool->ele );
+ for(;;) {
+ ulong next_idx = fd_funk_txn_idx( txn->sibling_next_cidx );
+ if( FD_UNLIKELY( next_idx==self_idx ) ) FD_LOG_CRIT(( "detected cycle in fork graph" ));
+ if( fd_funk_txn_idx_is_null( next_idx ) ) break;
+ fd_funk_txn_t * sibling = &cache->funk->txn_pool->ele[ next_idx ];
+ fd_progcache_txn_cancel_tree( cache, sibling );
+ }
+}
+
+void
+fd_progcache_txn_cancel( fd_progcache_admin_t * cache,
+ fd_funk_txn_xid_t const * xid ) {
+ fd_funk_t * funk = cache->funk;
+
+ fd_funk_txn_t * txn = fd_funk_txn_query( xid, funk->txn_map );
+ if( FD_UNLIKELY( !txn ) ) {
+ FD_LOG_CRIT(( "fd_progcache_txn_cancel failed: txn with xid %lu:%lu not found", xid->ul[0], xid->ul[1] ));
+ }
+
+ fd_progcache_txn_cancel_next_list( cache, txn );
+ fd_progcache_txn_cancel_tree( cache, txn );
+}
+
+/* fd_progcache_publish_recs publishes all of a progcache's records.
+ It is assumed at this point that the txn has no more concurrent
+ users. */
+
+static void
+fd_progcache_publish_recs( fd_progcache_admin_t * cache,
+ fd_funk_txn_t * txn ) {
+ fd_funk_txn_xid_t const root = { .ul = { ULONG_MAX, ULONG_MAX } };
+ /* Iterate record list */
+ uint head = txn->rec_head_idx;
+ txn->rec_head_idx = FD_FUNK_REC_IDX_NULL;
+ txn->rec_tail_idx = FD_FUNK_REC_IDX_NULL;
+ while( !fd_funk_rec_idx_is_null( head ) ) {
+ fd_funk_rec_t * rec = &cache->funk->rec_pool->ele[ head ];
+ uint next = rec->next_idx;
+ rec->prev_idx = FD_FUNK_REC_IDX_NULL;
+ rec->next_idx = FD_FUNK_REC_IDX_NULL;
+ fd_funk_txn_xid_st_atomic( rec->pair.xid, &root );
+ head = next;
+ }
+}
+
+/* fd_progcache_txn_publish_one merges an in-prep transaction whose
+ parent is the last published, into the parent. */
+
+static void
+fd_progcache_txn_publish_one( fd_progcache_admin_t * cache,
+ fd_funk_txn_xid_t const * xid ) {
+ fd_funk_t * funk = cache->funk;
+
+ /* Phase 1: Mark transaction as "last published" */
+
+ fd_funk_txn_t * txn = fd_funk_txn_query( xid, funk->txn_map );
+ if( FD_UNLIKELY( !txn ) ) {
+ FD_LOG_CRIT(( "fd_progcache_publish failed: txn with xid %lu:%lu not found", xid->ul[0], xid->ul[1] ));
+ }
+ FD_LOG_DEBUG(( "progcache txn laddr=%p xid %lu:%lu: publish", (void *)txn, txn->xid.ul[0], txn->xid.ul[1] ));
+ if( FD_UNLIKELY( !fd_funk_txn_idx_is_null( fd_funk_txn_idx( txn->parent_cidx ) ) ) ) {
+ FD_LOG_CRIT(( "fd_progcache_publish failed: txn with xid %lu:%lu is not a child of the last published txn", xid->ul[0], xid->ul[1] ));
+ }
+ fd_funk_txn_xid_st_atomic( funk->shmem->last_publish, xid );
+
+ /* Phase 2: Drain users from transaction */
+
+ fd_rwlock_write( txn->lock );
+ FD_VOLATILE( txn->state ) = FD_FUNK_TXN_STATE_PUBLISH;
+
+ /* Phase 3: Migrate records */
+
+ fd_progcache_publish_recs( cache, txn );
+
+ /* Phase 4: Remove transaction from fork graph
+
+ Because the transaction has no more records, removing it from the
+ fork graph has no visible side effects to concurrent query ops
+ (always return "no found") or insert ops (refuse to write to a
+ "publish" state txn). */
+
+ { /* Adjust the parent pointers of the children to point to "last published" */
+ ulong child_idx = fd_funk_txn_idx( txn->child_head_cidx );
+ while( FD_UNLIKELY( !fd_funk_txn_idx_is_null( child_idx ) ) ) {
+ funk->txn_pool->ele[ child_idx ].parent_cidx = fd_funk_txn_cidx( FD_FUNK_TXN_IDX_NULL );
+ child_idx = fd_funk_txn_idx( funk->txn_pool->ele[ child_idx ].sibling_next_cidx );
+ }
+ }
+
+ /* Phase 5: Remove transaction from index
+
+ The transaction is now an orphan and won't get any new records. */
+
+ fd_funk_txn_map_query_t query[1];
+ int remove_err = fd_funk_txn_map_remove( funk->txn_map, xid, NULL, query, 0 );
+ if( FD_UNLIKELY( remove_err!=FD_MAP_SUCCESS ) ) {
+ FD_LOG_CRIT(( "fd_progcache_publish failed: fd_funk_txn_map_remove failed: %i-%s", remove_err, fd_map_strerror( remove_err ) ));
+ }
+
+ /* Phase 6: Free transaction object */
+
+ fd_rwlock_unwrite( txn->lock );
+ FD_VOLATILE( txn->state ) = FD_FUNK_TXN_STATE_FREE;
+ fd_funk_txn_pool_release( funk->txn_pool, txn, 1 );
+}
+
+static void
+fd_progcache_txn_publish_parents( fd_progcache_admin_t * cache,
+ fd_funk_txn_t * txn ) {
+ /* Recurse until all of txn's parents are published */
+ fd_funk_t * funk = cache->funk;
+ if( !fd_funk_txn_idx_is_null( fd_funk_txn_idx( txn->parent_cidx ) ) ) {
+ ulong parent_idx = fd_funk_txn_idx( txn->parent_cidx );
+ fd_funk_txn_t * parent = &funk->txn_pool->ele[ parent_idx ];
+ if( FD_UNLIKELY( FD_VOLATILE_CONST( parent->state )!=FD_FUNK_TXN_STATE_PUBLISH ) ) {
+ fd_progcache_txn_publish_parents( cache, parent );
+ }
+ } else {
+ /* Root transaction */
+ ulong idx = (uint)( txn - funk->txn_pool->ele );
+ if( fd_funk_txn_idx( funk->shmem->child_head_cidx )==idx ) {
+ funk->shmem->child_head_cidx = txn->sibling_next_cidx;
+ }
+ if( fd_funk_txn_idx( funk->shmem->child_tail_cidx )==idx ) {
+ funk->shmem->child_tail_cidx = txn->sibling_prev_cidx;
+ }
+ }
+ fd_progcache_txn_publish_one( cache, fd_funk_txn_xid( txn ) );
+}
+
+void
+fd_progcache_txn_publish( fd_progcache_admin_t * cache,
+ fd_funk_txn_xid_t const * xid ) {
+ fd_funk_t * funk = cache->funk;
+
+ fd_funk_txn_t * txn = fd_funk_txn_query( xid, funk->txn_map );
+ if( FD_UNLIKELY( !txn ) ) {
+ FD_LOG_CRIT(( "fd_progcache_publish failed: txn with xid %lu:%lu not found", xid->ul[0], xid->ul[1] ));
+ }
+
+ fd_progcache_txn_cancel_prev_list( cache, txn );
+ fd_progcache_txn_cancel_next_list( cache, txn );
+ { /* Cancel left siblings */
+ ulong child_idx = fd_funk_txn_idx( txn->sibling_prev_cidx );
+ while( FD_UNLIKELY( !fd_funk_txn_idx_is_null( child_idx ) ) ) {
+ funk->txn_pool->ele[ child_idx ].parent_cidx = fd_funk_txn_cidx( FD_FUNK_TXN_IDX_NULL );
+ child_idx = fd_funk_txn_idx( funk->txn_pool->ele[ child_idx ].sibling_next_cidx );
+ }
+ }
+ { /* Cancel right siblings */
+ ulong child_idx = fd_funk_txn_idx( txn->sibling_next_cidx );
+ while( FD_UNLIKELY( !fd_funk_txn_idx_is_null( child_idx ) ) ) {
+ funk->txn_pool->ele[ child_idx ].parent_cidx = fd_funk_txn_cidx( FD_FUNK_TXN_IDX_NULL );
+ child_idx = fd_funk_txn_idx( funk->txn_pool->ele[ child_idx ].sibling_next_cidx );
+ }
+ }
+
+ fd_progcache_txn_publish_parents( cache, txn );
+ txn->child_head_cidx = UINT_MAX;
+ txn->child_tail_cidx = UINT_MAX;
+ txn->parent_cidx = UINT_MAX;
+ txn->sibling_prev_cidx = UINT_MAX;
+ txn->sibling_next_cidx = UINT_MAX;
+}
+
+/* reset_txn_list does a depth-first traversal of the txn tree.
+ Detaches all recs from txns by emptying rec linked lists. */
+
+static void
+reset_txn_list( fd_funk_t * funk,
+ ulong txn_head_idx ) {
+ fd_funk_txn_pool_t * txn_pool = funk->txn_pool;
+ for( ulong idx = txn_head_idx;
+ !fd_funk_txn_idx_is_null( idx );
+ ) {
+ fd_funk_txn_t * txn = &txn_pool->ele[ idx ];
+ fd_funk_txn_state_assert( txn, FD_FUNK_TXN_STATE_ACTIVE );
+ txn->rec_head_idx = FD_FUNK_REC_IDX_NULL;
+ txn->rec_tail_idx = FD_FUNK_REC_IDX_NULL;
+ reset_txn_list( funk, txn->child_head_cidx );
+ idx = fd_funk_txn_idx( txn->sibling_next_cidx );
+ }
+}
+
+/* reset_rec_map frees all records in a funk instance. */
+
+static void
+reset_rec_map( fd_funk_t * funk ) {
+ fd_wksp_t * wksp = funk->wksp;
+ fd_alloc_t * alloc = funk->alloc;
+ fd_funk_rec_map_t * rec_map = funk->rec_map;
+ fd_funk_rec_pool_t * rec_pool = funk->rec_pool;
+
+ ulong chain_cnt = fd_funk_rec_map_chain_cnt( rec_map );
+ for( ulong chain_idx=0UL; chain_idxmap_next );;
+
+ /* Remove rec object from map */
+ fd_funk_rec_map_query_t rec_query[1];
+ int err = fd_funk_rec_map_remove( rec_map, fd_funk_rec_pair( rec ), NULL, rec_query, FD_MAP_FLAG_BLOCKING );
+ fd_funk_rec_key_t key; fd_funk_rec_key_copy( &key, rec->pair.key );
+ if( FD_UNLIKELY( err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_funk_rec_map_remove failed (%i-%s)", err, fd_map_strerror( err ) ));
+
+ /* Free rec resources */
+ fd_funk_val_flush( rec, alloc, wksp );
+ fd_funk_rec_pool_release( rec_pool, rec, 1 );
+ iter.ele_idx = next;
+ }
+ }
+}
+
+void
+fd_progcache_reset( fd_progcache_admin_t * cache ) {
+ fd_funk_t * funk = cache->funk;
+ reset_txn_list( funk, fd_funk_txn_idx( funk->shmem->child_head_cidx ) );
+ reset_rec_map( funk );
+}
+
+void
+fd_progcache_clear( fd_progcache_admin_t * cache ) {
+ /* FIXME this descends the progcache txn tree multiple times */
+ fd_progcache_reset( cache );
+ fd_funk_txn_cancel_all( cache->funk );
+}
+
+void
+fd_progcache_verify( fd_progcache_admin_t * cache,
+ fd_progcache_verify_stat_t * out_stat ) {
+ (void)cache; (void)out_stat;
+}
diff --git a/src/flamenco/progcache/fd_progcache_admin.h b/src/flamenco/progcache/fd_progcache_admin.h
new file mode 100644
index 00000000000..0b896570098
--- /dev/null
+++ b/src/flamenco/progcache/fd_progcache_admin.h
@@ -0,0 +1,88 @@
+#ifndef HEADER_fd_src_flamenco_fd_progcache_admin_h
+#define HEADER_fd_src_flamenco_fd_progcache_admin_h
+
+#include "../../funk/fd_funk.h"
+
+struct fd_progcache_admin {
+ fd_funk_t funk[1];
+};
+
+typedef struct fd_progcache_admin fd_progcache_admin_t;
+
+FD_PROTOTYPES_BEGIN
+
+/* Constructors *******************************************************/
+
+/* fd_progcache_est_rec_max estimates the fd_funk rec_max parameter
+ given the cache's wksp footprint and mean cache entry heap
+ utilization. */
+
+ulong
+fd_progcache_est_rec_max( ulong wksp_footprint,
+ ulong mean_cache_entry_size );
+
+/* fd_progcache_join joins the caller to a program cache funk instance. */
+
+fd_progcache_admin_t *
+fd_progcache_admin_join( fd_progcache_admin_t * ljoin,
+ void * shfunk );
+
+/* fd_progcache_admin_leave detaches the caller from a program cache. */
+
+void *
+fd_progcache_admin_leave( fd_progcache_admin_t * cache,
+ void ** opt_shfunk );
+
+/* Transaction-level operations ***************************************/
+
+void
+fd_progcache_txn_prepare( fd_progcache_admin_t * cache,
+ fd_funk_txn_xid_t const * xid_parent,
+ fd_funk_txn_xid_t const * xid_new );
+
+void
+fd_progcache_txn_cancel( fd_progcache_admin_t * cache,
+ fd_funk_txn_xid_t const * xid );
+
+void
+fd_progcache_txn_publish( fd_progcache_admin_t * cache,
+ fd_funk_txn_xid_t const * xid );
+
+/* Reset operations ***************************************************/
+
+/* fd_progcache_flush removes all cache entries while leaving the txn
+ graph intact. Does not support concurrent usage. */
+
+void
+fd_progcache_reset( fd_progcache_admin_t * cache );
+
+/* fd_progcache_clear removes all cache entries and destroys the txn
+ graph. Does not support concurrent usage. */
+
+void
+fd_progcache_clear( fd_progcache_admin_t * cache );
+
+FD_PROTOTYPES_END
+
+/* Verify operations **************************************************/
+
+struct fd_progcache_verify_stat {
+ ulong txn_cnt;
+ ulong rec_cnt;
+};
+
+typedef struct fd_progcache_verify_stat fd_progcache_verify_stat_t;
+
+FD_PROTOTYPES_BEGIN
+
+/* fd_progcache_verify does various expensive data structure integrity
+ checks. Assumes no concurrent users of progcache. Collects stats
+ along the way. */
+
+void
+fd_progcache_verify( fd_progcache_admin_t * cache,
+ fd_progcache_verify_stat_t * out_stat );
+
+FD_PROTOTYPES_END
+
+#endif /* HEADER_fd_src_flamenco_fd_progcache_admin_h */
diff --git a/src/flamenco/progcache/fd_progcache_rec.c b/src/flamenco/progcache/fd_progcache_rec.c
new file mode 100644
index 00000000000..77ec6a96e24
--- /dev/null
+++ b/src/flamenco/progcache/fd_progcache_rec.c
@@ -0,0 +1,109 @@
+#include "fd_progcache_rec.h"
+#include "../vm/fd_vm.h" /* fd_vm_syscall_register_slot, fd_vm_validate */
+
+fd_progcache_rec_t *
+fd_progcache_rec_new( void * mem,
+ fd_sbpf_elf_info_t const * elf_info,
+ fd_sbpf_loader_config_t const * config,
+ ulong load_slot,
+ fd_features_t const * features,
+ void const * progdata,
+ ulong progdata_sz,
+ void * scratch,
+ ulong scratch_sz ) {
+
+ /* Format object */
+
+ int has_calldests = !fd_sbpf_enable_stricter_elf_headers_enabled( elf_info->sbpf_version );
+
+ FD_SCRATCH_ALLOC_INIT( l, mem );
+ fd_progcache_rec_t * rec = FD_SCRATCH_ALLOC_APPEND( l, fd_progcache_rec_align(), sizeof(fd_progcache_rec_t) );
+ void * calldests_mem = NULL;
+ if( has_calldests ) {
+ /* */calldests_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( elf_info->text_cnt ) );
+ }
+ void * rodata_mem = FD_SCRATCH_ALLOC_APPEND( l, 8UL, elf_info->bin_sz );
+ FD_SCRATCH_ALLOC_FINI( l, fd_progcache_rec_align() );
+ memset( rec, 0, sizeof(fd_progcache_rec_t) );
+ rec->calldests_off = has_calldests ? (uint)( (ulong)calldests_mem - (ulong)mem ) : 0U;
+ rec->rodata_off = (uint)( (ulong)rodata_mem - (ulong)mem );
+
+ rec->text_cnt = elf_info->text_cnt;
+ rec->text_off = elf_info->text_off;
+ rec->text_sz = (uint)elf_info->text_sz;
+ rec->sbpf_version = (uchar)elf_info->sbpf_version;
+
+ /* Set up sbpf_loader (redirect writes into progcache_rec object) */
+
+ fd_sbpf_program_t prog[1] = {{
+ .info = *elf_info,
+ .rodata = rodata_mem,
+ .text = (ulong *)((ulong)rodata_mem + elf_info->text_off), /* FIXME: WHAT IF MISALIGNED */
+ .entry_pc = ULONG_MAX
+ }};
+ if( has_calldests && elf_info->text_cnt>0UL ) {
+ prog->calldests_shmem = calldests_mem;
+ prog->calldests = fd_sbpf_calldests_join( fd_sbpf_calldests_new( calldests_mem, elf_info->text_cnt ) );
+ }
+
+ /* Loader requires syscall table */
+
+ fd_sbpf_syscalls_t _syscalls[ FD_SBPF_SYSCALLS_SLOT_CNT ];
+ fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_join( fd_sbpf_syscalls_new( _syscalls ) );
+ int syscalls_err = fd_vm_syscall_register_slot( syscalls, load_slot, features, /* is_deploy */ 0 );
+ if( FD_UNLIKELY( syscalls_err!=FD_VM_SUCCESS ) ) FD_LOG_CRIT(( "fd_vm_syscall_register_slot failed" ));
+
+ /* Run ELF loader */
+
+ if( FD_UNLIKELY( 0!=fd_sbpf_program_load( prog, progdata, progdata_sz, syscalls, config, scratch, scratch_sz ) ) ) {
+ return NULL;
+ }
+
+ rec->entry_pc = (uint)prog->entry_pc;
+ rec->rodata_sz = (uint)prog->rodata_sz;
+
+ /* Run bytecode validator */
+
+ fd_vm_t _vm[1];
+ fd_vm_t * vm = fd_vm_join( fd_vm_new( _vm ) );
+ if( FD_UNLIKELY( !vm ) ) FD_LOG_CRIT(( "fd_vm_new failed" ));
+ vm = fd_vm_init( vm,
+ NULL, /* OK since unused in `fd_vm_validate()` */
+ 0UL,
+ 0UL,
+ prog->rodata,
+ prog->rodata_sz,
+ prog->text,
+ prog->info.text_cnt,
+ prog->info.text_off,
+ prog->info.text_sz,
+ prog->entry_pc,
+ prog->calldests,
+ elf_info->sbpf_version,
+ syscalls,
+ NULL,
+ NULL,
+ NULL,
+ 0U,
+ NULL,
+ 0,
+ FD_FEATURE_ACTIVE( load_slot, features, bpf_account_data_direct_mapping ),
+ 0 );
+ if( FD_UNLIKELY( !vm ) ) FD_LOG_CRIT(( "fd_vm_init failed" ));
+
+ if( FD_UNLIKELY( fd_vm_validate( vm )!=FD_VM_SUCCESS ) ) return NULL;
+
+ rec->slot = load_slot;
+ rec->executable = 1;
+ return rec;
+}
+
+fd_progcache_rec_t *
+fd_progcache_rec_new_nx( void * mem,
+ ulong load_slot ) {
+ fd_progcache_rec_t * rec = mem;
+ memset( rec, 0, sizeof(fd_progcache_rec_t) );
+ rec->slot = load_slot;
+ rec->executable = 0;
+ return rec;
+}
diff --git a/src/flamenco/progcache/fd_progcache_rec.h b/src/flamenco/progcache/fd_progcache_rec.h
new file mode 100644
index 00000000000..8c0c1ac39da
--- /dev/null
+++ b/src/flamenco/progcache/fd_progcache_rec.h
@@ -0,0 +1,113 @@
+#ifndef HEADER_fd_src_flamenco_progcache_fd_progcache_rec_h
+#define HEADER_fd_src_flamenco_progcache_fd_progcache_rec_h
+
+#include "../fd_flamenco_base.h"
+#include "../../ballet/sbpf/fd_sbpf_loader.h"
+
+/* fd_progcache_rec_t is the fixed size header of a program cache entry
+ object. Entries are either non-executable (e.g. programs that failed
+ verification) or executable. Non-executable entry objects consist
+ only of this header struct. Executable entry objects are variable-
+ sized and contain additional structures past this header (rodata/ROM
+ segment, control flow metadata, ...). */
+
+struct fd_progcache_rec {
+ /* Slot number at which this cache entry was created.
+ Matches the XID's slot number for in-preparation transactions. */
+ ulong slot;
+
+ uint entry_pc;
+ uint text_cnt;
+ uint text_off;
+ uint text_sz;
+
+ uint rodata_sz;
+
+ uint calldests_off; /* offset to sbpf_calldests map */
+ uint rodata_off; /* offset to rodata segment */
+
+ /* SBPF version, SIMD-0161 */
+ uchar sbpf_version;
+
+ uint executable : 1; /* is this an executable entry? */
+ uint invalidate : 1; /* if ==1, limits visibility of this entry to this slot */
+};
+
+typedef struct fd_progcache_rec fd_progcache_rec_t;
+
+FD_PROTOTYPES_BEGIN
+
+/* Accessors */
+
+static inline uchar const *
+fd_progcache_rec_rodata( fd_progcache_rec_t const * rec ) {
+ return (uchar const *)rec + rec->rodata_off;
+}
+
+static inline fd_sbpf_calldests_t const *
+fd_progcache_rec_calldests( fd_progcache_rec_t const * rec ) {
+ return fd_sbpf_calldests_join( (void *)( (ulong)rec + rec->calldests_off ) );
+}
+
+/* Private APIs */
+
+/* fd_progcache_rec_{align,footprint} give the params of backing memory
+ of a progcache_rec object for the given ELF info. If elf_info is
+ NULL, implies a non-executable cache entry (sizeof(fd_progcache_rec_t)). */
+
+FD_FN_CONST static inline ulong
+fd_progcache_rec_align( void ) {
+ return alignof(fd_progcache_rec_t);
+}
+
+FD_FN_PURE FD_FN_UNUSED static ulong
+fd_progcache_rec_footprint( fd_sbpf_elf_info_t const * elf_info ) {
+ if( !elf_info ) return sizeof(fd_progcache_rec_t); /* non-executable */
+
+ int has_calldests = !fd_sbpf_enable_stricter_elf_headers_enabled( elf_info->sbpf_version );
+ ulong pc_max = fd_ulong_max( 1UL, elf_info->text_cnt );
+
+ ulong l = FD_LAYOUT_INIT;
+ l = FD_LAYOUT_APPEND( l, alignof(fd_progcache_rec_t), sizeof(fd_progcache_rec_t) );
+ if( has_calldests ) {
+ l = FD_LAYOUT_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( pc_max ) );
+ }
+ l = FD_LAYOUT_APPEND( l, 8UL, elf_info->bin_sz );
+ return FD_LAYOUT_FINI( l, fd_progcache_rec_align() );
+}
+
+/* fd_progcache_rec_new creates a new excutable progcache_rec object.
+ mem points to a memory region matching fd_progcache_rec_{align,
+ footprint}. Loads and verifies the given program data and returns
+ the newly created executable object on success. On failure, returns
+ NULL (the caller may call fd_progcache_rec_new_nx instead). */
+
+fd_progcache_rec_t *
+fd_progcache_rec_new( void * mem,
+ fd_sbpf_elf_info_t const * elf_info,
+ fd_sbpf_loader_config_t const * config,
+ ulong load_slot,
+ fd_features_t const * features,
+ void const * progdata,
+ ulong progdata_sz,
+ void * scratch,
+ ulong scratch_sz );
+
+/* fd_progcache_rec_new_nx creates a non-executable program_cache
+ object. fd_progcache_rec_t[1] is suitable for mem. */
+
+fd_progcache_rec_t *
+fd_progcache_rec_new_nx( void * mem,
+ ulong load_slot );
+
+/* fd_progcache_rec_delete destroys a progcache_rec object and returns
+ the backing memory region to the caller. */
+
+static inline void *
+fd_progcache_rec_delete( fd_progcache_rec_t * rec ) {
+ return rec;
+}
+
+FD_PROTOTYPES_END
+
+#endif /* HEADER_fd_src_flamenco_progcache_fd_progcache_rec_h */
diff --git a/src/flamenco/progcache/fd_progcache_user.c b/src/flamenco/progcache/fd_progcache_user.c
new file mode 100644
index 00000000000..e15ad80477b
--- /dev/null
+++ b/src/flamenco/progcache/fd_progcache_user.c
@@ -0,0 +1,696 @@
+#include "fd_prog_load.h"
+#include "fd_progcache_user.h"
+#include "fd_progcache_rec.h"
+
+FD_TL fd_progcache_metrics_t fd_progcache_metrics_default;
+
+fd_progcache_t *
+fd_progcache_join( fd_progcache_t * ljoin,
+ void * shfunk,
+ uchar * scratch,
+ ulong scratch_sz ) {
+ if( FD_UNLIKELY( !ljoin ) ) {
+ FD_LOG_WARNING(( "NULL ljoin" ));
+ return NULL;
+ }
+ if( FD_UNLIKELY( !shfunk ) ) {
+ FD_LOG_WARNING(( "NULL shfunk" ));
+ return NULL;
+ }
+ if( FD_LIKELY( scratch_sz ) ) {
+ if( FD_UNLIKELY( !scratch ) ) {
+ FD_LOG_WARNING(( "NULL scratch" ));
+ return NULL;
+ }
+ if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)scratch, FD_PROGCACHE_SCRATCH_ALIGN ) ) ) {
+ FD_LOG_WARNING(( "misaligned scratch" ));
+ return NULL;
+ }
+ }
+ memset( ljoin, 0, sizeof(fd_progcache_t) );
+ if( FD_UNLIKELY( !fd_funk_join( ljoin->funk, shfunk ) ) ) return NULL;
+
+ ljoin->metrics = &fd_progcache_metrics_default;
+ ljoin->scratch = scratch;
+ ljoin->scratch_sz = scratch_sz;
+
+ return ljoin;
+}
+
+void *
+fd_progcache_leave( fd_progcache_t * cache,
+ void ** opt_shfunk ) {
+ if( FD_UNLIKELY( !cache ) ) {
+ FD_LOG_WARNING(( "NULL cache" ));
+ return NULL;
+ }
+ if( FD_UNLIKELY( !fd_funk_leave( cache->funk, opt_shfunk ) ) ) return NULL;
+ cache->scratch = NULL;
+ cache->scratch_sz = 0UL;
+ return cache;
+}
+
+/* fd_progcache_load_fork pivots the progcache object to the selected
+ fork (identified by tip XID).
+
+ Populates cache->fork, which is a array-backed list of XIDs sorted
+ newest to oldest. Cache lookups only respect records with an XID
+ present in that list.
+
+ For any given xid, the epoch_slot0 is assumed to stay constant. */
+
+static void
+fd_progcache_load_fork_slow( fd_progcache_t * cache,
+ fd_funk_txn_xid_t const * xid,
+ ulong epoch_slot0 ) {
+ fd_funk_txn_xid_t next_xid = *xid;
+ if( FD_UNLIKELY( next_xid.ul[0]fork_depth = 0UL;
+
+ ulong txn_max = fd_funk_txn_pool_ele_max( cache->funk->txn_pool );
+ ulong i;
+ for( i=0UL; ifunk->txn_map, &next_xid, NULL, query, 0 );
+ if( FD_UNLIKELY( query_err==FD_MAP_ERR_AGAIN ) ) {
+ /* FIXME random backoff */
+ FD_SPIN_PAUSE();
+ continue;
+ }
+ if( query_err==FD_MAP_ERR_KEY ) goto done;
+ if( FD_UNLIKELY( query_err!=FD_MAP_SUCCESS ) ) {
+ FD_LOG_CRIT(( "fd_funk_txn_map_query_try failed: %i-%s", query_err, fd_map_strerror( query_err ) ));
+ }
+ break;
+ }
+
+ /* Lookup parent transaction while recovering from overruns
+ FIXME This would be a lot easier if transactions specified
+ parent by XID instead of by pointer ... */
+ candidate = fd_funk_txn_map_query_ele_const( query );
+ FD_COMPILER_MFENCE();
+ do {
+ found_xid = FD_VOLATILE_CONST( candidate->xid );
+ parent_idx = fd_funk_txn_idx( FD_VOLATILE_CONST( candidate->parent_cidx ) );
+ if( parent_idxfunk->txn_pool->ele[ parent_idx ];
+ parent_xid = FD_VOLATILE_CONST( parent->xid );
+ FD_COMPILER_MFENCE();
+ }
+ parent_idx = fd_funk_txn_idx( FD_VOLATILE_CONST( candidate->parent_cidx ) );
+ } while(0);
+ FD_COMPILER_MFENCE();
+
+ /* Verify speculative loads by ensuring txn still exists in map */
+ if( FD_UNLIKELY( fd_funk_txn_map_query_test( query )!=FD_MAP_SUCCESS ) ) {
+ FD_SPIN_PAUSE();
+ goto retry;
+ }
+
+ if( FD_UNLIKELY( !fd_funk_txn_xid_eq( &found_xid, &next_xid ) ) ) {
+ FD_LOG_CRIT(( "fd_progcache_load_fork_slow detected memory corruption: expected xid %lu:%lu at %p, found %lu:%lu",
+ next_xid.ul[0], next_xid.ul[1],
+ (void *)candidate,
+ found_xid.ul[0], found_xid.ul[1] ));
+ }
+
+ cache->fork[ i ] = next_xid;
+ if( fd_funk_txn_idx_is_null( parent_idx ) ||
+ next_xid.ul[0]fork_depth = i;
+
+ /* Only include published/rooted records if they include at least one
+ cache entry from the current epoch. */
+ if( fd_funk_last_publish( cache->funk )->ul[0] >= epoch_slot0 &&
+ cache->fork_depth < FD_PROGCACHE_DEPTH_MAX ) {
+ fd_funk_txn_xid_set_root( &cache->fork[ cache->fork_depth++ ] );
+ }
+}
+
+static inline void
+fd_progcache_load_fork( fd_progcache_t * cache,
+ fd_funk_txn_xid_t const * xid,
+ ulong epoch_slot0 ) {
+ /* Skip if already on the correct fork */
+ if( FD_LIKELY( (!!cache->fork_depth) & (!!fd_funk_txn_xid_eq( &cache->fork[ 0 ], xid ) ) ) ) return;
+ cache->metrics->fork_switch_cnt++;
+ fd_progcache_load_fork_slow( cache, xid, epoch_slot0 ); /* switch fork */
+}
+
+/* fd_progcache_query searches for a program cache entry on the current
+ fork. Stops short of an epoch boundary. */
+
+static int
+fd_progcache_fork_has_xid( fd_progcache_t const * cache,
+ fd_funk_txn_xid_t const * rec_xid ) {
+ /* FIXME unroll this a little */
+ ulong const fork_depth = cache->fork_depth;
+ for( ulong i=0UL; ifork[i], rec_xid ) ) return 1;
+ }
+ return 0;
+}
+
+static int
+fd_progcache_search_chain( fd_progcache_t const * cache,
+ ulong chain_idx,
+ fd_funk_rec_key_t const * key,
+ ulong epoch_slot0,
+ fd_funk_rec_t ** out_rec ) {
+ *out_rec = NULL;
+
+ fd_funk_rec_map_shmem_t * shmap = cache->funk->rec_map->map;
+ fd_funk_rec_map_shmem_private_chain_t const * chain_tbl = fd_funk_rec_map_shmem_private_chain_const( shmap, 0UL );
+ fd_funk_rec_map_shmem_private_chain_t const * chain = chain_tbl + chain_idx;
+ fd_funk_rec_t * rec_tbl = cache->funk->rec_pool->ele;
+ ulong rec_max = fd_funk_rec_pool_ele_max( cache->funk->rec_pool );
+ ulong ver_cnt = FD_VOLATILE_CONST( chain->ver_cnt );
+
+ /* Start a speculative transaction for the chain containing revisions
+ of the program cache key we are looking for. */
+ ulong cnt = fd_funk_rec_map_private_vcnt_cnt( ver_cnt );
+ if( FD_UNLIKELY( fd_funk_rec_map_private_vcnt_ver( ver_cnt )&1 ) ) {
+ return FD_MAP_ERR_AGAIN; /* chain is locked */
+ }
+ FD_COMPILER_MFENCE();
+ uint ele_idx = chain->head_cidx;
+
+ /* Walk the map chain, remember the best entry */
+ fd_funk_rec_t * best = NULL;
+ long best_slot = -1L;
+ for( ulong i=0UL; ipair.key, key ) ) ) continue;
+
+ /* Skip over records from an older epoch (FIXME could bail early
+ here if the chain is ordered) */
+ ulong found_slot = rec->pair.xid->ul[0];
+ if( FD_UNLIKELY( found_slotpair.xid ) ) ) continue;
+
+ best = rec;
+ best_slot = (long)found_slot;
+ if( FD_UNLIKELY( rec->map_next==ele_idx ) ) {
+ FD_LOG_CRIT(( "fd_progcache_search_chain detected cycle" ));
+ }
+ if( rec->map_next > rec_max ) {
+ if( FD_UNLIKELY( !fd_funk_rec_map_private_idx_is_null( rec->map_next ) ) ) {
+ FD_LOG_CRIT(( "fd_progcache_search_chain detected memory corruption: rec->map_next %u is out of bounds (rec_max %lu)",
+ rec->map_next, rec_max ));
+ }
+ }
+ }
+
+ /* Retry if we were overrun */
+ if( FD_UNLIKELY( FD_VOLATILE_CONST( chain->ver_cnt )!=ver_cnt ) ) {
+ return FD_MAP_ERR_AGAIN;
+ }
+
+ *out_rec = best;
+ return FD_MAP_SUCCESS;
+}
+
+static fd_funk_rec_t *
+fd_progcache_query( fd_progcache_t * cache,
+ fd_funk_txn_xid_t const * xid,
+ fd_funk_rec_key_t const * key,
+ ulong epoch_slot0 ) {
+ /* Hash key to chain */
+ fd_funk_xid_key_pair_t pair[1];
+ fd_funk_txn_xid_copy( pair->xid, xid );
+ fd_funk_rec_key_copy( pair->key, key );
+ fd_funk_rec_map_t const * rec_map = cache->funk->rec_map;
+ ulong hash = fd_funk_rec_map_key_hash( pair, rec_map->map->seed );
+ ulong chain_idx = (hash & (rec_map->map->chain_cnt-1UL) );
+
+ /* Traverse chain for candidate */
+ fd_funk_rec_t * rec = NULL;
+ for(;;) {
+ int err = fd_progcache_search_chain( cache, chain_idx, key, epoch_slot0, &rec );
+ if( FD_LIKELY( err==FD_MAP_SUCCESS ) ) break;
+ FD_SPIN_PAUSE();
+ /* FIXME backoff */
+ }
+
+ return rec;
+}
+
+fd_progcache_rec_t const *
+fd_progcache_peek_exact( fd_progcache_t * cache,
+ fd_funk_txn_xid_t const * xid,
+ void const * prog_addr ) {
+ fd_funk_xid_key_pair_t key[1];
+ fd_funk_txn_xid_copy( key->xid, xid );
+ memcpy( key->key->uc, prog_addr, 32UL );
+
+ for(;;) {
+ fd_funk_rec_map_query_t query[1];
+ int query_err = fd_funk_rec_map_query_try( cache->funk->rec_map, key, NULL, query, 0 );
+ if( query_err==FD_MAP_ERR_AGAIN ) {
+ FD_SPIN_PAUSE();
+ continue;
+ }
+ if( FD_UNLIKELY( query_err==FD_MAP_ERR_KEY ) ) return NULL;
+ if( FD_UNLIKELY( query_err!=FD_MAP_SUCCESS ) ) {
+ FD_LOG_CRIT(( "fd_funk_rec_map_query_try failed: %i-%s", query_err, fd_map_strerror( query_err ) ));
+ }
+ return fd_funk_val_const( fd_funk_rec_map_query_ele_const( query ), fd_funk_wksp( cache->funk ) );
+ }
+}
+
+fd_progcache_rec_t const *
+fd_progcache_peek( fd_progcache_t * cache,
+ fd_funk_txn_xid_t const * xid,
+ void const * prog_addr,
+ ulong epoch_slot0 ) {
+ if( FD_UNLIKELY( !cache || !cache->funk->shmem ) ) FD_LOG_CRIT(( "NULL progcache" ));
+ fd_progcache_load_fork( cache, xid, epoch_slot0 );
+ fd_funk_rec_key_t key[1]; memcpy( key->uc, prog_addr, 32UL );
+ fd_funk_rec_t const * rec = fd_progcache_query( cache, xid, key, epoch_slot0 );
+ if( FD_UNLIKELY( !rec ) ) return NULL;
+
+ fd_progcache_rec_t const * entry = fd_funk_val_const( rec, fd_funk_wksp( cache->funk ) );
+ if( entry->slot < epoch_slot0 ) entry = NULL;
+
+ cache->metrics->hit_cnt += !!entry;
+
+ return entry;
+}
+
+static void
+fd_funk_rec_push_tail( fd_funk_rec_t * rec_pool,
+ fd_funk_rec_t * rec,
+ uint * rec_head_idx,
+ uint * rec_tail_idx ) {
+ uint rec_idx = (uint)( rec - rec_pool );
+ for(;;) {
+
+ /* Doubly linked list append. Robust in the event of concurrent
+ publishes. Iteration during publish not supported. Sequence:
+ - Identify tail element
+ - Set new element's prev and next pointers
+ - Set tail element's next pointer
+ - Set tail pointer */
+
+ uint rec_prev_idx = FD_VOLATILE_CONST( *rec_tail_idx );
+ rec->prev_idx = rec_prev_idx;
+ rec->next_idx = FD_FUNK_REC_IDX_NULL;
+ FD_COMPILER_MFENCE();
+
+ uint * next_idx_p;
+ if( fd_funk_rec_idx_is_null( rec_prev_idx ) ) {
+ next_idx_p = rec_head_idx;
+ } else {
+ next_idx_p = &rec_pool[ rec_prev_idx ].next_idx;
+ }
+
+ if( FD_UNLIKELY( !__sync_bool_compare_and_swap( next_idx_p, FD_FUNK_REC_IDX_NULL, rec_idx ) ) ) {
+ /* Another thread beat us to the punch */
+ FD_SPIN_PAUSE();
+ continue;
+ }
+
+ if( FD_UNLIKELY( !__sync_bool_compare_and_swap( rec_tail_idx, rec_prev_idx, rec_idx ) ) ) {
+ /* This CAS is guaranteed to succeed if the previous CAS passed. */
+ FD_LOG_CRIT(( "Irrecoverable data race encountered while appending to txn rec list (invariant violation?): cas(%p,%u,%u)",
+ (void *)rec_tail_idx, rec_prev_idx, rec_idx ));
+ }
+
+ break;
+ }
+}
+
+__attribute__((warn_unused_result))
+static int
+fd_progcache_push( fd_progcache_t * cache,
+ fd_funk_txn_t * txn,
+ fd_funk_rec_t * rec,
+ void const * prog_addr ) {
+ fd_funk_t * funk = cache->funk;
+
+ /* Phase 1: Determine record's xid-key pair */
+
+ rec->tag = 0;
+ memcpy( rec->pair.key, prog_addr, 32UL );
+ if( FD_UNLIKELY( txn ) ) {
+ fd_funk_txn_xid_copy( rec->pair.xid, &txn->xid );
+ } else {
+ fd_funk_txn_xid_set_root( rec->pair.xid );
+ }
+
+ /* Phase 2: Lock rec_map chain, entering critical section */
+
+ struct {
+ fd_funk_rec_map_txn_t txn[1];
+ fd_funk_rec_map_txn_private_info_t info[1];
+ } _map_txn;
+ fd_funk_rec_map_txn_t * map_txn = fd_funk_rec_map_txn_init( _map_txn.txn, funk->rec_map, 1UL );
+ fd_funk_rec_map_txn_add( map_txn, &rec->pair, 1 );
+ int txn_err = fd_funk_rec_map_txn_try( map_txn, FD_MAP_FLAG_BLOCKING );
+ if( FD_UNLIKELY( txn_err!=FD_MAP_SUCCESS ) ) {
+ FD_LOG_CRIT(( "Failed to insert progcache record: canont lock funk rec map chain: %i-%s", txn_err, fd_map_strerror( txn_err ) ));
+ }
+
+ /* Phase 3: Atomically add record to funk txn's record list */
+
+ int insert_err = fd_funk_rec_map_txn_insert( funk->rec_map, rec );
+ if( FD_UNLIKELY( insert_err==FD_MAP_ERR_KEY ) ) {
+ fd_funk_rec_map_txn_test( map_txn );
+ fd_funk_rec_map_txn_fini( map_txn );
+ return 0; /* another thread was faster */
+ }
+
+ /* At this point, another thread could aggressively evict this entry.
+ But this entry is not yet present in rec_map! This is why we hold
+ a lock on the rec_map chain -- the rec_map_remove executed by the
+ eviction will be sequenced after completion of phase 5. */
+
+ /* Phase 4: Insert rec into rec_map */
+
+ if( txn ) {
+ fd_funk_rec_push_tail( funk->rec_pool->ele,
+ rec,
+ &txn->rec_head_idx,
+ &txn->rec_tail_idx );
+ }
+
+ /* Phase 5: Finish rec_map transaction */
+
+ int test_err = fd_funk_rec_map_txn_test( map_txn );
+ if( FD_UNLIKELY( test_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_funk_rec_map_txn_test failed: %i-%s", test_err, fd_map_strerror( test_err ) ));
+ fd_funk_rec_map_txn_fini( map_txn );
+ return 1;
+}
+
+static int
+fd_progcache_txn_try_lock( fd_funk_txn_t * txn ) {
+ for(;;) {
+ ushort * lock = &txn->lock->value;
+ ushort value = FD_VOLATILE_CONST( *lock );
+ if( FD_UNLIKELY( value>=0xFFFE ) ) return 0; /* txn is write-locked */
+ if( FD_LIKELY( FD_ATOMIC_CAS( lock, value, value+1 )==value ) ) {
+ return 1; /* transaction now read-locked */
+ }
+ }
+}
+
+static void
+fd_progcache_txn_unlock( fd_funk_txn_t * txn ) {
+ if( !txn ) return;
+ fd_rwlock_unread( txn->lock );
+}
+
+/* fd_progcache_lock_best_txn picks a fork graph node close to
+ target_slot and write locks it for program cache entry insertion.
+
+ The cache entry should be placed as far up the fork graph as
+ possible (so it can be shared across more downstream forks), but not
+ too early (or it would cause non-determinism).
+
+ Influenced by a number of things:
+
+ - Program modification time (cannot predate the program modification)
+ - Epoch boundaries (cannot span across epochs)
+ - Transaction publishing (cannot create a cache entry at a txn that
+ is in the process of being published) */
+
+static fd_funk_txn_t *
+fd_progcache_lock_best_txn( fd_progcache_t * cache,
+ ulong target_slot ) {
+
+ fd_funk_txn_xid_t last_publish[1];
+ fd_funk_txn_xid_ld_atomic( last_publish, fd_funk_last_publish( cache->funk ) );
+ if( target_slot <= last_publish->ul[0] &&
+ !fd_funk_txn_xid_eq_root( last_publish ) ) {
+ return NULL; /* publishing record immediately */
+ }
+
+ /* Scan fork graph for oldest node (>= program update slot) */
+ ulong target_xid_idx;
+ ulong fork_depth = cache->fork_depth;
+ for( target_xid_idx=0UL; target_xid_idxfork[ target_xid_idx ].ul[0]<=target_slot ) break;
+ }
+
+ /* Backtrack up to newer fork graph nodes (>= access slot)
+ Very old slots could have been rooted at this point */
+ fd_funk_txn_t * txn;
+ do {
+ /* Locate fork */
+ fd_funk_txn_xid_t const * xid = &cache->fork[ target_xid_idx ];
+ txn = fd_funk_txn_query( xid, cache->funk->txn_map );
+ if( FD_LIKELY( txn ) ) {
+ /* Attempt to read-lock transaction */
+ if( FD_LIKELY( fd_progcache_txn_try_lock( txn ) ) ) return txn;
+ }
+ /* Cannot insert at this fork graph node, try one newer */
+ target_xid_idx--;
+ } while( target_xid_idx!=ULONG_MAX );
+
+ /* There is no funk_txn in range [target_slot,load_slot] that we can
+ create a cache entry at. */
+ FD_LOG_CRIT(( "Could not find program cache fork graph node for target slot %lu", target_slot ));
+}
+
+static fd_progcache_rec_t const *
+fd_progcache_insert( fd_progcache_t * cache,
+ fd_funk_t * accdb,
+ fd_funk_txn_xid_t const * load_xid,
+ void const * prog_addr,
+ fd_prog_load_env_t const * env,
+ long slot_min ) {
+
+ /* XID overview:
+
+ - load_xid: tip of fork currently being executed
+ - modify_xid: xid in which program was last modified / deployed
+ - txn->xid: xid in which program cache entry is inserted
+
+ slot(load_xid) > slot(entry_xid) >= slot(txn->xid) */
+
+ /* Acquire reference to ELF binary data */
+
+ fd_funk_txn_xid_t modify_xid;
+ ulong progdata_sz;
+ uchar const * progdata = fd_prog_load_elf( accdb, load_xid, prog_addr, &progdata_sz, &modify_xid );
+ if( FD_UNLIKELY( !progdata ) ) return NULL;
+ ulong target_slot = modify_xid.ul[0];
+
+ /* Prevent cache entry from crossing epoch boundary */
+
+ target_slot = fd_ulong_max( target_slot, env->epoch_slot0 );
+
+ /* Prevent cache entry from shadowing invalidation */
+
+ target_slot = (ulong)fd_long_max( (long)target_slot, slot_min );
+
+ /* Allocate a funk_rec */
+
+ fd_funk_t * funk = cache->funk;
+ fd_funk_rec_t * funk_rec = fd_funk_rec_pool_acquire( funk->rec_pool, NULL, 0, NULL );
+ if( FD_UNLIKELY( !funk_rec ) ) {
+ FD_LOG_ERR(( "Program cache is out of memory: fd_funk_rec_pool_acquire failed (rec_max=%lu)",
+ fd_funk_rec_pool_ele_max( funk->rec_pool ) ));
+ }
+ memset( funk_rec, 0, sizeof(fd_funk_rec_t) );
+ fd_funk_val_init( funk_rec );
+
+ /* Pick and lock a txn in which cache entry is created at */
+
+ fd_funk_txn_t * txn = fd_progcache_lock_best_txn( cache, target_slot );
+
+ /* Load program */
+
+ fd_features_t const * features = env->features;
+ ulong const load_slot = env->slot;
+ fd_prog_versions_t versions = fd_prog_versions( features, load_slot );
+ fd_sbpf_loader_config_t config = {
+ .sbpf_min_version = versions.min_sbpf_version,
+ .sbpf_max_version = versions.max_sbpf_version,
+ };
+ fd_sbpf_elf_info_t elf_info[1];
+
+ fd_progcache_rec_t * rec = NULL;
+ if( FD_LIKELY( fd_sbpf_elf_peek( elf_info, progdata, progdata_sz, &config )==FD_SBPF_ELF_SUCCESS ) ) {
+
+ fd_funk_t * funk = cache->funk;
+ ulong rec_align = fd_progcache_rec_align();
+ ulong rec_footprint = fd_progcache_rec_footprint( elf_info );
+
+ void * rec_mem = fd_funk_val_truncate( funk_rec, funk->alloc, funk->wksp, rec_align, rec_footprint, NULL );
+ if( FD_UNLIKELY( !rec_mem ) ) {
+ FD_LOG_ERR(( "Program cache is out of memory: fd_alloc_malloc failed (requested align=%lu sz=%lu)",
+ rec_align, rec_footprint ));
+ }
+
+ rec = fd_progcache_rec_new( rec_mem, elf_info, &config, load_slot, features, progdata, progdata_sz, cache->scratch, cache->scratch_sz );
+ if( !rec ) {
+ fd_funk_val_flush( funk_rec, funk->alloc, funk->wksp );
+ }
+
+ }
+
+ /* Convert to tombstone if load failed */
+
+ if( !rec ) { /* load fail */
+ void * rec_mem = fd_funk_val_truncate( funk_rec, funk->alloc, funk->wksp, fd_progcache_rec_align(), fd_progcache_rec_footprint( NULL ), NULL );
+ if( FD_UNLIKELY( !rec_mem ) ) {
+ FD_LOG_ERR(( "Program cache is out of memory: fd_alloc_malloc failed (requested align=%lu sz=%lu)",
+ fd_progcache_rec_align(), fd_progcache_rec_footprint( NULL ) ));
+ }
+ rec = fd_progcache_rec_new_nx( rec_mem, load_slot );
+ }
+
+ /* Publish cache entry to funk index */
+
+ int push_ok = fd_progcache_push( cache, txn, funk_rec, prog_addr );
+
+ /* Done modifying transaction */
+
+ fd_progcache_txn_unlock( txn );
+
+ /* If another thread was faster publishing the same record, use that
+ one instead. FIXME POSSIBLE RACE CONDITION WHERE THE OTHER REC IS
+ EVICTED AFTER PEEK? */
+
+ if( !push_ok ) {
+ fd_progcache_rec_t const * other = fd_progcache_peek( cache, load_xid, prog_addr, env->epoch_slot0 );
+ if( FD_UNLIKELY( !other ) ) {
+ FD_LOG_CRIT(( "fd_progcache_push/fd_progcache_peek data race detected" ));
+ }
+ cache->metrics->dup_insert_cnt++;
+ return other;
+ }
+
+ cache->metrics->fill_cnt++;
+ cache->metrics->fill_tot_sz += rec->rodata_sz;
+
+ return rec;
+}
+
+fd_progcache_rec_t const *
+fd_progcache_pull( fd_progcache_t * cache,
+ fd_funk_t * accdb,
+ fd_funk_txn_xid_t const * xid,
+ void const * prog_addr,
+ fd_prog_load_env_t const * env ) {
+ if( FD_UNLIKELY( !cache || !cache->funk->shmem ) ) FD_LOG_CRIT(( "NULL progcache" ));
+ fd_progcache_load_fork( cache, xid, env->epoch_slot0 );
+
+ fd_progcache_rec_t const * found_rec = fd_progcache_peek( cache, xid, prog_addr, env->epoch_slot0 );
+ long slot_min = -1L;
+ if( !found_rec ) goto miss;
+
+ /* Cache invalidation, update next slot */
+ if( found_rec->invalidate ) {
+ slot_min = (long)found_rec->slot+1L;
+ if( FD_UNLIKELY( xid->ul[0] < (ulong)slot_min ) ) {
+ FD_LOG_CRIT(( "Program cache entry %016lx%016lx%016lx%016lx invalidated at slot %lu but loaded at slot %lu",
+ fd_ulong_bswap( FD_LOAD( ulong, (uchar const *)prog_addr ) ),
+ fd_ulong_bswap( FD_LOAD( ulong, (uchar const *)prog_addr+ 8 ) ),
+ fd_ulong_bswap( FD_LOAD( ulong, (uchar const *)prog_addr+16 ) ),
+ fd_ulong_bswap( FD_LOAD( ulong, (uchar const *)prog_addr+24 ) ),
+ found_rec->slot,
+ xid->ul[0] ));
+ }
+ goto miss;
+ }
+
+ /* Passed all checks */
+ cache->metrics->hit_cnt++;
+ cache->metrics->hit_tot_sz += found_rec->rodata_sz;
+ return found_rec;
+
+miss:
+ cache->metrics->miss_cnt++;
+ return fd_progcache_insert( cache, accdb, xid, prog_addr, env, slot_min );
+}
+
+fd_progcache_rec_t const *
+fd_progcache_invalidate( fd_progcache_t * cache,
+ fd_funk_txn_xid_t const * xid,
+ void const * prog_addr,
+ ulong slot ) {
+ fd_funk_t * funk = cache->funk;
+
+ if( FD_UNLIKELY( !cache || !funk->shmem ) ) FD_LOG_CRIT(( "NULL progcache" ));
+
+ fd_funk_txn_t * txn = fd_funk_txn_query( xid, funk->txn_map );
+ if( FD_UNLIKELY( !fd_progcache_txn_try_lock( txn ) ) ) {
+ FD_LOG_CRIT(( "fd_progcache_invalidate(xid=%lu,...) failed: txn is write-locked", xid->ul[0] ));
+ }
+
+ /* Allocate a funk_rec */
+
+ fd_funk_rec_t * funk_rec = fd_funk_rec_pool_acquire( funk->rec_pool, NULL, 0, NULL );
+ if( FD_UNLIKELY( !funk_rec ) ) {
+ FD_LOG_ERR(( "Program cache is out of memory: fd_funk_rec_pool_acquire failed (rec_max=%lu)",
+ fd_funk_rec_pool_ele_max( funk->rec_pool ) ));
+ }
+ memset( funk_rec, 0, sizeof(fd_funk_rec_t) );
+ fd_funk_val_init( funk_rec );
+
+ /* Create a tombstone */
+
+ void * rec_mem = fd_funk_val_truncate( funk_rec, funk->alloc, funk->wksp, fd_progcache_rec_align(), fd_progcache_rec_footprint( NULL ), NULL );
+ if( FD_UNLIKELY( !rec_mem ) ) {
+ FD_LOG_ERR(( "Program cache is out of memory: fd_alloc_malloc failed (requested align=%lu sz=%lu)",
+ fd_progcache_rec_align(), fd_progcache_rec_footprint( NULL ) ));
+ }
+ fd_progcache_rec_t * rec = fd_progcache_rec_new_nx( rec_mem, slot );
+ rec->invalidate = 1;
+
+ /* Publish cache entry to funk index */
+
+ int push_ok = fd_progcache_push( cache, txn, funk_rec, prog_addr );
+
+ /* Done modifying transaction */
+
+ fd_progcache_txn_unlock( txn );
+
+ /* If another thread was faster publishing the same record, use that
+ one instead. FIXME POSSIBLE RACE CONDITION WHERE THE OTHER REC IS
+ EVICTED AFTER PEEK? */
+
+ if( !push_ok ) {
+ fd_progcache_rec_t const * other = fd_progcache_peek_exact( cache, xid, prog_addr );
+ if( FD_UNLIKELY( !other ) ) {
+ FD_LOG_CRIT(( "fd_progcache_push/fd_progcache_peek data race detected" ));
+ }
+ cache->metrics->dup_insert_cnt++;
+ return other;
+ }
+
+ cache->metrics->invalidate_cnt++;
+
+ return rec;
+}
diff --git a/src/flamenco/progcache/fd_progcache_user.h b/src/flamenco/progcache/fd_progcache_user.h
new file mode 100644
index 00000000000..87327251cec
--- /dev/null
+++ b/src/flamenco/progcache/fd_progcache_user.h
@@ -0,0 +1,190 @@
+
+#ifndef HEADER_fd_src_flamenco_fd_progcache_h
+#define HEADER_fd_src_flamenco_fd_progcache_h
+
+/* fd_progcache_user.h provides an API for managing a cache of loaded
+ Solana on-chain program.
+
+ ### Background
+
+ Solana on-chain programs are rarely updated but frequently executed.
+ Before a program can be executed, it must be loaded and verified,
+ which is costly.
+
+ ### Fork management
+
+ The program cache is fork-aware (using funk transactions). Txn-level
+ operations take an exclusive lock over the cache (record ops are
+ stalled indefinitely until the txn completes).
+
+ ### Cache entry
+
+ Each Solana program can have a number of program cache entries
+ (typically only zero or one, in rare cases where the program content
+ differs across forks multiple).
+
+ A cache entry consists of a funk_rec object (from a preallocated
+ object pool), and a variable-sized fd_progcache_entry struct
+ (from an fd_alloc heap).
+
+ ### Cache fill policy
+
+ fd_progcache is lazily filled on reads, and eagerly invalidated
+ if underlying programs are written to.
+
+ ### Cache evict policy
+
+ Cache eviction (i.e. force removal of potentially useful records)
+ happens on fill. Specifically, cache eviction is triggered when a
+ cache fill fails to allocate from the wksp (fd_alloc) heap.
+
+ fd_progcache further has a concept of "generations" (gen). Each
+ cache fill operation specifies a 'gen' number. Only entries with a
+ lower 'gen' number may get evicted.
+
+ ### Garbage collect policy
+
+ fd_progcache cleans up unused entries eagerly when:
+
+ 1. a database fork is cancelled (e.g. slot is rooted and competing
+ history dies, or consensus layer prunes a fork)
+ 2. a cache entry is orphaned (updated or invalidated by an epoch
+ boundary) */
+
+#include "fd_progcache_rec.h"
+#include "fd_prog_load.h"
+#include "../../funk/fd_funk.h"
+#include "../runtime/fd_runtime_const.h"
+
+#define FD_PROGCACHE_DEPTH_MAX (128UL)
+
+struct fd_progcache_metrics {
+ ulong fork_switch_cnt;
+ ulong miss_cnt;
+ ulong hit_cnt;
+ ulong hit_tot_sz;
+ ulong fill_cnt;
+ ulong fill_tot_sz;
+ ulong fill_fail_cnt;
+ ulong dup_insert_cnt;
+ ulong invalidate_cnt;
+};
+
+typedef struct fd_progcache_metrics fd_progcache_metrics_t;
+
+/* fd_progcache_t is a thread-local client to a program cache funk
+ instance. This struct is quite large and therefore not local/stack
+ declaration-friendly. */
+
+struct fd_progcache {
+ fd_funk_t funk[1];
+
+ /* Current fork cache */
+ fd_funk_txn_xid_t fork[ FD_PROGCACHE_DEPTH_MAX ];
+ ulong fork_depth;
+
+ fd_progcache_metrics_t * metrics;
+
+ uchar * scratch;
+ ulong scratch_sz;
+};
+
+typedef struct fd_progcache fd_progcache_t;
+
+FD_PROTOTYPES_BEGIN
+
+extern FD_TL fd_progcache_metrics_t fd_progcache_metrics_default;
+
+/* Constructor */
+
+static inline ulong
+fd_progcache_align( void ) {
+ return alignof(fd_progcache_t);
+}
+
+static inline ulong
+fd_progcache_footprint( void ) {
+ return sizeof(fd_progcache_t);
+}
+
+static inline fd_progcache_t *
+fd_progcache_new( void * ljoin ) {
+ return ljoin;
+}
+
+static inline void *
+fd_progcache_delete( void * ljoin ) {
+ return ljoin;
+}
+
+/* fd_progcache_join joins the caller to a program cache funk instance.
+ scratch points to a FD_PROGCACHE_SCRATCH_ALIGN aligned scratch buffer
+ and scratch_sz is the size of the largest program/ELF binary that is
+ going to be loaded (typically max account data sz). */
+
+fd_progcache_t *
+fd_progcache_join( fd_progcache_t * ljoin,
+ void * shfunk,
+ uchar * scratch,
+ ulong scratch_sz );
+
+#define FD_PROGCACHE_SCRATCH_ALIGN (64UL)
+#define FD_PROGCACHE_SCRATCH_FOOTPRINT FD_RUNTIME_ACC_SZ_MAX
+
+/* fd_progcache_leave detaches the caller from a program cache. */
+
+void *
+fd_progcache_leave( fd_progcache_t * cache,
+ void ** opt_shfunk );
+
+/* Record-level operations ********************************************/
+
+/* fd_progcache_peek queries the program cache for an existing cache
+ entry. Does not fill the cache. Returns a pointer to the entry on
+ cache hit (invalidated by the next non-const API call). Returns NULL
+ on cache miss. */
+
+fd_progcache_rec_t const *
+fd_progcache_peek( fd_progcache_t * cache,
+ fd_funk_txn_xid_t const * xid,
+ void const * prog_addr,
+ ulong epoch_slot0 );
+
+/* fd_progcache_pull loads a program from cache, filling the cache if
+ necessary. The load operation can have a number of outcomes:
+ - Returns a pointer to an existing cache entry (cache hit, state
+ either "Loaded" or "FailedVerification")
+ - Returns a pointer to a newly created cache entry (cache fill,
+ state either "Loaded" or "FailedVerification")
+ - Returns NULL if the requested program account is not deployed (i.e.
+ account is missing, the program is under visibility delay, or user
+ has not finished uploading the program)
+ In other words, this method guarantees to return a cache entry if a
+ deployed program was found in the account database, and the program
+ either loaded successfully, or failed ELF/bytecode verification.
+ Or it returns */
+
+fd_progcache_rec_t const *
+fd_progcache_pull( fd_progcache_t * cache,
+ fd_funk_t * accdb,
+ fd_funk_txn_xid_t const * xid,
+ void const * prog_addr,
+ fd_prog_load_env_t const * env );
+
+/* fd_progcache_invalidate marks the program at the given address as
+ invalidated (typically due to a change of program content). This
+ creates a non-executable cache entry at the given xid.
+
+ After a program has been invalidated at xid, it is forbidden to pull
+ the same entry at the same xid. (Invalidations should happen after
+ replaying transactions). */
+
+fd_progcache_rec_t const *
+fd_progcache_invalidate( fd_progcache_t * cache,
+ fd_funk_txn_xid_t const * xid,
+ void const * prog_addr,
+ ulong slot );
+
+FD_PROTOTYPES_END
+
+#endif /* HEADER_fd_src_flamenco_fd_progcache_h */
diff --git a/src/flamenco/progcache/test_progcache.c b/src/flamenco/progcache/test_progcache.c
new file mode 100644
index 00000000000..66e14844fee
--- /dev/null
+++ b/src/flamenco/progcache/test_progcache.c
@@ -0,0 +1,726 @@
+/* test_progcache.c contains single-threaded correctness tests for
+ progcache. */
+
+#include "fd_progcache_admin.h"
+#include "fd_progcache_user.h"
+#include "../runtime/fd_bank.h"
+
+/* Load in programdata for tests */
+FD_IMPORT_BINARY( valid_program_data, "src/ballet/sbpf/fixtures/hello_solana_program.so" );
+FD_IMPORT_BINARY( bigger_valid_program_data, "src/ballet/sbpf/fixtures/clock_sysvar_program.so" );
+FD_IMPORT_BINARY( invalid_program_data, "src/ballet/sbpf/fixtures/malformed_bytecode.so" );
+
+struct test_env {
+ fd_wksp_t * wksp;
+
+ fd_progcache_admin_t progcache_admin[1];
+ fd_progcache_t progcache[1];
+ fd_funk_t accdb[1];
+ fd_features_t features[1];
+
+ uchar scratch[ FD_PROGCACHE_SCRATCH_FOOTPRINT ] __attribute__((aligned(FD_PROGCACHE_SCRATCH_ALIGN)));
+};
+
+typedef struct test_env test_env_t;
+
+/* test_env_create allocates a new account database (funk) and loaded
+ program cache (also funk) from a wksp. Joins an admin and user
+ client to the program cache, as well as a database client. */
+
+static test_env_t *
+test_env_create( fd_wksp_t * wksp ) {
+ ulong txn_max = 16UL;
+ ulong accdb_rec_max = 32UL;
+ ulong progcache_rec_max = 32UL;
+ ulong wksp_tag = 1UL;
+
+ void * accdb_mem = fd_wksp_alloc_laddr( wksp, fd_funk_align(), fd_funk_footprint( txn_max, accdb_rec_max ), wksp_tag );
+ FD_TEST( fd_funk_new( accdb_mem, wksp_tag, 1UL, txn_max, accdb_rec_max ) );
+
+ void * progcache_mem = fd_wksp_alloc_laddr( wksp, fd_funk_align(), fd_funk_footprint( txn_max, progcache_rec_max ), wksp_tag );
+ FD_TEST( fd_funk_new( progcache_mem, wksp_tag, 1UL, txn_max, progcache_rec_max ) );
+
+ test_env_t * env = fd_wksp_alloc_laddr( wksp, alignof(test_env_t), sizeof(test_env_t), wksp_tag );
+ FD_TEST( env );
+ memset( env, 0, sizeof(test_env_t) );
+
+ env->wksp = wksp;
+ FD_TEST( fd_progcache_admin_join( env->progcache_admin, progcache_mem ) );
+ FD_TEST( fd_progcache_join( env->progcache, progcache_mem, env->scratch, sizeof(env->scratch) ) );
+ FD_TEST( fd_funk_join( env->accdb, accdb_mem ) );
+
+ return env;
+}
+
+/* test_env_destroy frees all test env objects. */
+
+static void
+test_env_destroy( test_env_t * env ) {
+ fd_progcache_verify_stat_t stat[1];
+ fd_progcache_verify( env->progcache_admin, stat );
+
+ void * accdb_mem = NULL;
+ FD_TEST( fd_progcache_admin_leave( env->progcache_admin, &accdb_mem ) );
+ FD_TEST( fd_progcache_leave ( env->progcache, &accdb_mem ) );
+ fd_wksp_free_laddr( fd_funk_delete( accdb_mem ) );
+
+ void * progcache_mem = NULL;
+ FD_TEST( fd_funk_leave( env->accdb, &progcache_mem ) );
+ fd_wksp_free_laddr( fd_funk_delete( progcache_mem ) );
+
+ fd_wksp_free_laddr( env );
+}
+
+/* test_env_txn_prepare creates a new in-prep funk transaction off
+ parent with the given xid, in both accdb and progcache. */
+
+static void
+test_env_txn_prepare( test_env_t * env,
+ fd_funk_txn_xid_t const * parent,
+ fd_funk_txn_xid_t const * xid ) {
+ fd_funk_txn_xid_t root[1];
+ if( !parent ) {
+ fd_funk_txn_xid_set_root( root );
+ parent = root;
+ }
+ fd_funk_txn_prepare( env->accdb, parent, xid );
+ fd_progcache_txn_prepare( env->progcache_admin, parent, xid );
+}
+
+/* test_env_txn_cancel destroys a subtree of in-prep funk transactions
+ with root 'xid', in both accdb and progcache. */
+
+static void
+test_env_txn_cancel( test_env_t * env,
+ fd_funk_txn_xid_t const * xid ) {
+ fd_funk_txn_cancel( env->accdb, xid );
+ fd_progcache_txn_cancel( env->progcache_admin, xid );
+}
+
+/* test_env_txn_publish publishes (i.e. roots) a subtree of in-prep funk
+ transactions with root 'xid', in both accdb and progcache. */
+
+static void
+test_env_txn_publish( test_env_t * env,
+ fd_funk_txn_xid_t const * xid ) {
+ fd_funk_txn_publish( env->accdb, xid );
+ fd_progcache_txn_publish( env->progcache_admin, xid );
+}
+
+static fd_funk_rec_key_t
+test_key( ulong x ) {
+ fd_funk_rec_key_t key = {0};
+ key.ul[0] = x;
+ return key;
+}
+
+/* create_test_account creates an account in the account database. */
+
+static void
+create_test_account( test_env_t * env,
+ fd_funk_txn_xid_t const * xid,
+ void const * pubkey_,
+ void const * owner_,
+ void const * data,
+ ulong data_len,
+ uchar executable ) {
+ fd_pubkey_t pubkey = FD_LOAD( fd_pubkey_t, pubkey_ );
+ fd_pubkey_t owner = FD_LOAD( fd_pubkey_t, owner_ ) ;
+
+ fd_txn_account_t acc[1];
+ fd_funk_rec_prepare_t prepare = {0};
+ int err = fd_txn_account_init_from_funk_mutable( /* acc */ acc,
+ /* pubkey */ &pubkey,
+ /* funk */ env->accdb,
+ /* xid */ xid,
+ /* do_create */ 1,
+ /* min_data_sz */ data_len,
+ /* prepare */ &prepare );
+ FD_TEST( !err );
+
+ if( data ) {
+ fd_txn_account_set_data( acc, data, data_len );
+ }
+
+ acc->starting_lamports = 1UL;
+ acc->starting_dlen = data_len;
+ fd_txn_account_set_lamports( acc, 1UL );
+ fd_txn_account_set_executable( acc, executable );
+ fd_txn_account_set_owner( acc, &owner );
+
+ fd_txn_account_mutable_fini( acc, env->accdb, &prepare );
+}
+
+/* query_rec_exact fetches a funk record at a precise xid:key pair. */
+
+static fd_funk_rec_t const *
+query_rec_exact( test_env_t * env,
+ fd_funk_txn_xid_t const * xid,
+ fd_funk_rec_key_t const * key ) {
+ fd_funk_xid_key_pair_t pair[1];
+ fd_funk_txn_xid_copy( pair->xid, xid );
+ fd_funk_rec_key_copy( pair->key, key );
+
+ fd_funk_rec_map_query_t query[1];
+ int query_err = fd_funk_rec_map_query_try( env->progcache->funk->rec_map, pair, NULL, query, 0 );
+ if( query_err==FD_MAP_ERR_KEY ) return NULL;
+ if( FD_UNLIKELY( query_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_funk_rec_map_query_try failed: %i-%s", query_err, fd_map_strerror( query_err ) ));
+
+ return fd_funk_rec_map_query_ele_const( query );
+}
+
+/* test_empty: Account database and progcache completely empty.
+ Query at root should fail. */
+
+static void
+test_empty( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+
+ fd_funk_txn_xid_t xid[1]; fd_funk_txn_xid_set_root( xid );
+ fd_funk_rec_key_t key = test_key( 1UL );
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec = fd_progcache_pull( env->progcache, env->accdb, xid, &key, &load_env );
+ FD_TEST( !rec );
+
+ test_env_destroy( env );
+}
+
+/* test_account_does_not_exist: Program account missing, but querying at
+ a fork. */
+
+static void
+test_account_does_not_exist( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ (void)test_env_txn_publish;
+
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_invalid_owner: Account exists but is not owned by BPF loader */
+
+static void
+test_invalid_owner( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_system_program_id, /* not a BPF laoder */
+ invalid_program_data,
+ invalid_program_data_sz,
+ 1 );
+
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ FD_TEST( !fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env ) );
+
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_invalid_program: Program account exists but fails loading */
+
+static void
+test_invalid_program( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_bpf_loader_program_id,
+ invalid_program_data,
+ invalid_program_data_sz,
+ 1 );
+
+ FD_TEST( !fd_progcache_peek( env->progcache, &fork_a, &key, 0UL ) );
+ FD_TEST( env->progcache->fork_depth==2UL );
+ FD_TEST( fd_funk_txn_xid_eq( &env->progcache->fork[ 0 ], &fork_a ) );
+ FD_TEST( fd_funk_txn_xid_eq( &env->progcache->fork[ 1 ], fd_funk_root( env->progcache->funk ) ) );
+
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec = fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env );
+ FD_TEST( rec );
+ FD_TEST( !rec->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, load_env.epoch_slot0 )==rec );
+
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_valid_program: Load a valid program account */
+
+static void
+test_valid_program( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_bpf_loader_program_id,
+ valid_program_data,
+ valid_program_data_sz,
+ 1 );
+
+ FD_TEST( !fd_progcache_peek( env->progcache, &fork_a, &key, 0UL ) );
+ FD_TEST( env->progcache->fork_depth==2UL );
+ FD_TEST( fd_funk_txn_xid_eq( &env->progcache->fork[ 0 ], &fork_a ) );
+ FD_TEST( fd_funk_txn_xid_eq( &env->progcache->fork[ 1 ], fd_funk_root( env->progcache->funk ) ) );
+
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec = fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env );
+ FD_TEST( rec );
+ FD_TEST( rec->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+ FD_TEST( env->progcache->fork_depth==2UL );
+
+ fd_funk_txn_xid_t fork_b = { .ul = { 64UL, 2UL } };
+ test_env_txn_prepare( env, &fork_a, &fork_b );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, 0UL )==rec );
+ FD_TEST( env->progcache->fork_depth==3UL );
+
+ load_env.slot = 64UL;
+ load_env.epoch = 0UL;
+ load_env.epoch_slot0 = 0UL;
+ fd_progcache_rec_t const * rec2 = fd_progcache_pull( env->progcache, env->accdb, &fork_b, &key, &load_env );
+ FD_TEST( rec==rec2 );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, 0UL )==rec );
+
+ test_env_txn_cancel( env, &fork_a ); /* should also cancel fork_b */
+ test_env_destroy( env );
+}
+
+/* test_epoch_boundary: Ensure that a valid program gets re-verified
+ after an epoch boundary. */
+
+static void
+test_epoch_boundary( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_bpf_loader_program_id,
+ valid_program_data,
+ valid_program_data_sz,
+ 1 );
+
+ FD_TEST( !fd_progcache_peek( env->progcache, &fork_a, &key, 0UL ) );
+ FD_TEST( env->progcache->fork_depth==2UL );
+ FD_TEST( fd_funk_txn_xid_eq( &env->progcache->fork[ 0 ], &fork_a ) );
+ FD_TEST( fd_funk_txn_xid_eq( &env->progcache->fork[ 1 ], fd_funk_root( env->progcache->funk ) ) );
+
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec = fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env );
+ FD_TEST( rec );
+ FD_TEST( rec->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ fd_funk_txn_xid_t fork_b = { .ul = { 64UL, 2UL } };
+ test_env_txn_prepare( env, &fork_a, &fork_b );
+ load_env.slot = 64UL;
+ load_env.epoch = 1UL;
+ load_env.epoch_slot0 = 64UL;
+ fd_progcache_rec_t const * rec2 = fd_progcache_pull( env->progcache, env->accdb, &fork_b, &key, &load_env );
+ FD_TEST( rec2 );
+ FD_TEST( rec!=rec2 );
+ FD_TEST( rec2->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, load_env.epoch_slot0 )==rec2 );
+
+ test_env_txn_cancel( env, &fork_b );
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_invalidate: Ensure that an fd_progcache_invalidate call
+ overrides a previously created cache entry. */
+
+static void
+test_invalidate( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_bpf_loader_program_id,
+ valid_program_data,
+ valid_program_data_sz,
+ 1 );
+
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec = fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env );
+ FD_TEST( rec );
+ FD_TEST( rec->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ fd_funk_txn_xid_t fork_b = { .ul = { 2UL, 1UL } };
+ test_env_txn_prepare( env, &fork_a, &fork_b );
+ fd_progcache_rec_t const * rec2 = fd_progcache_invalidate( env->progcache, &fork_b, &key, fork_b.ul[0] );
+ FD_TEST( rec2!=rec );
+ FD_TEST( !rec2->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, 0UL )==rec2 );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_invalidate_nonexistent: fd_progcache_invalidate should create an
+ entry even if there are no other cache entries for the same key.
+ (To prevent a future fill predating the invalidation from having side
+ effects for slots after the invalidation.) */
+
+static void
+test_invalidate_nonexistent( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ fd_progcache_rec_t const * rec = fd_progcache_invalidate( env->progcache, &fork_a, &key, fork_a.ul[0] );
+ FD_TEST( rec );
+ FD_TEST( !rec->executable );
+
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_invalidate_pull: fd_progcache_pull should recover from an
+ earlier fd_progcache_invalidate call (in a future slot). */
+
+static void
+test_invalidate_pull( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_bpf_loader_program_id,
+ valid_program_data,
+ valid_program_data_sz,
+ 1 );
+
+ /* Create initial cache entry */
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec = fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env );
+ FD_TEST( rec );
+ FD_TEST( rec->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ /* Create cache invalidation entry */
+ fd_funk_txn_xid_t fork_b = { .ul = { 2UL, 1UL } };
+ test_env_txn_prepare( env, &fork_a, &fork_b );
+ fd_progcache_rec_t const * rec2 = fd_progcache_invalidate( env->progcache, &fork_b, &key, fork_b.ul[0] );
+ FD_TEST( rec2!=rec );
+ FD_TEST( !rec2->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, 0UL )==rec2 );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ /* Loading the program should create another cache entry */
+ fd_funk_txn_xid_t fork_c = { .ul = { 3UL, 2UL } };
+ test_env_txn_prepare( env, &fork_b, &fork_c );
+ load_env.slot = 3UL;
+ fd_progcache_rec_t const * rec3 = fd_progcache_pull( env->progcache, env->accdb, &fork_c, &key, &load_env );
+ FD_TEST( rec3 );
+ FD_TEST( rec3!=rec2 && rec3!=rec );
+ FD_TEST( rec3->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_c, &key, 0UL )==rec3 );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, 0UL )==rec2 );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_invalidate_dup: fd_progcache_invalidate should create a cache
+ invalidation entry, even if last update was an invalidation. Because
+ a future cache access could create a cache entry between the two
+ retro-actively. */
+
+static void
+test_invalidate_dup( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_bpf_loader_program_id,
+ valid_program_data,
+ valid_program_data_sz,
+ 1 );
+
+ /* Create initial cache entry */
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec = fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env );
+ FD_TEST( rec );
+ FD_TEST( rec->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ /* Create cache invalidation entry */
+ fd_funk_txn_xid_t fork_b = { .ul = { 2UL, 1UL } };
+ test_env_txn_prepare( env, &fork_a, &fork_b );
+ fd_progcache_rec_t const * rec2 = fd_progcache_invalidate( env->progcache, &fork_b, &key, fork_b.ul[0] );
+ FD_TEST( rec2!=rec );
+ FD_TEST( !rec2->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, 0UL )==rec2 );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ /* Create cache invalidation entry */
+ fd_funk_txn_xid_t fork_c = { .ul = { 3UL, 2UL } };
+ test_env_txn_prepare( env, &fork_b, &fork_c );
+ fd_progcache_rec_t const * rec3 = fd_progcache_invalidate( env->progcache, &fork_c, &key, fork_c.ul[0] );
+ FD_TEST( rec3 && rec2!=rec3 );
+ FD_TEST( !rec3->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_c, &key, 0UL )==rec3 );
+
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_invalidate_epoch_boundary: fd_progcache_invalidate after a cache
+ entry, even if the program is already invalid (due to an epoch
+ boundary). Because a future cache access could create a cache entry
+ between the two retro-actively. */
+
+static void
+test_invalidate_epoch_boundary( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_bpf_loader_program_id,
+ valid_program_data,
+ valid_program_data_sz,
+ 1 );
+
+ FD_TEST( !fd_progcache_peek( env->progcache, &fork_a, &key, 0UL ) );
+ FD_TEST( env->progcache->fork_depth==2UL );
+ FD_TEST( fd_funk_txn_xid_eq( &env->progcache->fork[ 0 ], &fork_a ) );
+ FD_TEST( fd_funk_txn_xid_eq( &env->progcache->fork[ 1 ], fd_funk_root( env->progcache->funk ) ) );
+
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec = fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env );
+ FD_TEST( rec );
+ FD_TEST( rec->executable );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec );
+
+ fd_funk_txn_xid_t fork_b = { .ul = { 64UL, 2UL } };
+ test_env_txn_prepare( env, &fork_a, &fork_b );
+ load_env.slot = 64UL;
+ load_env.epoch = 1UL;
+ load_env.epoch_slot0 = 64UL;
+ fd_progcache_rec_t const * rec2 = fd_progcache_invalidate( env->progcache, &fork_b, &key, fork_b.ul[0] );
+ FD_TEST( rec2 && rec!=rec2 );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, load_env.epoch_slot0 )==rec2 );
+
+ test_env_txn_cancel( env, &fork_b );
+ test_env_txn_cancel( env, &fork_a );
+ test_env_destroy( env );
+}
+
+/* test_publish_gc: fd_progcache_txn_publish should garbage-collect
+ stale entries. */
+
+static void
+test_publish_gc( fd_wksp_t * wksp ) {
+ test_env_t * env = test_env_create( wksp );
+ fd_funk_txn_xid_t fork_a = { .ul = { 1UL, 1UL } };
+ test_env_txn_prepare( env, NULL, &fork_a );
+
+ fd_funk_rec_key_t key = test_key( 1UL );
+ create_test_account( env, &fork_a, &key,
+ &fd_solana_bpf_loader_program_id,
+ valid_program_data,
+ valid_program_data_sz,
+ 1 );
+
+ fd_prog_load_env_t load_env = {
+ .features = env->features,
+ .slot = 1UL,
+ .epoch = 0UL,
+ .epoch_slot0 = 0UL
+ };
+ fd_progcache_rec_t const * rec_a = fd_progcache_pull( env->progcache, env->accdb, &fork_a, &key, &load_env );
+ FD_TEST( rec_a );
+ FD_TEST( rec_a->executable );
+
+ fd_funk_txn_xid_t fork_b = { .ul = { 2UL, 1UL } };
+ test_env_txn_prepare( env, &fork_a, &fork_b );
+ fd_progcache_rec_t const * rec_b = fd_progcache_invalidate( env->progcache, &fork_b, &key, fork_b.ul[0] );
+ FD_TEST( rec_b );
+
+ fd_funk_txn_xid_t fork_c = { .ul = { 3UL, 2UL } };
+ test_env_txn_prepare( env, &fork_b, &fork_c );
+ load_env.slot = 3UL;
+ fd_progcache_rec_t const * rec_c = fd_progcache_pull( env->progcache, env->accdb, &fork_c, &key, &load_env );
+ FD_TEST( rec_c );
+ FD_TEST( rec_c->executable );
+
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec_a );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_b, &key, 0UL )==rec_b );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_c, &key, 0UL )==rec_c );
+
+ fd_funk_rec_t const * frec_a = query_rec_exact( env, &fork_a, &key );
+ fd_funk_rec_t const * frec_b = query_rec_exact( env, &fork_b, &key );
+ fd_funk_rec_t const * frec_c = query_rec_exact( env, &fork_c, &key );
+ FD_TEST( frec_a ); FD_TEST( frec_b ); FD_TEST( frec_c );
+ FD_TEST( frec_a!=frec_b && frec_a!=frec_c && frec_b!=frec_c );
+
+ fd_funk_txn_xid_t root; fd_funk_txn_xid_set_root( &root );
+ test_env_txn_publish( env, &fork_a );
+ FD_TEST( query_rec_exact( env, &fork_a, &key )==NULL );
+ FD_TEST( query_rec_exact( env, &root, &key )==frec_a );
+ FD_TEST( fd_progcache_peek( env->progcache, &fork_a, &key, 0UL )==rec_a );
+
+ test_env_txn_publish( env, &fork_b );
+ FD_TEST( query_rec_exact( env, &fork_a, &key )==NULL );
+ FD_TEST( query_rec_exact( env, &fork_b, &key )==NULL );
+ FD_TEST( query_rec_exact( env, &root, &key )==frec_b );
+
+ test_env_txn_publish( env, &fork_c );
+ FD_TEST( query_rec_exact( env, &fork_a, &key )==NULL );
+ FD_TEST( query_rec_exact( env, &fork_b, &key )==NULL );
+ FD_TEST( query_rec_exact( env, &fork_c, &key )==NULL );
+ FD_TEST( query_rec_exact( env, &root, &key )==frec_c );
+
+ test_env_destroy( env );
+}
+
+static void
+test_publish_trivial( fd_wksp_t * wksp ) {
+ /* Exercise a sequence of prepare/publish operations seen when running
+ 'firedancer-dev backtest' */
+
+ test_env_t * env = test_env_create( wksp );
+
+ fd_funk_txn_xid_t root; fd_funk_txn_xid_set_root( &root );
+ fd_funk_txn_xid_t fork_368528500 = { .ul = { 368528500UL, 368528500UL } };
+ fd_progcache_txn_prepare( env->progcache_admin, &root, &fork_368528500 );
+ fd_progcache_txn_publish( env->progcache_admin, &fork_368528500 );
+
+ /* FIXME more operations here ... */
+}
+
+
+struct test_case {
+ char const * name;
+ void (* fn)( fd_wksp_t * wksp );
+};
+
+static int
+match_test_name( char const * test_name,
+ int argc,
+ char ** argv ) {
+ if( argc<=1 ) return 1;
+ for( int i=1; ifd_shmem_cpu_cnt() ) cpu_idx = 0UL;
+
+ char const * _page_sz = fd_env_strip_cmdline_cstr ( &argc, &argv, "--page-sz", NULL, "gigantic" );
+ ulong page_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--page-cnt", NULL, 2UL );
+ ulong numa_idx = fd_env_strip_cmdline_ulong( &argc, &argv, "--numa-idx", NULL, fd_shmem_numa_idx( cpu_idx ) );
+
+ ulong page_sz = fd_cstr_to_shmem_page_sz( _page_sz );
+ if( FD_UNLIKELY( !page_sz ) ) FD_LOG_ERR(( "unsupported --page-sz" ));
+
+ FD_LOG_NOTICE(( "Creating workspace (--page-cnt %lu, --page-sz %s, --numa-idx %lu)", page_cnt, _page_sz, numa_idx ));
+ fd_wksp_t * wksp = fd_wksp_new_anonymous( page_sz, page_cnt, fd_shmem_cpu_idx( numa_idx ), "wksp", 0UL );
+ FD_TEST( wksp );
+
+# define TEST( name ) { #name, name }
+ struct test_case cases[] = {
+ TEST( test_empty ),
+ TEST( test_account_does_not_exist ),
+ TEST( test_invalid_owner ),
+ TEST( test_invalid_program ),
+ TEST( test_valid_program ),
+ TEST( test_epoch_boundary ),
+ TEST( test_invalidate ),
+ TEST( test_invalidate_nonexistent ),
+ TEST( test_invalidate_pull ),
+ TEST( test_invalidate_dup ),
+ TEST( test_invalidate_epoch_boundary ),
+ TEST( test_publish_gc ),
+ TEST( test_publish_trivial ),
+ {0}
+ };
+# undef TEST
+ for( struct test_case * tc = cases; tc->name; tc++ ) {
+ if( match_test_name( tc->name, argc, argv ) ) {
+ FD_LOG_NOTICE(( "Running %s", tc->name ));
+ tc->fn( wksp );
+ }
+ }
+
+ fd_wksp_delete_anonymous( wksp );
+
+ FD_LOG_NOTICE(( "pass" ));
+ fd_halt();
+ return 0;
+}
diff --git a/src/flamenco/runtime/context/fd_exec_txn_ctx.h b/src/flamenco/runtime/context/fd_exec_txn_ctx.h
index a37182a3403..c644ae66cde 100644
--- a/src/flamenco/runtime/context/fd_exec_txn_ctx.h
+++ b/src/flamenco/runtime/context/fd_exec_txn_ctx.h
@@ -8,6 +8,7 @@
#include "../fd_txncache.h"
#include "../fd_bank_hash_cmp.h"
#include "../../../funk/fd_funk.h"
+#include "../../progcache/fd_progcache_user.h"
#include "../fd_compute_budget_details.h"
#include "../../../disco/pack/fd_microblock.h"
@@ -55,6 +56,8 @@ struct fd_exec_txn_ctx {
int enable_exec_recording;
fd_bank_hash_cmp_t * bank_hash_cmp;
fd_funk_t funk[1];
+ fd_progcache_t * progcache;
+ fd_progcache_t _progcache[1];
fd_funk_txn_xid_t xid[1];
ulong slot;
ulong bank_idx;
diff --git a/src/flamenco/runtime/fd_acc_mgr.h b/src/flamenco/runtime/fd_acc_mgr.h
index a427b6b4d34..b719186a6a9 100644
--- a/src/flamenco/runtime/fd_acc_mgr.h
+++ b/src/flamenco/runtime/fd_acc_mgr.h
@@ -92,18 +92,9 @@ FD_FN_PURE static inline fd_funk_rec_key_t
fd_funk_acc_key( fd_pubkey_t const * pubkey ) {
fd_funk_rec_key_t key = {0};
memcpy( key.uc, pubkey, sizeof(fd_pubkey_t) );
- key.uc[ FD_FUNK_REC_KEY_FOOTPRINT - 1 ] = FD_FUNK_KEY_TYPE_ACC;
return key;
}
-/* fd_funk_key_is_acc returns 1 if given fd_funk key is an account
- and 0 otherwise. */
-
-FD_FN_PURE static inline int
-fd_funk_key_is_acc( fd_funk_rec_key_t const * id ) {
- return id->uc[ FD_FUNK_REC_KEY_FOOTPRINT - 1 ] == FD_FUNK_KEY_TYPE_ACC;
-}
-
/* Account Access from Funk APIs *************************************************/
/* The following fd_funk_acc_mgr APIs translate between the runtime account DB abstraction
diff --git a/src/flamenco/runtime/fd_core_bpf_migration.c b/src/flamenco/runtime/fd_core_bpf_migration.c
index b05ef247bde..b65c0b51c13 100644
--- a/src/flamenco/runtime/fd_core_bpf_migration.c
+++ b/src/flamenco/runtime/fd_core_bpf_migration.c
@@ -424,7 +424,7 @@ fd_migrate_builtin_to_core_bpf( fd_bank_t * bank,
/* Deploy the new target Core BPF program.
https://github.com/anza-xyz/agave/blob/v2.1.0/runtime/src/bank/builtins/core_bpf_migration/mod.rs#L268-L271 */
err = fd_directly_invoke_loader_v3_deploy( bank,
- funk,
+ funk->shmem,
&migration_xid,
builtin_program_id,
fd_txn_account_get_data( new_target_program_data_account ) + PROGRAMDATA_METADATA_SIZE,
diff --git a/src/flamenco/runtime/fd_executor.c b/src/flamenco/runtime/fd_executor.c
index 7531f9bcf75..214cb90a1e1 100644
--- a/src/flamenco/runtime/fd_executor.c
+++ b/src/flamenco/runtime/fd_executor.c
@@ -20,7 +20,6 @@
#include "program/fd_vote_program.h"
#include "program/fd_zk_elgamal_proof_program.h"
#include "sysvar/fd_sysvar_cache.h"
-#include "program/fd_program_cache.h"
#include "sysvar/fd_sysvar_epoch_schedule.h"
#include "sysvar/fd_sysvar_instructions.h"
#include "sysvar/fd_sysvar_rent.h"
@@ -29,8 +28,6 @@
#include "tests/fd_dump_pb.h"
#include "../../ballet/base58/fd_base58.h"
-#include "../../disco/pack/fd_pack.h"
-#include "../../disco/pack/fd_pack_cost.h"
#include "../../util/bits/fd_uwide.h"
@@ -1369,12 +1366,24 @@ fd_executor_reclaim_account( fd_exec_txn_ctx_t * txn_ctx,
void
fd_exec_txn_ctx_setup( fd_bank_t * bank,
- fd_funk_t * funk,
+ void * accdb_shfunk,
+ void * progcache_shfunk,
fd_funk_txn_xid_t const * xid,
fd_txncache_t * status_cache,
fd_exec_txn_ctx_t * ctx,
- fd_bank_hash_cmp_t * bank_hash_cmp ) {
- ctx->funk[0] = *funk;
+ fd_bank_hash_cmp_t * bank_hash_cmp,
+ void * progcache_scratch,
+ ulong progcache_scratch_sz ) {
+ if( FD_UNLIKELY( !fd_funk_join( ctx->funk, accdb_shfunk ) ) ) {
+ FD_LOG_CRIT(( "fd_funk_join(accdb) failed" ));
+ }
+
+ if( progcache_shfunk ) {
+ ctx->progcache = fd_progcache_join( ctx->_progcache, progcache_shfunk, progcache_scratch, progcache_scratch_sz );
+ if( FD_UNLIKELY( !ctx->progcache ) ) {
+ FD_LOG_CRIT(( "fd_progcache_join() failed" ));
+ }
+ }
ctx->xid[0] = *xid;
diff --git a/src/flamenco/runtime/fd_executor.h b/src/flamenco/runtime/fd_executor.h
index cbca19a4e62..36861a8ee68 100644
--- a/src/flamenco/runtime/fd_executor.h
+++ b/src/flamenco/runtime/fd_executor.h
@@ -135,11 +135,14 @@ fd_instr_stack_pop( fd_exec_txn_ctx_t * txn_ctx,
void
fd_exec_txn_ctx_setup( fd_bank_t * bank,
- fd_funk_t * funk,
+ void * accdb_shfunk,
+ void * progcache_shfunk,
fd_funk_txn_xid_t const * xid,
fd_txncache_t * status_cache,
fd_exec_txn_ctx_t * ctx,
- fd_bank_hash_cmp_t * bank_hash_cmp );
+ fd_bank_hash_cmp_t * bank_hash_cmp,
+ void * progcache_scratch,
+ ulong progcache_scratch_sz );
FD_PROTOTYPES_END
diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c
index 41ca065eb8c..567cadf7d15 100644
--- a/src/flamenco/runtime/fd_runtime.c
+++ b/src/flamenco/runtime/fd_runtime.c
@@ -15,13 +15,13 @@
#include "../stakes/fd_stakes.h"
#include "../rewards/fd_rewards.h"
+#include "../progcache/fd_progcache_user.h"
#include "context/fd_exec_txn_ctx.h"
#include "program/fd_stake_program.h"
#include "program/fd_builtin_programs.h"
#include "program/fd_vote_program.h"
-#include "program/fd_program_cache.h"
#include "program/fd_bpf_loader_program.h"
#include "program/fd_address_lookup_table_program.h"
@@ -1086,6 +1086,7 @@ fd_runtime_save_account( fd_funk_t * funk,
void
fd_runtime_finalize_txn( fd_funk_t * funk,
+ fd_progcache_t * progcache,
fd_txncache_t * txncache,
fd_funk_txn_xid_t const * xid,
fd_exec_txn_ctx_t * txn_ctx,
@@ -1160,7 +1161,7 @@ fd_runtime_finalize_txn( fd_funk_t * funk,
ulong current_slot = fd_bank_slot_get( bank );
for( uchar i=0; iprograms_to_reverify_cnt; i++ ) {
fd_pubkey_t const * program_key = &txn_ctx->programs_to_reverify[i];
- fd_program_cache_queue_program_for_reverification( funk, xid, program_key, current_slot );
+ fd_progcache_invalidate( progcache, xid, program_key, current_slot );
}
}
@@ -1573,71 +1574,6 @@ fd_runtime_process_new_epoch( fd_banks_t * banks,
} FD_SPAD_FRAME_END;
}
-/******************************************************************************/
-/* Block Parsing */
-/******************************************************************************/
-
-/* Block iteration and parsing */
-
-/* As a note, all of the logic in this section is used by the full firedancer
- client. The store tile uses these APIs to help parse raw (micro)blocks
- received from the network. */
-
-/* Helpers */
-
-void
-fd_runtime_update_program_cache( fd_bank_t * bank,
- fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_txn_p_t const * txn_p,
- fd_spad_t * runtime_spad ) {
- fd_txn_t const * txn_descriptor = TXN( txn_p );
-
- FD_SPAD_FRAME_BEGIN( runtime_spad ) {
-
- /* Iterate over account keys referenced directly in the transaction first */
- fd_acct_addr_t const * acc_addrs = fd_txn_get_acct_addrs( txn_descriptor, txn_p );
- for( ushort acc_idx=0; acc_idxacct_addr_cnt; acc_idx++ ) {
- fd_pubkey_t const * account = fd_type_pun_const( &acc_addrs[acc_idx] );
- fd_program_cache_update_program( bank, funk, xid, account, runtime_spad );
- }
-
- if( txn_descriptor->transaction_version==FD_TXN_V0 ) {
-
- /* Iterate over account keys referenced in ALUTs */
- fd_acct_addr_t alut_accounts[256];
- fd_slot_hashes_global_t const * slot_hashes_global = fd_sysvar_slot_hashes_read( funk, xid, runtime_spad );
- if( FD_UNLIKELY( !slot_hashes_global ) ) {
- return;
- }
-
- fd_slot_hash_t * slot_hash = deq_fd_slot_hash_t_join( (uchar *)slot_hashes_global + slot_hashes_global->hashes_offset );
-
- /* TODO: This is done twice, once in the replay tile and once in the
- exec tile. We should consolidate the account resolution into a
- single place, but also keep in mind from a conformance
- perspective that these ALUT resolution checks happen after some
- things like compute budget instruction parsing */
- if( FD_UNLIKELY( fd_runtime_load_txn_address_lookup_tables(
- txn_descriptor,
- txn_p->payload,
- funk,
- xid,
- fd_bank_slot_get( bank ),
- slot_hash,
- alut_accounts ) ) ) {
- return;
- }
-
- for( ushort alut_idx=0; alut_idxaddr_table_adtl_cnt; alut_idx++ ) {
- fd_pubkey_t const * account = fd_type_pun_const( &alut_accounts[alut_idx] );
- fd_program_cache_update_program( bank, funk, xid, account, runtime_spad );
- }
- }
-
- } FD_SPAD_FRAME_END;
-}
-
/******************************************************************************/
/* Genesis */
/*******************************************************************************/
diff --git a/src/flamenco/runtime/fd_runtime.h b/src/flamenco/runtime/fd_runtime.h
index 3a5168e4f25..1d5b0c96a0d 100644
--- a/src/flamenco/runtime/fd_runtime.h
+++ b/src/flamenco/runtime/fd_runtime.h
@@ -464,6 +464,7 @@ fd_runtime_prepare_and_execute_txn( fd_banks_t * banks,
void
fd_runtime_finalize_txn( fd_funk_t * funk,
+ fd_progcache_t * progcache,
fd_txncache_t * txncache,
fd_funk_txn_xid_t const * xid,
fd_exec_txn_ctx_t * txn_ctx,
@@ -489,22 +490,6 @@ fd_runtime_block_pre_execute_process_new_epoch( fd_banks_t * banks,
fd_spad_t * runtime_spad,
int * is_epoch_boundary );
-/* `fd_runtime_update_program_cache()` is responsible for updating the
- program cache with any programs referenced in the current
- transaction. See fd_program_cache.h for more details.
-
- Note that ALUTs must be resolved because programs referenced in ALUTs
- can be invoked via CPI.
-
- TODO: We need to remove the ALUT resolution from this function
- because it is redundant (ALUTs get resolved again in the exec tile). */
-void
-fd_runtime_update_program_cache( fd_bank_t * bank,
- fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_txn_p_t const * txn_p,
- fd_spad_t * runtime_spad );
-
/* Offline Replay *************************************************************/
void
diff --git a/src/flamenco/runtime/program/Local.mk b/src/flamenco/runtime/program/Local.mk
index cc9d5c05295..a9ff544f868 100644
--- a/src/flamenco/runtime/program/Local.mk
+++ b/src/flamenco/runtime/program/Local.mk
@@ -8,9 +8,6 @@ $(call add-objs,fd_builtin_programs,fd_flamenco)
$(call add-hdrs,fd_bpf_loader_serialization.h)
$(call add-objs,fd_bpf_loader_serialization,fd_flamenco)
-$(call add-hdrs,fd_program_cache.h)
-$(call add-objs,fd_program_cache,fd_flamenco)
-
### Precompiles
$(call add-hdrs,fd_precompiles.h)
@@ -48,13 +45,4 @@ $(call add-objs,fd_zk_elgamal_proof_program,fd_flamenco)
$(call add-hdrs,fd_native_cpi.h)
$(call add-objs,fd_native_cpi,fd_flamenco)
-### Tests
-ifdef FD_HAS_HOSTED
-ifdef FD_HAS_SECP256K1
-$(call make-unit-test,test_program_cache,test_program_cache,fd_flamenco fd_ballet fd_funk fd_util)
-$(call run-unit-test,test_program_cache)
-$(call make-unit-test,test_program_cache_concur,test_program_cache_concur,fd_flamenco fd_vm fd_funk fd_ballet fd_util)
-endif
-endif
-
endif
diff --git a/src/flamenco/runtime/program/fd_bpf_loader_program.c b/src/flamenco/runtime/program/fd_bpf_loader_program.c
index 0ac25276637..241a0817739 100644
--- a/src/flamenco/runtime/program/fd_bpf_loader_program.c
+++ b/src/flamenco/runtime/program/fd_bpf_loader_program.c
@@ -2,6 +2,7 @@
/* For additional context see https://solana.com/docs/programs/deploying#state-accounts */
+#include "../../progcache/fd_prog_load.h"
#include "../fd_pubkey_utils.h"
#include "../../../ballet/sbpf/fd_sbpf_loader.h"
#include "../sysvar/fd_sysvar.h"
@@ -92,30 +93,6 @@ calculate_heap_cost( ulong heap_size, ulong heap_cost ) {
#undef KIBIBYTE_MUL_PAGES_SUB_1
}
-void
-fd_bpf_get_sbpf_versions( uint * sbpf_min_version,
- uint * sbpf_max_version,
- ulong slot,
- fd_features_t const * features ) {
- int disable_v0 = FD_FEATURE_ACTIVE( slot, features, disable_sbpf_v0_execution );
- int reenable_v0 = FD_FEATURE_ACTIVE( slot, features, reenable_sbpf_v0_execution );
- int enable_v0 = !disable_v0 || reenable_v0;
- int enable_v1 = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v1_deployment_and_execution );
- int enable_v2 = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v2_deployment_and_execution );
- int enable_v3 = FD_FEATURE_ACTIVE( slot, features, enable_sbpf_v3_deployment_and_execution );
-
- *sbpf_min_version = enable_v0 ? FD_SBPF_V0 : FD_SBPF_V3;
- if( enable_v3 ) {
- *sbpf_max_version = FD_SBPF_V3;
- } else if( enable_v2 ) {
- *sbpf_max_version = FD_SBPF_V2;
- } else if( enable_v1 ) {
- *sbpf_max_version = FD_SBPF_V1;
- } else {
- *sbpf_max_version = FD_SBPF_V0;
- }
-}
-
/* https://github.com/anza-xyz/agave/blob/574bae8fefc0ed256b55340b9d87b7689bcdf222/programs/bpf_loader/src/lib.rs#L105-L171
Our arguments to deploy_program are different from the Agave version because
@@ -160,16 +137,12 @@ fd_deploy_program( fd_exec_instr_ctx_t * instr_ctx,
/* Load executable */
fd_sbpf_elf_info_t elf_info[ 1UL ];
- uint min_sbpf_version, max_sbpf_version;
- fd_bpf_get_sbpf_versions( &min_sbpf_version,
- &max_sbpf_version,
- instr_ctx->txn_ctx->slot,
- &instr_ctx->txn_ctx->features );
+ fd_prog_versions_t versions = fd_prog_versions( &instr_ctx->txn_ctx->features, instr_ctx->txn_ctx->slot );
fd_sbpf_loader_config_t config = { 0 };
config.elf_deploy_checks = deploy_mode;
- config.sbpf_min_version = min_sbpf_version;
- config.sbpf_max_version = max_sbpf_version;
+ config.sbpf_min_version = versions.min_sbpf_version;
+ config.sbpf_max_version = versions.max_sbpf_version;
if( FD_UNLIKELY( fd_sbpf_elf_peek( elf_info, programdata, programdata_size, &config )<0 ) ) {
//TODO: actual log, this is a custom Firedancer msg
@@ -395,9 +368,9 @@ common_close_account( fd_pubkey_t * authority_address,
https://github.com/anza-xyz/agave/blob/574bae8fefc0ed256b55340b9d87b7689bcdf222/programs/bpf_loader/src/lib.rs#L1332-L1501 */
int
-fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx,
- fd_program_cache_entry_t const * cache_entry,
- uchar is_deprecated ) {
+fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx,
+ fd_progcache_rec_t const * cache_entry,
+ uchar is_deprecated ) {
int err = FD_EXECUTOR_INSTR_SUCCESS;
fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_new( fd_spad_alloc( instr_ctx->txn_ctx->spad,
@@ -414,12 +387,12 @@ fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx,
0 );
/* https://github.com/anza-xyz/agave/blob/574bae8fefc0ed256b55340b9d87b7689bcdf222/programs/bpf_loader/src/lib.rs#L1362-L1368 */
- ulong input_sz = 0UL;
- ulong pre_lens[256] = {0};
- fd_vm_input_region_t input_mem_regions[1000] = {0}; /* We can have a max of (3 * num accounts + 1) regions */
- fd_vm_acc_region_meta_t acc_region_metas[256] = {0}; /* instr acc idx to idx */
- uint input_mem_regions_cnt = 0U;
- int direct_mapping = FD_FEATURE_ACTIVE_BANK( instr_ctx->txn_ctx->bank, bpf_account_data_direct_mapping );
+ ulong input_sz = 0UL;
+ ulong pre_lens[256] = {0};
+ fd_vm_input_region_t input_mem_regions[1000] = {0}; /* We can have a max of (3 * num accounts + 1) regions */
+ fd_vm_acc_region_meta_t acc_region_metas[256] = {0}; /* instr acc idx to idx */
+ uint input_mem_regions_cnt = 0U;
+ int direct_mapping = FD_FEATURE_ACTIVE_BANK( instr_ctx->txn_ctx->bank, bpf_account_data_direct_mapping );
uchar * input = NULL;
err = fd_bpf_loader_input_serialize_parameters( instr_ctx, &input_sz, pre_lens,
@@ -461,14 +434,14 @@ fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx,
/* instr_ctx */ instr_ctx,
/* heap_max */ heap_size,
/* entry_cu */ instr_ctx->txn_ctx->compute_budget_details.compute_meter,
- /* rodata */ fd_program_cache_get_rodata( cache_entry ),
+ /* rodata */ fd_progcache_rec_rodata( cache_entry ),
/* rodata_sz */ cache_entry->rodata_sz,
- /* text */ (ulong *)((ulong)fd_program_cache_get_rodata( cache_entry ) + (ulong)cache_entry->text_off), /* Note: text_off is byte offset */
+ /* text */ (ulong *)((ulong)fd_progcache_rec_rodata( cache_entry ) + (ulong)cache_entry->text_off), /* Note: text_off is byte offset */
/* text_cnt */ cache_entry->text_cnt,
/* text_off */ cache_entry->text_off,
/* text_sz */ cache_entry->text_sz,
/* entry_pc */ cache_entry->entry_pc,
- /* calldests */ fd_program_cache_get_calldests( cache_entry ),
+ /* calldests */ fd_progcache_rec_calldests( cache_entry ),
/* sbpf_version */ cache_entry->sbpf_version,
/* syscalls */ syscalls,
/* trace */ NULL,
@@ -2578,29 +2551,14 @@ fd_bpf_loader_program_execute( fd_exec_instr_ctx_t * ctx ) {
}
}
- /* Sadly, we have to tie the cache in with consensus. We tried our
- best to avoid this, but Agave's program loading logic is too
- complex to solely rely on checks without significant redundancy.
-
- For example, devnet and testnet have older programs that were
- deployed before stricter ELF / VM validation checks were put in
- place, causing these older programs to fail newer validation
- checks and be unexecutable. At the instruction level, we have no
- way of checking if this validation passed or not here without
- querying our program cache, otherwise we would have to copy-paste
- all of our validation checks here.
-
- Any failures here would indicate an attempt to interact with a
- deployed programs that either failed to load or failed bytecode
- verification. This applies for v1, v2, and v3 programs. This
- could also theoretically cause some currently-deployed programs
- to fail in the future if ELF / VM checks are eventually made
- stricter. */
- fd_program_cache_entry_t const * cache_entry = NULL;
- if( FD_UNLIKELY( fd_program_cache_load_entry( ctx->txn_ctx->funk,
- ctx->txn_ctx->xid,
- program_id,
- &cache_entry )!=0 ) ) {
+ fd_prog_load_env_t load_env[1]; fd_prog_load_env_from_bank( load_env, ctx->txn_ctx->bank );
+ fd_progcache_rec_t const * cache_entry =
+ fd_progcache_pull( ctx->txn_ctx->progcache,
+ ctx->txn_ctx->funk,
+ ctx->txn_ctx->xid,
+ program_id,
+ load_env );
+ if( FD_UNLIKELY( !cache_entry ) ) {
fd_log_collector_msg_literal( ctx, "Program is not cached" );
/* https://github.com/anza-xyz/agave/blob/89872fdb074e6658646b2b57a299984f0059cc84/programs/bpf_loader/src/lib.rs#L460-L467 */
@@ -2611,7 +2569,7 @@ fd_bpf_loader_program_execute( fd_exec_instr_ctx_t * ctx ) {
}
/* The program may be in the cache but could have failed verification in the current epoch. */
- if( FD_UNLIKELY( cache_entry->failed_verification ) ) {
+ if( FD_UNLIKELY( cache_entry->executable==0 ) ) {
fd_log_collector_msg_literal( ctx, "Program is not deployed" );
if( FD_FEATURE_ACTIVE_BANK( ctx->txn_ctx->bank, remove_accounts_executable_flag_checks ) ) {
return FD_EXECUTOR_INSTR_ERR_UNSUPPORTED_PROGRAM_ID;
@@ -2631,7 +2589,7 @@ fd_bpf_loader_program_execute( fd_exec_instr_ctx_t * ctx ) {
int
fd_directly_invoke_loader_v3_deploy( fd_bank_t * bank,
- fd_funk_t * funk,
+ void * accdb_shfunk,
fd_funk_txn_xid_t const * xid,
fd_pubkey_t const * program_key,
uchar const * elf,
@@ -2641,11 +2599,14 @@ fd_directly_invoke_loader_v3_deploy( fd_bank_t * bank,
fd_exec_txn_ctx_t * txn_ctx = fd_exec_txn_ctx_join( fd_exec_txn_ctx_new( fd_spad_alloc( runtime_spad, FD_EXEC_TXN_CTX_ALIGN, FD_EXEC_TXN_CTX_FOOTPRINT ) ), runtime_spad, fd_wksp_containing( runtime_spad ) );
fd_exec_txn_ctx_setup( bank,
- funk,
+ accdb_shfunk,
+ NULL,
xid,
NULL,
txn_ctx,
- NULL );
+ NULL,
+ NULL,
+ 0UL );
fd_exec_txn_ctx_setup_basic( txn_ctx );
txn_ctx->instr_stack_sz = 1;
diff --git a/src/flamenco/runtime/program/fd_bpf_loader_program.h b/src/flamenco/runtime/program/fd_bpf_loader_program.h
index c79ea328cfb..547921df0de 100644
--- a/src/flamenco/runtime/program/fd_bpf_loader_program.h
+++ b/src/flamenco/runtime/program/fd_bpf_loader_program.h
@@ -6,9 +6,10 @@
Address: BPFLoaderUpgradeab1e11111111111111111111111 */
-#include "fd_program_cache.h"
+#include "../../progcache/fd_progcache_rec.h"
#include "../../features/fd_features.h"
#include "../../types/fd_types.h"
+#include "../../../funk/fd_funk_base.h"
/* https://github.com/anza-xyz/agave/blob/77daab497df191ef485a7ad36ed291c1874596e5/programs/bpf_loader/src/lib.rs#L67-L69 */
#define DEFAULT_LOADER_COMPUTE_UNITS (570UL )
@@ -65,12 +66,6 @@ fd_bpf_loader_program_get_state( fd_txn_account_t const * acct,
fd_spad_t * spad,
int * opt_err );
-void
-fd_bpf_get_sbpf_versions( uint * sbpf_min_version,
- uint * sbpf_max_version,
- ulong slot,
- fd_features_t const * features );
-
int
fd_deploy_program( fd_exec_instr_ctx_t * instr_ctx,
fd_pubkey_t const * program_key,
@@ -79,9 +74,9 @@ fd_deploy_program( fd_exec_instr_ctx_t * instr_ctx,
fd_spad_t * spad );
int
-fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx,
- fd_program_cache_entry_t const * cache_entry,
- uchar is_deprecated );
+fd_bpf_execute( fd_exec_instr_ctx_t * instr_ctx,
+ fd_progcache_rec_t const * program,
+ uchar is_deprecated );
int
fd_bpf_loader_program_execute( fd_exec_instr_ctx_t * instr_ctx );
@@ -98,7 +93,7 @@ fd_bpf_loader_program_execute( fd_exec_instr_ctx_t * instr_ctx );
https://github.com/anza-xyz/agave/blob/v2.1.0/runtime/src/bank/builtins/core_bpf_migration/mod.rs#L155-L233 */
int
fd_directly_invoke_loader_v3_deploy( fd_bank_t * bank,
- fd_funk_t * funk,
+ void * accdb_shfunk,
fd_funk_txn_xid_t const * xid,
fd_pubkey_t const * program_key,
uchar const * elf,
diff --git a/src/flamenco/runtime/program/fd_loader_v4_program.c b/src/flamenco/runtime/program/fd_loader_v4_program.c
index 3a394a73c92..3d874c90fbd 100644
--- a/src/flamenco/runtime/program/fd_loader_v4_program.c
+++ b/src/flamenco/runtime/program/fd_loader_v4_program.c
@@ -861,34 +861,16 @@ fd_loader_v4_program_execute( fd_exec_instr_ctx_t * instr_ctx ) {
return rc;
}
- /* See note in `fd_bpf_loader_program_execute()` as to why we must tie the cache into consensus :(
- https://github.com/anza-xyz/agave/blob/v2.2.6/programs/loader-v4/src/lib.rs#L522-L528 */
- fd_program_cache_entry_t const * cache_entry = NULL;
- if( FD_UNLIKELY( fd_program_cache_load_entry( instr_ctx->txn_ctx->funk,
- instr_ctx->txn_ctx->xid,
- program_id,
- &cache_entry )!=0 ) ) {
- fd_log_collector_msg_literal( instr_ctx, "Program is not cached" );
- return FD_EXECUTOR_INSTR_ERR_UNSUPPORTED_PROGRAM_ID;
- }
+ /* Work around differences in program caching behavior between
+ Fireadncer and Agave here.
- /* The program may be in the cache but could have failed verification in the current epoch. */
- if( FD_UNLIKELY( cache_entry->failed_verification ) ) {
- fd_log_collector_msg_literal( instr_ctx, "Program is not deployed" );
- return FD_EXECUTOR_INSTR_ERR_UNSUPPORTED_PROGRAM_ID;
- }
+ Agave includes load failures due to program metadata (e.g.
+ "DelayVisibility", "Closed") in their cache. Firedancer
+ does not create cache entries for these states, instead only
+ including load results ("FailedVerification", "Loaded").
- /* After the program is deployed, we wait a slot before adding it to our program cache. Agave, on the other hand,
- updates their program cache after every transaction. Because of this, for a program that was deployed in the
- current slot, Agave would log "Program is not deployed", while we would log "Program is not cached" since
- the program is not in the cache yet. The same thing holds for very old programs that fail ELF / VM validation
- checks and are thus non-invokable - if this program was invoked, Agave would keep it in their program cache and label
- it as "FailedVerification", while we would not include it at all.
-
- Because of the difference in our caching behavior, we need to perform checks that will filter out every single program
- from execution that Agave would. In Agave's `load_program_accounts()` function, they filter any retracted programs ("Closed")
- and mark any programs deployed in the current slot as "DelayedVisibility". Any programs that fail verification will also
- not be in the cache anyways. */
+ Therefore, Firedancer recovers the DelayVisibility and Closed
+ states on-the-fly before querying cahce. */
fd_loader_v4_state_t const * state = fd_loader_v4_get_state( program.acct, &rc );
if( FD_UNLIKELY( rc ) ) {
@@ -907,6 +889,25 @@ fd_loader_v4_program_execute( fd_exec_instr_ctx_t * instr_ctx ) {
return FD_EXECUTOR_INSTR_ERR_UNSUPPORTED_PROGRAM_ID;
}
+ /* https://github.com/anza-xyz/agave/blob/v2.2.6/programs/loader-v4/src/lib.rs#L522-L528 */
+ fd_prog_load_env_t load_env[1]; fd_prog_load_env_from_bank( load_env, instr_ctx->txn_ctx->bank );
+ fd_progcache_rec_t const * cache_entry =
+ fd_progcache_pull( instr_ctx->txn_ctx->progcache,
+ instr_ctx->txn_ctx->funk,
+ instr_ctx->txn_ctx->xid,
+ program_id,
+ load_env );
+ if( FD_UNLIKELY( !cache_entry ) ) {
+ fd_log_collector_msg_literal( instr_ctx, "Program is not cached" );
+ return FD_EXECUTOR_INSTR_ERR_UNSUPPORTED_PROGRAM_ID;
+ }
+
+ /* The program may be in the cache but could have failed verification in the current epoch. */
+ if( FD_UNLIKELY( cache_entry->executable==0 ) ) {
+ fd_log_collector_msg_literal( instr_ctx, "Program is not deployed" );
+ return FD_EXECUTOR_INSTR_ERR_UNSUPPORTED_PROGRAM_ID;
+ }
+
/* https://github.com/anza-xyz/agave/blob/v2.2.6/programs/loader-v4/src/lib.rs#L531 */
fd_borrowed_account_drop( &program );
diff --git a/src/flamenco/runtime/program/fd_program_cache.c b/src/flamenco/runtime/program/fd_program_cache.c
deleted file mode 100644
index aa2374b431e..00000000000
--- a/src/flamenco/runtime/program/fd_program_cache.c
+++ /dev/null
@@ -1,686 +0,0 @@
-#include "fd_program_cache.h"
-#include "fd_bpf_loader_program.h"
-#include "fd_loader_v4_program.h"
-#include "../../vm/fd_vm.h"
-#include "../sysvar/fd_sysvar_epoch_schedule.h"
-#include "../../../funk/fd_funk_rec.h"
-
-#include
-
-fd_program_cache_entry_t *
-fd_program_cache_entry_new( void * mem,
- fd_sbpf_elf_info_t const * elf_info,
- ulong last_slot_modified,
- ulong last_slot_verified ) {
- fd_program_cache_entry_t * cache_entry = (fd_program_cache_entry_t *)mem;
- cache_entry->magic = FD_PROGRAM_CACHE_ENTRY_MAGIC;
-
- /* Failed verification flag */
- cache_entry->failed_verification = 0;
-
- /* Last slot the program was modified */
- cache_entry->last_slot_modified = last_slot_modified;
-
- /* Last slot verification checks were ran for this program */
- cache_entry->last_slot_verified = last_slot_verified;
-
- ulong l = FD_LAYOUT_INIT;
-
- /* calldests backing memory */
- l = FD_LAYOUT_APPEND( l, alignof(fd_program_cache_entry_t), sizeof(fd_program_cache_entry_t) );
- cache_entry->calldests_shmem_off = l;
-
- /* rodata backing memory */
- l = FD_LAYOUT_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( elf_info->text_cnt ) );
- cache_entry->rodata_off = l;
-
- /* SBPF version */
- cache_entry->sbpf_version = elf_info->sbpf_version;
-
- return cache_entry;
-}
-
-/* Sets up defined fields for a record that failed verification. */
-static void
-fd_program_cache_entry_set_failed_verification( void * mem,
- ulong last_slot_verified ) {
- fd_program_cache_entry_t * cache_entry = (fd_program_cache_entry_t *)mem;
- cache_entry->magic = FD_PROGRAM_CACHE_ENTRY_MAGIC;
-
- /* Failed verification flag */
- cache_entry->failed_verification = 1;
-
- /* Last slot the program was modified */
- cache_entry->last_slot_modified = 0UL;
-
- /* Last slot verification checks were ran for this program */
- cache_entry->last_slot_verified = last_slot_verified;
-
- /* All other fields are undefined. */
-}
-
-ulong
-fd_program_cache_entry_footprint( fd_sbpf_elf_info_t const * elf_info ) {
- ulong l = FD_LAYOUT_INIT;
- l = FD_LAYOUT_APPEND( l, alignof(fd_program_cache_entry_t), sizeof(fd_program_cache_entry_t) );
- l = FD_LAYOUT_APPEND( l, fd_sbpf_calldests_align(), fd_sbpf_calldests_footprint( elf_info->text_cnt ) );
- l = FD_LAYOUT_APPEND( l, 8UL, elf_info->bin_sz );
- l = FD_LAYOUT_FINI( l, alignof(fd_program_cache_entry_t) );
- return l;
-}
-
-/* Returns the footprint of a record that failed verification. Note that
- all other fields are undefined besides the `failed_verification`,
- `last_slot_verified`, and `last_slot_modified` fields, so we can
- just return the core struct's size. */
-static inline FD_FN_PURE ulong
-fd_program_cache_failed_verification_entry_footprint( void ) {
- ulong l = FD_LAYOUT_INIT;
- l = FD_LAYOUT_APPEND( l, alignof(fd_program_cache_entry_t), sizeof(fd_program_cache_entry_t) );
- l = FD_LAYOUT_FINI( l, alignof(fd_program_cache_entry_t) );
- return l;
-}
-
-fd_funk_rec_key_t
-fd_program_cache_key( fd_pubkey_t const * pubkey ) {
- fd_funk_rec_key_t id;
- memcpy( id.uc, pubkey, sizeof(fd_pubkey_t) );
- memset( id.uc + sizeof(fd_pubkey_t), 0, sizeof(fd_funk_rec_key_t) - sizeof(fd_pubkey_t) );
-
- id.uc[ FD_FUNK_REC_KEY_FOOTPRINT - 1 ] = FD_FUNK_KEY_TYPE_ELF_CACHE;
-
- return id;
-}
-
-/* Parse ELF info from programdata. */
-static int
-fd_program_cache_parse_elf_info( fd_bank_t * bank,
- fd_sbpf_elf_info_t * elf_info,
- uchar const * program_data,
- ulong program_data_len ) {
- uint min_sbpf_version, max_sbpf_version;
- fd_bpf_get_sbpf_versions( &min_sbpf_version,
- &max_sbpf_version,
- fd_bank_slot_get( bank ),
- fd_bank_features_query( bank ) );
-
- fd_sbpf_loader_config_t config = { 0 };
- config.elf_deploy_checks = 0;
- config.sbpf_min_version = min_sbpf_version;
- config.sbpf_max_version = max_sbpf_version;
-
- if( FD_UNLIKELY( fd_sbpf_elf_peek( elf_info, program_data, program_data_len, &config )<0 ) ) {
- FD_LOG_DEBUG(( "fd_sbpf_elf_peek() failed" ));
- return -1;
- }
- return 0;
-}
-
-/* Similar to the below function, but gets the executable program content for the v4 loader.
- Unlike the v3 loader, the programdata is stored in a single program account. The program must
- NOT be retracted to be added to the cache. Returns a pointer to the programdata on success,
- and NULL on failure.
-
- Reasons for failure include:
- - The program state cannot be read from the account data or is in the `retracted` state. */
-static uchar const *
-fd_get_executable_program_content_for_v4_loader( fd_txn_account_t const * program_acc,
- ulong * program_data_len ) {
- int err;
-
- /* Get the current loader v4 state. This implicitly also checks the dlen. */
- fd_loader_v4_state_t const * state = fd_loader_v4_get_state( program_acc, &err );
- if( FD_UNLIKELY( err ) ) {
- return NULL;
- }
-
- /* The program must be deployed or finalized. */
- if( FD_UNLIKELY( fd_loader_v4_status_is_retracted( state ) ) ) {
- return NULL;
- }
-
- /* This subtraction is safe because get_state() implicitly checks the
- dlen. */
- *program_data_len = fd_txn_account_get_data_len( program_acc )-LOADER_V4_PROGRAM_DATA_OFFSET;
- return fd_txn_account_get_data( program_acc )+LOADER_V4_PROGRAM_DATA_OFFSET;
-}
-
-/* Gets the programdata for a v3 loader-owned account by decoding the account data
- as well as the programdata account. Returns a pointer to the programdata on success,
- and NULL on failure.
-
- Reasons for failure include:
- - The program account data cannot be decoded or is not in the `program` state.
- - The programdata account is not large enough to hold at least `PROGRAMDATA_METADATA_SIZE` bytes. */
-static uchar const *
-fd_get_executable_program_content_for_upgradeable_loader( fd_funk_t const * funk,
- fd_funk_txn_xid_t const * xid,
- fd_txn_account_t const * program_acc,
- ulong * program_data_len ) {
- fd_bpf_upgradeable_loader_state_t program_account_state[1];
- if( FD_UNLIKELY( !fd_bincode_decode_static(
- bpf_upgradeable_loader_state,
- program_account_state,
- fd_txn_account_get_data( program_acc ),
- fd_txn_account_get_data_len( program_acc ),
- NULL ) ) ) {
- return NULL;
- }
- if( !fd_bpf_upgradeable_loader_state_is_program( program_account_state ) ) {
- return NULL;
- }
-
- fd_pubkey_t * programdata_address = &program_account_state->inner.program.programdata_address;
- fd_txn_account_t programdata_acc[1];
- if( fd_txn_account_init_from_funk_readonly( programdata_acc, programdata_address, funk, xid )!=FD_ACC_MGR_SUCCESS ) {
- return NULL;
- }
-
- /* We don't actually need to decode here, just make sure that the account
- can be decoded successfully. */
- fd_bincode_decode_ctx_t ctx_programdata = {
- .data = fd_txn_account_get_data( programdata_acc ),
- .dataend = fd_txn_account_get_data( programdata_acc ) + fd_txn_account_get_data_len( programdata_acc ),
- };
-
- ulong total_sz = 0UL;
- if( FD_UNLIKELY( fd_bpf_upgradeable_loader_state_decode_footprint( &ctx_programdata, &total_sz ) ) ) {
- return NULL;
- }
-
- if( FD_UNLIKELY( fd_txn_account_get_data_len( programdata_acc )failed_verification = 1;
- return -1;
- }
-
- /* Allocate syscalls */
-
- void * syscalls_mem = fd_spad_alloc_check( runtime_spad, fd_sbpf_syscalls_align(), fd_sbpf_syscalls_footprint() );
- fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_join( fd_sbpf_syscalls_new( syscalls_mem ) );
- if( FD_UNLIKELY( !syscalls ) ) {
- FD_LOG_CRIT(( "Call to fd_sbpf_syscalls_new() failed" ));
- }
-
- fd_vm_syscall_register_slot( syscalls,
- fd_bank_slot_get( bank ),
- fd_bank_features_query( bank ),
- 0 );
-
- /* Load program. */
-
- fd_sbpf_loader_config_t config = { 0 };
- void * scratch = fd_spad_alloc( runtime_spad, 1UL, program_data_len );
- if( FD_UNLIKELY( 0!=fd_sbpf_program_load( prog, program_data, program_data_len, syscalls, &config, scratch, program_data_len ) ) ) {
- FD_LOG_DEBUG(( "fd_sbpf_program_load() failed" ));
- cache_entry->failed_verification = 1;
- fd_sbpf_syscalls_leave( syscalls );
- return -1;
- }
-
- /* Validate the program. */
-
- fd_vm_t _vm[ 1UL ];
- fd_vm_t * vm = fd_vm_join( fd_vm_new( _vm ) );
- if( FD_UNLIKELY( !vm ) ) {
- FD_LOG_CRIT(( "fd_vm_new() or fd_vm_join() failed" ));
- }
-
- int direct_mapping = FD_FEATURE_ACTIVE( fd_bank_slot_get( bank ), fd_bank_features_query( bank ), bpf_account_data_direct_mapping );
-
- vm = fd_vm_init( vm,
- NULL, /* OK since unused in `fd_vm_validate()` */
- 0UL,
- 0UL,
- prog->rodata,
- prog->rodata_sz,
- prog->text,
- prog->info.text_cnt,
- prog->info.text_off,
- prog->info.text_sz,
- prog->entry_pc,
- prog->calldests,
- elf_info->sbpf_version,
- syscalls,
- NULL,
- NULL,
- NULL,
- 0U,
- NULL,
- 0,
- direct_mapping,
- 0 );
-
- if( FD_UNLIKELY( !vm ) ) {
- FD_LOG_CRIT(( "fd_vm_init() failed" ));
- }
-
- int res = fd_vm_validate( vm );
- fd_sbpf_syscalls_leave( syscalls );
- if( FD_UNLIKELY( res ) ) {
- FD_LOG_DEBUG(( "fd_vm_validate() failed" ));
- cache_entry->failed_verification = 1;
- return -1;
- }
-
- /* FIXME: Super expensive memcpy. */
- if( FD_LIKELY( prog->calldests_shmem ) ) {
- fd_memcpy( fd_program_cache_get_calldests_shmem( cache_entry ), prog->calldests_shmem, fd_sbpf_calldests_footprint( prog->info.text_cnt ) );
- } else {
- fd_memset( fd_program_cache_get_calldests_shmem( cache_entry ), 0, fd_sbpf_calldests_footprint( prog->info.text_cnt ) );
- }
-
- cache_entry->entry_pc = prog->entry_pc;
- cache_entry->text_off = prog->info.text_off;
- cache_entry->text_cnt = prog->info.text_cnt;
- cache_entry->text_sz = prog->info.text_sz;
- cache_entry->rodata_sz = prog->rodata_sz;
- cache_entry->failed_verification = 0;
-
- return 0;
-}
-
-/* Publishes an in-prepare funk record for a program that failed
- verification. Creates a default sBPF validated program with the
- `failed_verification` flag set to 1 and `last_slot_verified` set to
- the current slot. The passed-in funk record is expected to be in a
- prepare. */
-static void
-fd_program_cache_publish_failed_verification_rec( fd_funk_t * funk,
- fd_funk_rec_prepare_t * prepare,
- fd_funk_rec_t * rec,
- ulong current_slot ) {
- /* Truncate the record to have a minimal footprint */
- ulong record_sz = fd_program_cache_failed_verification_entry_footprint();
- void * data = fd_funk_val_truncate( rec, fd_funk_alloc( funk ), fd_funk_wksp( funk ), 0UL, record_sz, NULL );
- if( FD_UNLIKELY( data==NULL ) ) {
- FD_LOG_ERR(( "fd_funk_val_truncate() failed to truncate record to size %lu", record_sz ));
- }
-
- fd_program_cache_entry_set_failed_verification( data, current_slot );
- fd_funk_rec_publish( funk, prepare );
-}
-
-/* Validates an SBPF program and adds it to the program cache.
- Reasons for verification failure include:
- - The ELF info cannot be parsed or validated from the programdata.
- - The sBPF program fails to be validated.
-
- The program will still be added to the cache even if verifications
- fail. This is to prevent a DOS vector where an attacker could spam
- invocations to programs that failed verification. */
-static void
-fd_program_cache_create_cache_entry( fd_bank_t * bank,
- fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_txn_account_t const * program_acc,
- fd_spad_t * runtime_spad ) {
- FD_SPAD_FRAME_BEGIN( runtime_spad ) {
- ulong current_slot = fd_bank_slot_get( bank );
-
- /* Prepare the funk record for the program cache. */
- fd_pubkey_t const * program_pubkey = program_acc->pubkey;
- fd_funk_rec_key_t id = fd_program_cache_key( program_pubkey );
-
- /* Try to get the programdata for the account. If it doesn't exist,
- simply return without publishing anything. The program could have
- been closed, but we do not want to touch the cache in this case. */
- ulong program_data_len = 0UL;
- uchar const * program_data = fd_program_cache_get_account_programdata( funk, xid, program_acc, &program_data_len );
-
- /* This prepare should never fail. */
- int funk_err = FD_FUNK_SUCCESS;
- fd_funk_rec_prepare_t prepare[1];
- fd_funk_rec_t * rec = fd_funk_rec_prepare( funk, xid, &id, prepare, &funk_err );
- if( rec == NULL || funk_err != FD_FUNK_SUCCESS ) {
- FD_LOG_CRIT(( "fd_funk_rec_prepare() failed: %i-%s", funk_err, fd_funk_strerror( funk_err ) ));
- }
-
- /* In Agave's load_program_with_pubkey(), if program data cannot be
- obtained, a tombstone cache entry of type Closed or
- FailedVerification is created. For correctness, we could just
- not insert a cache entry when there is no valid program data.
- Nonetheless, for purely conformance on instruction error log
- messages reasons, specifically "Program is not deployed" vs
- "Program is not cached", we would like to have a cache entry
- precisely when Agave does, such that we match Agave exactly on
- this error log. So, we insert a cache entry here. */
- if( FD_UNLIKELY( program_data==NULL ) ) {
- fd_program_cache_publish_failed_verification_rec( funk, prepare, rec, current_slot );
- return;
- }
-
- fd_sbpf_elf_info_t elf_info = {0};
- if( FD_UNLIKELY( fd_program_cache_parse_elf_info( bank, &elf_info, program_data, program_data_len ) ) ) {
- fd_program_cache_publish_failed_verification_rec( funk, prepare, rec, current_slot );
- return;
- }
-
- ulong val_sz = fd_program_cache_entry_footprint( &elf_info );
- void * val = fd_funk_val_truncate(
- rec,
- fd_funk_alloc( funk ),
- fd_funk_wksp( funk ),
- 0UL,
- val_sz,
- &funk_err );
- if( FD_UNLIKELY( funk_err ) ) {
- FD_LOG_ERR(( "fd_funk_val_truncate(sz=%lu) for account failed (%i-%s)", val_sz, funk_err, fd_funk_strerror( funk_err ) ));
- }
-
- /* Note that the cache entry points to the funk record data
- and writes into the record directly to avoid an expensive memcpy.
- Since this record is fresh, we should set the last slot modified
- to 0. */
- fd_program_cache_entry_t * cache_entry = fd_program_cache_entry_new( val, &elf_info, 0UL, current_slot );
- int res = fd_program_cache_validate_sbpf_program( bank, &elf_info, program_data, program_data_len, runtime_spad, cache_entry );
- if( FD_UNLIKELY( res ) ) {
- fd_program_cache_publish_failed_verification_rec( funk, prepare, rec, current_slot );
- return;
- }
-
- fd_funk_rec_publish( funk, prepare );
- } FD_SPAD_FRAME_END;
-}
-
-int
-fd_program_cache_load_entry( fd_funk_t const * funk,
- fd_funk_txn_xid_t const * xid,
- fd_pubkey_t const * program_pubkey,
- fd_program_cache_entry_t const ** cache_entry ) {
- fd_funk_rec_key_t id = fd_program_cache_key( program_pubkey );
-
- for(;;) {
- fd_funk_rec_query_t query[1];
- fd_funk_rec_t const * rec = fd_funk_rec_query_try_global(funk, xid, &id, NULL, query);
-
- if( FD_UNLIKELY( !rec ) ) {
- /* If rec is NULL, we shouldn't inspect query below because it
- would contain uninitialized fields. */
- return -1;
- }
-
- void const * data = fd_funk_val_const( rec, fd_funk_wksp(funk) );
-
- *cache_entry = (fd_program_cache_entry_t const *)data;
-
- /* This test is actually too early. It should happen after the
- data is actually consumed.
-
- TODO: this is likely fine because nothing else is modifying the
- program cache records at the same time. */
- if( FD_LIKELY( fd_funk_rec_query_test( query ) == FD_FUNK_SUCCESS ) ) {
- if( FD_UNLIKELY( (*cache_entry)->magic != FD_PROGRAM_CACHE_ENTRY_MAGIC ) ) FD_LOG_ERR(( "invalid magic" ));
- return 0;
- }
-
- /* Try again */
- }
-}
-
-void
-fd_program_cache_update_program( fd_bank_t * bank,
- fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_pubkey_t const * program_key,
- fd_spad_t * runtime_spad ) {
-FD_SPAD_FRAME_BEGIN( runtime_spad ) {
- fd_txn_account_t exec_rec[1];
- fd_funk_rec_key_t id = fd_program_cache_key( program_key );
-
- /* No need to touch the cache if the account no longer exists. */
- if( FD_UNLIKELY( fd_txn_account_init_from_funk_readonly( exec_rec,
- program_key,
- funk,
- xid ) ) ) {
- return;
- }
-
- /* The account owner must be a BPF loader to even be considered. */
- if( FD_UNLIKELY( !fd_executor_pubkey_is_bpf_loader( fd_txn_account_get_owner( exec_rec ) ) ) ) {
- return;
- }
-
- /* If the program is not present in the cache yet, then we should run
- verifications and add it to the cache.
- `fd_program_cache_create_cache_entry()` will insert the program
- into the cache and update the entry's flags accordingly if it fails
- verification. */
- fd_program_cache_entry_t const * existing_entry = NULL;
- int err = fd_program_cache_load_entry( funk, xid, program_key, &existing_entry );
- if( FD_UNLIKELY( err ) ) {
- fd_program_cache_create_cache_entry( bank, funk, xid, exec_rec, runtime_spad );
- return;
- }
-
- /* At this point, the program exists in the cache but we may need to
- reverify it and update the program cache. A program does NOT
- require reverification if both of the following conditions are met:
- - The program was already reverified in the current epoch
- - EITHER of the following are met:
- - The program was not modified in a recent slot (i.e. the program
- was already reverified since the last time it was modified)
- - The program was already verified in the current slot
-
- We must reverify the program for the current epoch if it has not
- been reverified yet. Additionally, we cannot break the invariant
- that a program cache entry will get reverified more than once
- per slot, otherwise the replay and exec tiles may race. */
- fd_epoch_schedule_t const * epoch_schedule = fd_bank_epoch_schedule_query( bank );
- ulong current_slot = fd_bank_slot_get( bank );
- ulong current_epoch = fd_bank_epoch_get( bank );
- ulong last_epoch_verified = fd_slot_to_epoch( epoch_schedule, existing_entry->last_slot_verified, NULL );
- ulong last_slot_modified = existing_entry->last_slot_modified;
- if( FD_LIKELY( last_epoch_verified==current_epoch &&
- ( last_slot_modifiedlast_slot_verified ||
- current_slot==existing_entry->last_slot_verified ) ) ) {
- return;
- }
-
- /* Get the program data from the account. If the programdata does not
- exist, there is no need to touch the cache - if someone tries to
- invoke this program, account loading checks will fail. */
- ulong program_data_len = 0UL;
- uchar const * program_data = fd_program_cache_get_account_programdata( funk,
- xid,
- exec_rec,
- &program_data_len );
- if( FD_UNLIKELY( program_data==NULL ) ) {
- /* Unlike in fd_program_cache_create_cache_entry(), where we need to
- insert an entry into the cache when we cannot obtain valid
- program data, here we could simply return because we know, at
- this point, that the program is already in the cache. So we
- don't need to do anything extra for matching Agave on cache entry
- presence. */
- return;
- }
-
- /* From here on out, we need to reverify the program. */
-
- /* Parse out the ELF info so we can determine the record footprint.
- If the parsing fails, then we need to make sure to later publish
- a failed verification record. */
- uchar failed_elf_parsing = 0;
- ulong record_sz = 0UL;
- fd_sbpf_elf_info_t elf_info = {0};
- if( FD_UNLIKELY( fd_program_cache_parse_elf_info( bank, &elf_info, program_data, program_data_len ) ) ) {
- failed_elf_parsing = 1;
- record_sz = fd_program_cache_failed_verification_entry_footprint();
- } else {
- failed_elf_parsing = 0;
- record_sz = fd_program_cache_entry_footprint( &elf_info );
- }
-
- /* Resize the record to the new footprint if needed */
- uchar data[ record_sz ];
- fd_program_cache_entry_t * writable_entry = fd_type_pun( data );
-
- /* If the ELF header parsing failed, publish a failed verification
- record. */
- if( FD_UNLIKELY( failed_elf_parsing ) ) {
- fd_program_cache_entry_set_failed_verification( writable_entry, current_slot );
- } else {
- /* Validate the sBPF program. This will set the program's flags
- accordingly. We publish the funk record regardless of the return
- code. */
- writable_entry = fd_program_cache_entry_new( data, &elf_info, last_slot_modified, current_slot );
- int res = fd_program_cache_validate_sbpf_program( bank, &elf_info, program_data, program_data_len, runtime_spad, writable_entry );
- if( FD_UNLIKELY( res ) ) {
- FD_LOG_DEBUG(( "fd_program_cache_validate_sbpf_program() failed" ));
- }
- }
-
- fd_funk_rec_insert_para( funk, xid, &id, alignof(fd_program_cache_entry_t), record_sz, writable_entry );
-
-} FD_SPAD_FRAME_END;
-}
-
-void
-fd_program_cache_queue_program_for_reverification( fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_pubkey_t const * program_key,
- ulong current_slot ) {
-
- /* We want to access the program cache entry for this pubkey and
- queue it for reverification. If it exists, clone it down to the
- current funk txn and update the entry's `last_slot_modified`
- field.
-
- This read is thread-safe because if you have transaction A that
- modifies the program and transaction B that references the program,
- the dispatcher will not attempt to execute transaction B until
- transaction A is finalized. */
- fd_program_cache_entry_t const * existing_entry = NULL;
- int err = fd_program_cache_load_entry( funk, xid, program_key, &existing_entry );
- if( FD_UNLIKELY( err ) ) {
- return;
- }
-
- /* Ensure the record is in the current funk transaction */
- fd_funk_rec_key_t id = fd_program_cache_key( program_key );
- fd_program_cache_entry_t entry = *existing_entry;
- entry.last_slot_modified = current_slot;
- fd_funk_rec_insert_para( funk, xid, &id, alignof(fd_program_cache_entry_t), sizeof(fd_program_cache_entry_t), &entry );
-}
-
-int
-fd_funk_rec_insert_para( fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_funk_rec_key_t const * key,
- ulong val_align,
- ulong val_sz,
- void * val ) {
- if( FD_UNLIKELY( !funk ) ) FD_LOG_ERR(( "NULL funk" ));
- if( FD_UNLIKELY( !xid ) ) FD_LOG_ERR(( "NULL xid" ));
- if( FD_UNLIKELY( !key ) ) FD_LOG_ERR(( "NULL key" ));
- if( FD_UNLIKELY( !val && val_sz ) ) FD_LOG_ERR(( "NULL val" ));
-
- fd_funk_xid_key_pair_t pair[1];
- fd_funk_txn_xid_copy( pair->xid, xid );
- fd_funk_rec_key_copy( pair->key, key );
-
- for(;;) {
- /* See if the record already exists. */
- fd_funk_rec_query_t query[1];
- int err = fd_funk_rec_map_query_try( funk->rec_map, pair, NULL, query, 0 );
- if( err == FD_MAP_SUCCESS ) {
- fd_funk_rec_t * rec = fd_funk_rec_map_query_ele( query );
- /* Set the value of the record */
- if( !fd_funk_val_truncate( rec, fd_funk_alloc( funk ), fd_funk_wksp( funk ), val_align, val_sz, &err ) ) {
- FD_LOG_ERR(( "fd_funk_val_truncate() failed (out of memory?)" ));
- return err;
- }
- memcpy( fd_funk_val( rec, fd_funk_wksp( funk ) ), val, val_sz );
- return FD_FUNK_SUCCESS;
- }
-
- /* The record doesn't exist. Create it. */
- fd_funk_rec_prepare_t prepare[1];
- fd_funk_rec_t * rec = fd_funk_rec_prepare( funk, xid, key, prepare, &err );
- if( FD_UNLIKELY( !rec ) ) {
- FD_LOG_CRIT(( "fd_funk_rec_prepare returned err=%d", err ));
- return err;
- }
- /* Set the value of the record */
- if( !fd_funk_val_truncate( rec, fd_funk_alloc( funk ), fd_funk_wksp( funk ), val_align, val_sz, &err ) ) {
- FD_LOG_ERR(( "fd_funk_val_truncate() failed (out of memory?)" ));
- return err;
- }
- memcpy( fd_funk_val( rec, fd_funk_wksp( funk ) ), val, val_sz );
- fd_funk_rec_publish( funk, prepare );
- return FD_FUNK_SUCCESS;
- }
-}
diff --git a/src/flamenco/runtime/program/fd_program_cache.h b/src/flamenco/runtime/program/fd_program_cache.h
deleted file mode 100644
index 3e4b63bd59e..00000000000
--- a/src/flamenco/runtime/program/fd_program_cache.h
+++ /dev/null
@@ -1,325 +0,0 @@
-#ifndef HEADER_fd_src_flamenco_runtime_program_fd_program_cache_h
-#define HEADER_fd_src_flamenco_runtime_program_fd_program_cache_h
-
-#include "../../types/fd_types_custom.h"
-#include "../../../funk/fd_funk_base.h"
-#include "../../../ballet/sbpf/fd_sbpf_loader.h"
-
-/* fd_program_cache contains the core logic for the program cache's
- behavior, including accesses, insertions, updates, and verifies.
- Specifically, the program cache operates in a lazy manner where
- programs are only inserted / reverified when they are referenced
- in a transaction.
-
- The program cache lives in Funk and is fork-aware. The size of the
- program cache is bounded by the size of the Funk instance.
-
- On bootup, the program cache starts out empty. As programs (defined
- as accounts owned by one of the BPF loaders that can be invoked
- by the user) are referenced as account keys within transactions
- (either statically or through address lookup tables),
- `fd_program_cache_update_program()` accepts a pubkey as input and
- will attempt to either insert the program into the cache (performing
- sBPF + ELF validations) if missing, or reverify the program if
- needed.
-
- During transaction execution, we accumulate any program pubkeys that
- are deployed, extended, or upgraded (basically, any calls to
- `fd_deploy_program()` in the BPF loaders) since those instructions
- may have changed the program's executable data (closed / retracted
- programs also do, but see comments below on how we handle that).
- After a transaction is executed successfully, we iterate through
- these accumulated program keys and queue them for reverification.
- To mark a program for reverification, we simply update the
- `last_slot_modified` field to the current slot. When the program is
- next referenced in a transaction, `fd_program_cache_update_program()`
- will check if the program was modified since the last time it was
- reverified, and reverify the program accordingly to update the cache
- entry's sBPF info, calldests, etc. Once the program has been
- reverified, the entry's `last_slot_verified` field will be updated
- to the current slot.
-
- When a program is reverified and updated in the cache, we clone the
- record down to the current funk transaction from an ancestor, and
- then modify the record within the current funk transaction.
-
- A program cache entry may also be reverified after crossing an
- epoch boundary, even if it was not modified recently. This is because
- the network's active feature set may have changed, and any previously
- {valid,invalid} programs may now be {valid,invalid}. Therefore,
- if a program is referenced in a transaction that has not been
- reverified since the last epoch (regardless of it having been
- modified or not), it will be reverified and updated.
-
- If a program fails verification due to invalid ELF headers / sBPF
- loading failures, then we update the `failed_verification` tag,
- set `last_slot_verified` to the current slot, and set
- `last_slot_modified` to 0. All other fields have undefined value. A
- future program upgrade / network feature set change could make this
- program valid again.
-
- A key invariant with the program cache design is that it does not
- evict any entries - it only grows through the lifetime of the
- client's execution. Because of this invariant, we do not touch cache
- entries for programs which are closed, and instead let
- transaction-level account loading and BPF loader program checks
- catch cases where a user tries to invoke a closed program.
-
- Another key invariant is that for any given program, we will insert /
- update its cache entry at most ONCE per slot, and this will happen
- the FIRST time the program is referenced in a transaction before it
- is dispatched to the exec tiles. This is important because it ensures
- that the replay tile does not race with the exec tiles by updating
- the cache entry for a program while an exec tile is already executing
- it. Even if a program is upgraded in the same slot after it has been
- reverified, the network forbids invoking programs in the same slot as
- they are deployed / upgraded, so we can wait for a future slot to
- update the program cache entry when it is invoked next.
-
- EDGE CASES (and how we handle them):
- - Deploying programs
- - Deploying a program
- - `fd_program_cache_queue_program_for_reverification()` will
- not do anything because the program does not exist in the
- cache yet, and so there is nothing to update. The next time
- the program is referenced in a transaction,
- `fd_program_cache_update_program()` will see that the program
- is missing in the cache and will perform ELF / sBPF
- verifications and insert the entry into the cache.
- - Deploying and invoking in the same slot
- - BPF loader checks will fail because the program account's
- slot value will be equal to the current slot, which
- gets caught by the `DelayedVisibility` checks.
- - Upgrading programs
- - Upgrading a program
- - The program account will have been referenced as writable
- in the transaction, and thus
- `fd_program_cache_queue_program_for_reverification()` will
- update the `last_slot_modified` to queue the program
- for reverification the next time it is referenced in a
- transaction.
- - Upgrading and invoking in the same transaction
- - Same as "deploy" case
- - Closing programs
- - Closing a program
- - We do not touch the program cache here. We let account
- loading / BPF loader checks handle cases where a user may try
- to invoke a closed program.
- - Closing + invoking a program in the same transaction
- - The program account's state will be set to uninitialized /
- retracted, so any future instructions that invoke the program
- will fail when the BPF loader checks the account state.
- - Closing + invoking a program in separate transactions
- - The program account's owner will be set to the system program
- and thus fail account loading checks.
-
- TL;DR: Deploys and upgrades are treated the same way - if the cache
- entry exists, it's queued for reverification. If it doesn't, the
- program cache will verify and add it to the cache the next time it's
- invoked. Cases where the program is closed / retracted are not
- explicitly handled.
-
- Q: Is the program cache concurrency-safe?
- A: Yes. In any given slot, the replay tile first iterates through
- each transaction in a single-threaded manner. For each
- transaction, it will read the accounts and will call
- `fd_program_cache_update_program()` for each program, and then
- dispatch the transaction to the exec tiles. This means
- that the first time any program is referenced in any transaction
- in the slot, the replay tile will update the associated program
- cache entry before dispatching the transaction to any of the exec
- tiles. Furthermore, we are not allowed to insert / reverify a
- program cache entry more than once within a single slot. This
- guarantees that any read / write accesses for a particular program
- in the program cache by the exec tiles will occur after the cache
- entries have been processed by the replay tile in any given slot.
- Furthermore, if the program was upgraded, the exec tile simply
- updates a single header in the existing program cache entry
- `last_slot_modified`, which is behind a blocking write lock.
- Note that if there is read-write or write-write-contention
- between two transactions for any accounts, the scheduler will
- ensure that those two transactions are scheduled and finalized
- in a serial manner. */
-
-/* `fd_program_cache_entry` defines the structure for a single
- program cache entry. */
-struct fd_program_cache_entry {
- ulong magic;
-
- /* For any programs that fail verification, we retain this flag for
- to prevent any reverification attempts for the remainder of the
- epoch / until the program is modified again (instead of removing
- them from the cache). When `failed_verification` is set,
- the values of all other fields except for
- `last_slot_verified` are undefined. Any invocations of a
- program that fails verification will continue to fail until the
- program is queued for reverification by an eligible BPF loader
- instruction (e.g. the program is upgraded). */
- uchar failed_verification;
-
- /* Stores the last slot the program was modified. This field is
- updated at the end of a transaction for a program account that was
- referenced as a writable account within an instruction. For any
- programs that are freshly added to the cache, this field is set to
- 0. */
- ulong last_slot_modified;
-
- /* Stores the last slot verification checks were ran for a program.
- Programs are reverified if they are mentioned in the current
- transaction, and if one of the following are true:
- - It is the first time they are referenced in a transaction in the
- current epoch
- - The program was a writable account in a transaction
-
- We reverify referenced programs at least once every epoch
- regardless of whether they were modified or not because changes
- in the active feature set may cause existing deployed programs
- to be invalided (e.g. stricter ELF / VM / sBPF checks). */
- ulong last_slot_verified;
-
- ulong entry_pc;
- ulong text_cnt;
- ulong text_off;
- ulong text_sz;
-
- ulong rodata_sz;
-
- /* We use offsets to store the calldests and rodata in order to keep
- the cache entry gaddr-aware so it can be passed around different
- tiles. */
- ulong calldests_shmem_off;
- ulong rodata_off;
-
- /* SBPF version, SIMD-0161 */
- ulong sbpf_version;
-};
-typedef struct fd_program_cache_entry fd_program_cache_entry_t;
-
-static inline uchar *
-fd_program_cache_get_rodata( fd_program_cache_entry_t const * cache_entry ) {
- return (uchar *)cache_entry + cache_entry->rodata_off;
-}
-
-static inline fd_sbpf_calldests_t *
-fd_program_cache_get_calldests_shmem( fd_program_cache_entry_t const * cache_entry ) {
- return (fd_sbpf_calldests_t *)((uchar *)cache_entry + cache_entry->calldests_shmem_off);
-}
-
-static inline fd_sbpf_calldests_t *
-fd_program_cache_get_calldests( fd_program_cache_entry_t const * cache_entry ) {
- return fd_sbpf_calldests_join( fd_program_cache_get_calldests_shmem( cache_entry ) );
-}
-
-/* arbitrary unique value, in this case
- echo -n "fd_program_cache_entry" | sha512sum | head -c 16 */
-#define FD_PROGRAM_CACHE_ENTRY_MAGIC 0xb45640baf006ddf6
-
-FD_PROTOTYPES_BEGIN
-
-fd_program_cache_entry_t *
-fd_program_cache_entry_new( void * mem,
- fd_sbpf_elf_info_t const * elf_info,
- ulong last_slot_modified,
- ulong last_slot_verified );
-
-ulong
-fd_program_cache_entry_footprint( fd_sbpf_elf_info_t const * elf_info );
-
-/* Gets the program cache funk record key for a given program pubkey. */
-fd_funk_rec_key_t
-fd_program_cache_key( fd_pubkey_t const * pubkey );
-
-/* Loads a single program cache entry for a given pubkey. Returns 0 on
- success and -1 on failure. On success, `*cache_entry` holds a pointer
- to the program cache entry. */
-int
-fd_program_cache_load_entry( fd_funk_t const * funk,
- fd_funk_txn_xid_t const * xid,
- fd_pubkey_t const * program_pubkey,
- fd_program_cache_entry_t const ** cache_entry );
-
-/* Parses the programdata from a program account. Returns a pointer to
- the program data and sets `out_program_data_len` on success. Returns
- NULL on failure or if the program account is not owned by a BPF
- loader program ID, and leaves `out_program_data_len` in an undefined
- state. Reasons for failure vary on the loader version. See the
- respective functions in this file for more details.
-
- This is essentially load_program_with_pubkey(), sans the ELF parsing
- and whatnot which is done in load_program_from_bytes(). */
-uchar const *
-fd_program_cache_get_account_programdata( fd_funk_t const * funk,
- fd_funk_txn_xid_t const * xid,
- fd_txn_account_t const * program_acc,
- ulong * out_program_data_len );
-
-/* Updates the program cache for a single program. This function is
- called for every program that is referenced in a transaction, plus
- every single account in a lookup table referenced in the transaction.
- This function...
- - Accepts a pubkey and reads the programdata from the account
- - Creates a program cache entry for the program if it doesn't exist
- in the cache already
- - Reverifies programs if either...
- - The program was recently modified in a STRICTLY PRIOR SLOT
- - The program has not been verified yet for the current epoch
- - Invalidated programs that fail ELF / sBPF verification
- - Updates the program cache entry for the program after
- reverification (syscalls, calldests, etc)
-
- With this design, the program cache is designed to only grow as new
- programs are deployed / invoked. If a program fails verification, it
- stays in the cache so that repeated calls won't DOS the validator by
- forcing reverifications (since we won't be able to distinguish failed
- verifications from new deployments).
-
- When a program is reverified (and the cache entry already exists in
- some ancestor funk transaction), we clone the record down to the
- current funk transaction and acquire a blocking write lock on the
- cloned funk record. */
-void
-fd_program_cache_update_program( fd_bank_t * bank,
- fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_pubkey_t const * program_key,
- fd_spad_t * runtime_spad );
-
-/* Queues a single program account for reverification. This function
- queries the cache for an existing entry and queues it for
- reverification by setting the `last_slot_modified` field in the
- program cache entry to the current slot. If the cache entry
- for the program does not exist yet (e.g. newly deployed programs),
- this function does nothing and instead,
- `fd_program_cache_publish_failed_verification_rec()` will insert
- the program into the cache.
-
- If the record exists in the program cache, this function will
- acquire a write-lock on the program cache entry. */
-void
-fd_program_cache_queue_program_for_reverification( fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_pubkey_t const * program_key,
- ulong current_slot );
-
-/* fd_funk_rec_insert_para does thread-safe insertion of a funk record.
-
- Detailed Behavior:
-
- More specifically, first this function will query the transaction
- stack to identify what the youngest transaction with the key is.
- If a record is found in some ancestor txn or if the
- record doesn't exist, we will allocate a new account record and add
- this into the transaction. In either case, the record is set to the
- given value. */
-
-int
-fd_funk_rec_insert_para( fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_funk_rec_key_t const * key,
- ulong val_align,
- ulong val_sz,
- void * val );
-
-FD_PROTOTYPES_END
-
-#endif /* HEADER_fd_src_flamenco_runtime_program_fd_program_cache_h */
diff --git a/src/flamenco/runtime/program/test_program_cache.c b/src/flamenco/runtime/program/test_program_cache.c
deleted file mode 100644
index 84197c59153..00000000000
--- a/src/flamenco/runtime/program/test_program_cache.c
+++ /dev/null
@@ -1,736 +0,0 @@
-#include "fd_program_cache.h"
-#include "../fd_bank.h"
-
-#if FD_HAS_HOSTED
-
-#define TEST_WKSP_TAG 1234UL
-
-/* Load in programdata for tests */
-FD_IMPORT_BINARY( valid_program_data, "src/ballet/sbpf/fixtures/hello_solana_program.so" );
-FD_IMPORT_BINARY( bigger_valid_program_data, "src/ballet/sbpf/fixtures/clock_sysvar_program.so" );
-FD_IMPORT_BINARY( zero_text_cnt_elf, "src/ballet/sbpf/fixtures/zero_text_cnt.elf" );
-
-static uchar const invalid_program_data[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07};
-
-/* Test program pubkeys */
-static fd_pubkey_t const test_program_pubkey = {
- .uc = { 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99,
- 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99 }
-};
-
-static ulong const SPAD_MEM_MAX = 100UL << 20; /* 100MB */
-
-/* Test setup and teardown helpers */
-static fd_wksp_t * test_wksp = NULL;
-static fd_funk_t * test_funk = NULL;
-static fd_spad_t * test_spad = NULL;
-static fd_bank_t * test_bank = NULL;
-static fd_banks_t * test_banks = NULL;
-static fd_funk_txn_xid_t test_xid = {0};
-
-static void
-test_teardown( void ) {
-
- if( test_spad ) {
- fd_spad_leave( test_spad );
- test_spad = NULL;
- }
-
- if( test_funk ) {
- fd_funk_leave( test_funk, NULL );
- test_funk = NULL;
- }
-
- if( test_wksp ) {
- fd_wksp_delete_anonymous( test_wksp );
- test_wksp = NULL;
- }
-}
-
-/* Helper to create a funk transaction */
-static fd_funk_txn_xid_t
-create_test_funk_txn( void ) {
- fd_funk_txn_xid_t xid = fd_funk_generate_xid();
- fd_funk_txn_xid_t root; fd_funk_txn_xid_set_root( &root );
- fd_funk_txn_prepare( test_funk, &root, &xid );
- return xid;
-}
-
-/* Helper to create a test account */
-static void
-create_test_account( fd_pubkey_t const * pubkey,
- fd_pubkey_t const * owner,
- uchar const * data,
- ulong data_len,
- uchar executable ) {
- fd_txn_account_t acc[1];
- fd_funk_rec_prepare_t prepare = {0};
- int err = fd_txn_account_init_from_funk_mutable( /* acc */ acc,
- /* pubkey */ pubkey,
- /* funk */ test_funk,
- /* xid */ &test_xid,
- /* do_create */ 1,
- /* min_data_sz */ data_len,
- /* prepare */ &prepare );
- FD_TEST( !err );
-
- if( data ) {
- fd_txn_account_set_data( acc, data, data_len );
- }
-
- acc->starting_lamports = 1UL;
- acc->starting_dlen = data_len;
- fd_txn_account_set_lamports( acc, 1UL );
- fd_txn_account_set_executable( acc, executable );
- fd_txn_account_set_owner( acc, owner );
-
- fd_txn_account_mutable_fini( acc, test_funk, &prepare );
-}
-
-static void
-update_account_data( fd_pubkey_t const * pubkey,
- uchar const * data,
- ulong data_len ) {
- fd_txn_account_t acc[1];
- fd_funk_rec_prepare_t prepare = {0};
- int err = fd_txn_account_init_from_funk_mutable( /* acc */ acc,
- /* pubkey */ pubkey,
- /* funk */ test_funk,
- /* xid */ &test_xid,
- /* do_create */ 0,
- /* min_data_sz */ data_len,
- /* prepare */ &prepare );
- FD_TEST( !err );
- FD_TEST( data );
-
- fd_txn_account_set_data( acc, data, data_len );
- fd_txn_account_mutable_fini( acc, test_funk, &prepare );
-}
-
-/* Test 1: Account doesn't exist */
-static void
-test_account_does_not_exist( void ) {
- FD_LOG_NOTICE(( "Testing: Account doesn't exist" ));
-
- test_xid = create_test_funk_txn();
-
- /* Call with a non-existent pubkey */
- fd_pubkey_t const non_existent_pubkey = {0};
-
- /* This should return early without doing anything */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &non_existent_pubkey, test_spad );
-
- /* Verify no cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &non_existent_pubkey, &valid_prog );
- FD_TEST( err==-1 ); /* Should not exist */
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 2: Account exists but is not owned by a BPF loader */
-static void
-test_account_not_bpf_loader_owner( void ) {
- FD_LOG_NOTICE(( "Testing: Account exists but is not owned by a BPF loader" ));
-
- test_xid = create_test_funk_txn();
-
- /* Create an account owned by a non-BPF loader */
- create_test_account( &test_program_pubkey,
- &fd_solana_system_program_id,
- invalid_program_data,
- sizeof(invalid_program_data),
- 1 );
-
- /* This should return early without doing anything */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify no cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( err==-1 ); /* Should not exist */
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 3: Program is not in cache yet (first time), but program fails validations */
-static void
-test_invalid_program_not_in_cache_first_time( void ) {
- FD_LOG_NOTICE(( "Testing: Program is not in cache yet (first time), but program fails validations" ));
-
- test_xid = create_test_funk_txn();
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- invalid_program_data,
- sizeof(invalid_program_data),
- 1 );
-
- /* This should create a cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err ); /* Should exist */
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==fd_bank_slot_get( test_bank ) );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 4: Program is not in cache yet (first time), but program passes validations */
-static void
-test_valid_program_not_in_cache_first_time( void ) {
- FD_LOG_NOTICE(( "Testing: Program is not in cache yet (first time), but program passes validations" ));
-
- test_xid = create_test_funk_txn();
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- valid_program_data,
- valid_program_data_sz,
- 1 );
-
- /* This should create a cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err ); /* Should exist */
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==fd_bank_slot_get( test_bank ) );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 5: Program is in cache but needs reverification
- (different epoch) */
-static void
-test_program_in_cache_needs_reverification( void ) {
- FD_LOG_NOTICE(( "Testing: Program is in cache but needs reverification (different epoch)" ));
-
- test_xid = create_test_funk_txn();
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- valid_program_data,
- valid_program_data_sz,
- 1 );
-
- /* First call to create cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==fd_bank_slot_get( test_bank ) );
- FD_TEST( valid_prog->last_slot_modified==0UL );
-
- /* Fast forward to next epoch */
- fd_bank_slot_set( test_bank, fd_bank_slot_get( test_bank ) + 432000UL );
-
- /* This should trigger reverification */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify the cache entry was updated */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==fd_bank_slot_get( test_bank ) );
- FD_TEST( valid_prog->last_slot_modified==0UL );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 6: Program is in cache and was just modified, so it should be
- queued for reverification */
-static void
-test_program_in_cache_queued_for_reverification( void ) {
- FD_LOG_NOTICE(( "Testing: Program is in cache and was just modified, so it should be queued for reverification" ));
-
- test_xid = create_test_funk_txn();
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- valid_program_data,
- valid_program_data_sz,
- 1 );
-
- /* First call to create cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==fd_bank_slot_get( test_bank ) );
-
- /* Fast forward to a future slot */
- ulong original_slot = fd_bank_slot_get( test_bank );
- fd_bank_slot_set( test_bank, fd_bank_slot_get( test_bank ) + 1000UL );
- ulong future_slot = fd_bank_slot_get( test_bank );
- FD_TEST( future_slot>original_slot );
-
- /* Get the program account to pass to the queue function */
- fd_txn_account_t program_acc[1];
- err = fd_txn_account_init_from_funk_readonly( program_acc, &test_program_pubkey, test_funk, &test_xid );
- FD_TEST( !err );
-
- /* Queue the program for reverification */
- fd_program_cache_queue_program_for_reverification( test_funk, &test_xid, &test_program_pubkey, future_slot );
-
- /* Verify the cache entry was updated with the future slot as last_slot_modified */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_modified==future_slot );
- FD_TEST( valid_prog->last_slot_verified==original_slot );
- FD_TEST( valid_prog->last_slot_modified>original_slot );
-
- /* Reverify the cache entry at the future slot */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify the cache entry was updated */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_modified==future_slot );
- FD_TEST( valid_prog->last_slot_verified==future_slot );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 7: Program queued for reverification but program doesn't exist
- in the cache yet */
-static void
-test_program_queued_for_reverification_account_does_not_exist( void ) {
- FD_LOG_NOTICE(( "Testing: Program queued for reverification but account doesn't exist" ));
-
- test_xid = create_test_funk_txn();
-
- /* Create a BPF loader account but don't add it to the cache */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- valid_program_data,
- valid_program_data_sz,
- 1 );
-
- /* Fast forward to a future slot */
- ulong original_slot = fd_bank_slot_get( test_bank );
- fd_bank_slot_set( test_bank, fd_bank_slot_get( test_bank ) + 1000UL );
- ulong future_slot = fd_bank_slot_get( test_bank );
- FD_TEST( future_slot>original_slot );
-
- /* Get the program account to pass to the queue function */
- fd_txn_account_t program_acc[1];
- int err = fd_txn_account_init_from_funk_readonly( program_acc, &test_program_pubkey, test_funk, &test_xid );
- FD_TEST( !err );
-
- /* Try to queue the program for reverification - this should return early since it's not in cache */
- fd_program_cache_queue_program_for_reverification( test_funk, &test_xid, &test_program_pubkey, future_slot );
-
- /* Verify no cache entry was created since the program wasn't in the cache */
- fd_program_cache_entry_t const * valid_prog = NULL;
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( err ); /* Should not exist */
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 8: Program is in cache and was just modified and queued for
- reverification, so when it is next reverified the
- `last_slot_verified` should be set to the current slot */
-static void
-test_program_in_cache_queued_for_reverification_and_processed( void ) {
- FD_LOG_NOTICE(( "Testing: Program is in cache and was just modified and queued for reverification, so the last slot reverification ran should be set to the current slot" ));
-
- test_xid = create_test_funk_txn();
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- valid_program_data,
- valid_program_data_sz,
- 1 );
-
- /* First call to create cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==fd_bank_slot_get( test_bank ) );
-
- /* Fast forward to a future slot */
- ulong original_slot = fd_bank_slot_get( test_bank );
- fd_bank_slot_set( test_bank, fd_bank_slot_get( test_bank ) + 11000UL );
- ulong future_slot = fd_bank_slot_get( test_bank );
- FD_TEST( future_slot>original_slot );
-
- /* Get the program account to pass to the queue function */
- fd_txn_account_t program_acc[1];
- err = fd_txn_account_init_from_funk_readonly( program_acc, &test_program_pubkey, test_funk, &test_xid );
- FD_TEST( !err );
-
- /* Queue the program for reverification */
- fd_program_cache_queue_program_for_reverification( test_funk, &test_xid, &test_program_pubkey, future_slot );
-
- /* Verify the cache entry was updated with the future slot as last_slot_modified */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_modified==future_slot );
- FD_TEST( valid_prog->last_slot_verified==original_slot );
- FD_TEST( valid_prog->last_slot_modified>original_slot );
-
- /* Fast forward to a future slot */
- fd_bank_slot_set( test_bank, fd_bank_slot_get( test_bank ) + 11000UL );
- ulong future_update_slot = fd_bank_slot_get( test_bank );
- FD_TEST( future_update_slot>future_slot );
-
- /* Now update the cache entry at the future slot */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify the cache entry was updated */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==future_update_slot );
- FD_TEST( valid_prog->last_slot_modified==future_slot );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 9: Genesis program fails verification, and is reverified later */
-static void
-test_invalid_genesis_program_reverified_after_genesis( void ) {
- FD_LOG_NOTICE(( "Testing: Program fails verification in genesis, and is reverified later" ));
-
- test_xid = create_test_funk_txn();
- fd_bank_slot_set( test_bank, 0UL );
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- invalid_program_data,
- sizeof(invalid_program_data),
- 1 );
-
- /* First call to create cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_modified==0UL );
- FD_TEST( valid_prog->last_slot_verified==0UL );
-
- /* Fast forward to a future slot */
- ulong original_slot = fd_bank_slot_get( test_bank );
- fd_bank_slot_set( test_bank, fd_bank_slot_get( test_bank ) + 11000UL );
- ulong future_slot = fd_bank_slot_get( test_bank );
- FD_TEST( future_slot>original_slot );
-
- /* Program invoked, update cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify the cache entry was updated */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==future_slot );
- FD_TEST( valid_prog->last_slot_modified==0UL );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 10: Genesis program passes verification, and is reverified
- later */
-static void
-test_valid_genesis_program_reverified_after_genesis( void ) {
- FD_LOG_NOTICE(( "Testing: Program passes verification in genesis, and is reverified later" ));
-
- test_xid = create_test_funk_txn();
- fd_bank_slot_set( test_bank, 0UL );
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- valid_program_data,
- valid_program_data_sz,
- 1 );
-
- /* First call to create cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_modified==0UL );
- FD_TEST( valid_prog->last_slot_verified==0UL );
-
- /* Fast forward to a future slot */
- ulong original_slot = fd_bank_slot_get( test_bank );
- fd_bank_slot_set( test_bank, fd_bank_slot_get( test_bank ) + 11000UL );
- ulong future_slot = fd_bank_slot_get( test_bank );
- FD_TEST( future_slot>original_slot );
-
- /* Program invoked, update cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify the cache entry was updated */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==future_slot );
- FD_TEST( valid_prog->last_slot_modified==0UL );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-/* Test 11: Program gets upgraded with a larger programdata size */
-static void
-test_program_upgraded_with_larger_programdata( void ) {
- FD_LOG_NOTICE(( "Testing: Program gets upgraded with a larger programdata size" ));
-
- test_xid = create_test_funk_txn();
- fd_bank_slot_set( test_bank, 0UL );
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- valid_program_data,
- valid_program_data_sz,
- 1 );
-
- /* First call to create cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_modified==0UL );
- FD_TEST( valid_prog->last_slot_verified==0UL );
-
- /* Fast forward to a future slot */
- ulong original_slot = fd_bank_slot_get( test_bank );
- fd_bank_slot_set( test_bank, fd_bank_slot_get( test_bank ) + 11000UL );
- ulong future_slot = fd_bank_slot_get( test_bank );
- FD_TEST( future_slot>original_slot );
-
- /* "Upgrade" the program by modifying the programdata */
- update_account_data( &test_program_pubkey, bigger_valid_program_data, bigger_valid_program_data_sz );
-
- /* Queue the program for reverification */
- fd_program_cache_queue_program_for_reverification( test_funk, &test_xid, &test_program_pubkey, future_slot );
-
- /* Verify the cache entry was updated with the future slot as last_slot_modified */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_modified==future_slot );
- FD_TEST( valid_prog->last_slot_verified==original_slot );
- FD_TEST( valid_prog->last_slot_modified>original_slot );
-
- /* Store the old program cache funk record size */
- fd_funk_rec_key_t id = fd_program_cache_key( &test_program_pubkey );
- fd_funk_rec_query_t query[1];
- fd_funk_rec_t const * prev_rec = fd_funk_rec_query_try_global( test_funk, &test_xid, &id, NULL, query );
- FD_TEST( prev_rec );
- ulong prev_rec_sz = prev_rec->val_sz;
- FD_TEST( !fd_funk_rec_query_test( query ) );
-
- /* Program invoked, update cache entry */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Get the new program cache funk record size, and make sure it's
- larger */
- fd_funk_rec_t const * new_rec = fd_funk_rec_query_try_global( test_funk, &test_xid, &id, NULL, query );
- FD_TEST( new_rec );
- ulong new_rec_sz = new_rec->val_sz;
- FD_TEST( new_rec_sz>prev_rec_sz );
- FD_TEST( !fd_funk_rec_query_test( query ) );
-
- /* Verify the cache entry was updated */
- err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err );
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( !valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==future_slot );
- FD_TEST( valid_prog->last_slot_modified==future_slot );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-static void
-test_zero_text_cnt_program_account( void ) {
- FD_LOG_NOTICE(( "Testing: Inserting an ELF with text_cnt=0 into the program cache" ));
-
- test_xid = create_test_funk_txn();
-
- /* Create a BPF loader account */
- create_test_account( &test_program_pubkey,
- &fd_solana_bpf_loader_program_id,
- zero_text_cnt_elf,
- zero_text_cnt_elf_sz,
- 1 );
-
- /* VM validate checks should catch this */
- fd_program_cache_update_program( test_bank, test_funk, &test_xid, &test_program_pubkey, test_spad );
-
- /* Verify failed verification cache entry was created */
- fd_program_cache_entry_t const * valid_prog = NULL;
- int err = fd_program_cache_load_entry( test_funk, &test_xid, &test_program_pubkey, &valid_prog );
- FD_TEST( !err ); /* Should exist */
- FD_TEST( valid_prog );
- FD_TEST( valid_prog->magic==FD_PROGRAM_CACHE_ENTRY_MAGIC );
- FD_TEST( valid_prog->failed_verification );
- FD_TEST( valid_prog->last_slot_verified==fd_bank_slot_get( test_bank ) );
-
- fd_funk_txn_cancel( test_funk, &test_xid );
-}
-
-
-int
-main( int argc,
- char ** argv ) {
- fd_boot( &argc, &argv );
-
- FD_LOG_NOTICE(( "Starting BPF program cache tests" ));
-
- /* Create workspace */
- test_wksp = fd_wksp_new_anonymous( FD_SHMEM_GIGANTIC_PAGE_SZ, 3UL, fd_log_cpu_id(), "test_wksp", 0UL );
- FD_TEST( test_wksp );
-
- /* Create funk */
- ulong funk_align = fd_funk_align();
- ulong funk_footprint = fd_funk_footprint( 1024UL, 1024UL );
- void * funk_mem = fd_wksp_alloc_laddr( test_wksp, funk_align, funk_footprint, TEST_WKSP_TAG );
- FD_TEST( funk_mem );
-
- void * shfunk = fd_funk_new( funk_mem, 1234UL, 5678UL, 1024UL, 1024UL );
- FD_TEST( shfunk );
-
- fd_funk_t funk_[1];
- test_funk = fd_funk_join( funk_, shfunk );
- FD_TEST( test_funk );
-
- /* Create spad */
- ulong spad_align = fd_spad_align();
- ulong spad_footprint = fd_spad_footprint( SPAD_MEM_MAX );
- void * spad_mem = fd_wksp_alloc_laddr( test_wksp, spad_align, spad_footprint, TEST_WKSP_TAG );
- FD_TEST( spad_mem );
-
- test_spad = fd_spad_join( fd_spad_new( spad_mem, SPAD_MEM_MAX ) );
- FD_TEST( test_spad );
-
- FD_SPAD_FRAME_BEGIN( test_spad ) {
-
- /* Set up bank */
- ulong banks_footprint = fd_banks_footprint( 1UL, 1UL );
- uchar * banks_mem = fd_wksp_alloc_laddr( test_wksp, fd_banks_align(), banks_footprint, TEST_WKSP_TAG );
- FD_TEST( banks_mem );
-
- fd_banks_t * banks = fd_banks_join( fd_banks_new( banks_mem, 1UL, 1UL ) );
- FD_TEST( banks );
- fd_bank_t * bank = fd_banks_init_bank( banks );
- fd_bank_slot_set( bank, 433000UL );
- FD_TEST( bank );
-
- test_bank = bank;
- test_banks = banks;
-
- fd_epoch_schedule_t epoch_schedule = {
- .slots_per_epoch = 432000UL,
- .leader_schedule_slot_offset = 432000UL,
- .warmup = 0,
- .first_normal_epoch = 0UL,
- .first_normal_slot = 0UL
- };
- fd_bank_epoch_schedule_set( bank, epoch_schedule );
-
- test_account_does_not_exist();
- test_account_not_bpf_loader_owner();
- test_invalid_program_not_in_cache_first_time();
- test_valid_program_not_in_cache_first_time();
- test_program_in_cache_needs_reverification();
- test_program_in_cache_queued_for_reverification();
- test_program_queued_for_reverification_account_does_not_exist();
- test_program_in_cache_queued_for_reverification_and_processed();
- test_invalid_genesis_program_reverified_after_genesis();
- test_valid_genesis_program_reverified_after_genesis();
- test_program_upgraded_with_larger_programdata();
- test_zero_text_cnt_program_account();
- } FD_SPAD_FRAME_END;
-
- test_teardown();
-
- FD_LOG_NOTICE(( "pass" ));
- fd_halt();
- return 0;
-}
-
-#undef TEST_WKSP_TAG
-
-#else
-
-int
-main( int argc,
- char ** argv ) {
- fd_boot( &argc, &argv );
- FD_LOG_WARNING(( "skip: unit test requires FD_HAS_HOSTED capabilities" ));
- fd_halt();
- return 0;
-}
-
-#endif
diff --git a/src/flamenco/runtime/program/test_program_cache_concur.cxx b/src/flamenco/runtime/program/test_program_cache_concur.cxx
deleted file mode 100644
index 057779323fd..00000000000
--- a/src/flamenco/runtime/program/test_program_cache_concur.cxx
+++ /dev/null
@@ -1,138 +0,0 @@
-#include "../../../funk/fd_funk_rec.h"
-#include "../../../funk/fd_funk_txn.h"
-#include "../../../funk/test_funk_common.hxx"
-#include "fd_program_cache.h"
-#include
-#include
-
-#define NUM_THREADS 4
-#define MAX_TXN_CNT 4096
-#define NUM_KEYS 64
-
-static volatile uint exp_val[NUM_KEYS] = {0};
-
-struct test_funk_thread_arg {
- fd_funk_t * funk;
- fd_funk_txn_xid_t xid;
- uint start_idx;
- uint end_idx;
-};
-typedef struct test_funk_thread_arg test_funk_thread_arg_t;
-
-static void * work_thread( void * _arg ) {
- test_funk_thread_arg_t * arg = (test_funk_thread_arg_t *)_arg;
- fd_funk_t * funk = arg->funk;
- fd_funk_txn_xid_t const * xid = &arg->xid;
-
- for( ulong i=0UL; i<1024UL; i++ ) {
- uint key_idx = ((uint)lrand48() % (arg->end_idx - arg->start_idx)) + arg->start_idx;
-
- /* Increment the value. */
- FD_ATOMIC_FETCH_AND_ADD( &exp_val[key_idx], 1 );
-
- /* Update the record. */
- fd_funk_rec_key_t key = {};
- key.ul[0] = key_idx;
- fd_funk_rec_insert_para( funk, xid, &key, alignof(uint), sizeof(uint), (void*)&exp_val[key_idx] );
- }
- return NULL;
-
-}
-
-int main( int argc, char ** argv ) {
- srand(1234);
-
- fd_boot( &argc, &argv );
-
- char const * _page_sz = fd_env_strip_cmdline_cstr ( &argc, &argv, "--page-sz", NULL, "gigantic" );
- ulong page_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--page-cnt", NULL, 1UL );
- ulong near_cpu = fd_env_strip_cmdline_ulong( &argc, &argv, "--near-cpu", NULL, fd_log_cpu_id() );
-
- ulong page_sz = fd_cstr_to_shmem_page_sz( _page_sz );
-
- ulong txn_max = MAX_TXN_CNT;
- uint rec_max = 1<<20;
- fd_wksp_t * wksp = fd_wksp_new_anonymous( page_sz, page_cnt, near_cpu, "wksp", 0UL );
- FD_TEST( wksp );
- void * mem = fd_wksp_alloc_laddr(
- wksp,
- fd_funk_align(),
- fd_funk_footprint( txn_max, rec_max ),
- FD_FUNK_MAGIC );
- FD_TEST( mem );
- fd_funk_t funk_[1];
- fd_funk_t * funk = fd_funk_join( funk_, fd_funk_new( mem, 1, 1234U, txn_max, rec_max ) );
- FD_TEST( funk );
-
- /* Insert the records with their initial values. (0) */
- for( uint i = 0; i < NUM_KEYS; ++i ) {
- fd_funk_rec_key_t key = {};
- key.ul[0] = i;
- fd_funk_rec_prepare_t prepare[1];
- fd_funk_rec_t * rec = fd_funk_rec_prepare( funk, fd_funk_last_publish( funk ), &key, prepare, NULL );
- FD_TEST( rec );
-
- void * val = fd_funk_val_truncate(
- rec,
- fd_funk_alloc( funk ),
- fd_funk_wksp( funk ),
- alignof(uint),
- sizeof(uint),
- NULL
- );
- FD_TEST( val );
-
- /* Set the value to 0. */
- uint * val_ul = (uint *)val;
- *val_ul = 0U;
- fd_funk_rec_publish( funk, prepare );
- }
-
- fd_funk_txn_xid_t parent_xid; fd_funk_txn_xid_set_root( &parent_xid );
- fd_funk_txn_xid_t xid = {{0}};
-
- /* Number of iterations to run. */
- for( ulong i=0UL; ifunk );
+ fd_progcache_clear( runner->progcache_admin );
}
/* Sets up block execution context from an input test case to execute against the runtime.
@@ -182,6 +183,7 @@ fd_runtime_fuzz_block_ctx_create( fd_solfuzz_runner_t * runner,
/* Create temporary funk transaction and slot / epoch contexts */
fd_funk_txn_xid_t parent_xid; fd_funk_txn_xid_set_root( &parent_xid );
fd_funk_txn_prepare( funk, &parent_xid, xid );
+ fd_progcache_txn_prepare( runner->progcache_admin, &parent_xid, xid );
/* Restore feature flags */
fd_features_t features = {0};
@@ -302,9 +304,6 @@ fd_runtime_fuzz_block_ctx_create( fd_solfuzz_runner_t * runner,
from the parent slot */
fd_bank_epoch_set( bank, fd_slot_to_epoch( epoch_schedule, parent_slot, NULL ) );
- /* Refresh the program cache */
- fd_runtime_fuzz_refresh_program_cache( bank, funk, xid, test_ctx->acct_states, test_ctx->acct_states_count, runner->spad );
-
/* Update vote cache for epoch T-1 */
vote_states_prev = fd_bank_vote_states_prev_locking_modify( bank );
fd_runtime_fuzz_block_update_prev_epoch_votes_cache( vote_states_prev,
@@ -350,6 +349,7 @@ fd_runtime_fuzz_block_ctx_create( fd_solfuzz_runner_t * runner,
/* Make a new funk transaction since we're done loading in accounts for context */
fd_funk_txn_xid_t fork_xid = { .ul = { slot, slot } };
fd_funk_txn_prepare( funk, xid, &fork_xid );
+ fd_progcache_txn_prepare( runner->progcache_admin, xid, &fork_xid );
xid[0] = fork_xid;
/* Reset the lthash to zero, because we are in a new Funk transaction now */
@@ -434,9 +434,6 @@ fd_runtime_fuzz_block_ctx_exec( fd_solfuzz_runner_t * runner,
for( ulong i=0UL; ibank, runner->funk, xid, txn, runner->spad );
-
/* Execute the transaction against the runtime */
res = FD_RUNTIME_EXECUTE_SUCCESS;
fd_exec_txn_ctx_t * txn_ctx = fd_runtime_fuzz_txn_ctx_exec( runner, xid, txn, &res );
@@ -449,6 +446,7 @@ fd_runtime_fuzz_block_ctx_exec( fd_solfuzz_runner_t * runner,
/* Finalize the transaction */
fd_runtime_finalize_txn(
runner->funk,
+ runner->progcache,
NULL,
xid,
txn_ctx,
diff --git a/src/flamenco/runtime/tests/fd_dump_pb.c b/src/flamenco/runtime/tests/fd_dump_pb.c
index 41807cddbd5..1fe4ee88e26 100644
--- a/src/flamenco/runtime/tests/fd_dump_pb.c
+++ b/src/flamenco/runtime/tests/fd_dump_pb.c
@@ -7,7 +7,7 @@
#include "../fd_bank.h"
#include "../program/fd_address_lookup_table_program.h"
#include "../../../ballet/nanopb/pb_encode.h"
-#include "../program/fd_program_cache.h"
+#include "../../progcache/fd_prog_load.h"
#include /* fopen */
@@ -1238,10 +1238,8 @@ FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
/* Get the programdata for the account */
ulong program_data_len = 0UL;
- uchar const * program_data = fd_program_cache_get_account_programdata( txn_ctx->funk,
- txn_ctx->xid,
- program_acc,
- &program_data_len );
+ uchar const * program_data =
+ fd_prog_load_elf( txn_ctx->funk, txn_ctx->xid, program_acc, &program_data_len, NULL );
if( program_data==NULL ) {
return;
}
diff --git a/src/flamenco/runtime/tests/fd_elf_harness.c b/src/flamenco/runtime/tests/fd_elf_harness.c
index b5229c01449..f9f92fe7954 100644
--- a/src/flamenco/runtime/tests/fd_elf_harness.c
+++ b/src/flamenco/runtime/tests/fd_elf_harness.c
@@ -4,6 +4,7 @@
#include "../../../ballet/sbpf/fd_sbpf_loader.h"
#include "../program/fd_bpf_loader_program.h"
#include "../../vm/fd_vm_base.h"
+#include "../../progcache/fd_prog_load.h"
#define SORT_NAME sort_ulong
#define SORT_KEY_T ulong
@@ -57,11 +58,10 @@ fd_solfuzz_elf_loader_run( fd_solfuzz_runner_t * runner,
fd_sbpf_loader_config_t config = {
.elf_deploy_checks = input->deploy_checks,
};
- fd_bpf_get_sbpf_versions(
- &config.sbpf_min_version,
- &config.sbpf_max_version,
- UINT_MAX,
- &feature_set );
+
+ fd_prog_versions_t versions = fd_prog_versions( &feature_set, UINT_MAX );
+ config.sbpf_min_version = versions.min_sbpf_version;
+ config.sbpf_max_version = versions.max_sbpf_version;
err = fd_sbpf_elf_peek( &info, elf_bin, elf_sz, &config );
if( FD_UNLIKELY( err ) ) {
diff --git a/src/flamenco/runtime/tests/fd_harness_common.c b/src/flamenco/runtime/tests/fd_harness_common.c
index bc65069ba36..0aefb809423 100644
--- a/src/flamenco/runtime/tests/fd_harness_common.c
+++ b/src/flamenco/runtime/tests/fd_harness_common.c
@@ -1,4 +1,3 @@
-#include "../program/fd_program_cache.h"
#include "generated/context.pb.h"
#include "../fd_acc_mgr.h"
#include "../../features/fd_features.h"
@@ -70,17 +69,3 @@ fd_runtime_fuzz_restore_features( fd_features_t * features,
}
return 1;
}
-
-void
-fd_runtime_fuzz_refresh_program_cache( fd_bank_t * bank,
- fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_exec_test_acct_state_t const * acct_states,
- ulong acct_states_count,
- fd_spad_t * runtime_spad ) {
- for( ushort i=0; iprogcache_admin, &parent_xid, xid );
/* Allocate contexts */
uchar * txn_ctx_mem = fd_spad_alloc( runner->spad,FD_EXEC_TXN_CTX_ALIGN, FD_EXEC_TXN_CTX_FOOTPRINT );
@@ -70,17 +71,22 @@ fd_runtime_fuzz_instr_ctx_create( fd_solfuzz_runner_t * runner,
fd_memset( fd_blockhash_deq_push_tail_nocopy( blockhashes->d.deque ), 0, sizeof(fd_hash_t) );
/* Set up mock txn descriptor */
- fd_txn_p_t * txn = fd_spad_alloc( runner->spad, fd_txn_align(), fd_txn_footprint( 1UL, 0UL ) );
+ fd_txn_p_t * txn = fd_spad_alloc_check( runner->spad, fd_txn_align(), fd_txn_footprint( 1UL, 0UL ) );
fd_txn_t * txn_descriptor = TXN( txn );
txn_descriptor->transaction_version = FD_TXN_V0;
txn_descriptor->acct_addr_cnt = (ushort)test_ctx->accounts_count;
+ uchar * progcache_scratch = fd_spad_alloc_check( runner->spad, FD_PROGCACHE_SCRATCH_ALIGN, FD_PROGCACHE_SCRATCH_FOOTPRINT );
+
fd_exec_txn_ctx_setup( runner->bank,
- runner->funk,
+ runner->funk->shmem,
+ runner->progcache->funk->shmem,
xid,
NULL,
txn_ctx,
- NULL );
+ NULL,
+ progcache_scratch,
+ FD_PROGCACHE_SCRATCH_FOOTPRINT );
fd_exec_txn_ctx_setup_basic( txn_ctx );
txn_ctx->txn = *txn;
@@ -249,9 +255,6 @@ fd_runtime_fuzz_instr_ctx_create( fd_solfuzz_runner_t * runner,
}
}
- /* Refresh the program cache */
- fd_runtime_fuzz_refresh_program_cache( runner->bank, funk, xid, test_ctx->accounts, test_ctx->accounts_count, runner->spad );
-
/* Load instruction accounts */
if( FD_UNLIKELY( test_ctx->instr_accounts_count > MAX_TX_ACCOUNT_LOCKS ) ) {
@@ -308,11 +311,14 @@ fd_runtime_fuzz_instr_ctx_create( fd_solfuzz_runner_t * runner,
/* Refresh the setup from the updated slot and epoch ctx. */
fd_exec_txn_ctx_setup( runner->bank,
- runner->funk,
+ runner->funk->shmem,
+ runner->progcache->funk->shmem,
xid,
NULL,
txn_ctx,
- NULL );
+ NULL,
+ progcache_scratch,
+ FD_PROGCACHE_SCRATCH_FOOTPRINT );
fd_log_collector_init( &ctx->txn_ctx->log_collector, 1 );
fd_base58_encode_32( txn_ctx->account_keys[ ctx->instr->program_id ].uc, NULL, ctx->program_id_base58 );
@@ -320,16 +326,14 @@ fd_runtime_fuzz_instr_ctx_create( fd_solfuzz_runner_t * runner,
return 1;
}
-
-
void
fd_runtime_fuzz_instr_ctx_destroy( fd_solfuzz_runner_t * runner,
fd_exec_instr_ctx_t * ctx ) {
if( !ctx ) return;
fd_funk_txn_cancel_all( runner->funk );
+ fd_progcache_clear( runner->progcache_admin );
}
-
ulong
fd_solfuzz_instr_run( fd_solfuzz_runner_t * runner,
void const * input_,
diff --git a/src/flamenco/runtime/tests/fd_solfuzz.c b/src/flamenco/runtime/tests/fd_solfuzz.c
index bf409032b44..5feb286fe96 100644
--- a/src/flamenco/runtime/tests/fd_solfuzz.c
+++ b/src/flamenco/runtime/tests/fd_solfuzz.c
@@ -77,26 +77,37 @@ fd_solfuzz_runner_new( fd_wksp_t * wksp,
fd_solfuzz_runner_options_t const * options ) {
/* Allocate objects */
- ulong const txn_max = 64UL;
- ulong const rec_max = 1024UL;
+ ulong const txn_max = 16UL;
+ ulong const rec_max = 128UL;
ulong const spad_max = FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_FUZZ;
ulong const bank_max = 1UL;
ulong const fork_max = 1UL;
- fd_solfuzz_runner_t * runner = fd_wksp_alloc_laddr( wksp, alignof(fd_solfuzz_runner_t), sizeof(fd_solfuzz_runner_t), wksp_tag );
- void * funk_mem = fd_wksp_alloc_laddr( wksp, fd_funk_align(), fd_funk_footprint( txn_max, rec_max ), wksp_tag );
- void * spad_mem = fd_wksp_alloc_laddr( wksp, fd_spad_align(), fd_spad_footprint( spad_max ), wksp_tag );
- void * banks_mem = fd_wksp_alloc_laddr( wksp, fd_banks_align(), fd_banks_footprint( bank_max, fork_max ), wksp_tag );
- if( FD_UNLIKELY( !runner ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(solfuzz_runner) failed" )); goto bail1; }
- if( FD_UNLIKELY( !funk_mem ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(funk) failed" )); goto bail1; }
- if( FD_UNLIKELY( !spad_mem ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(spad) failed (spad_max=%g)", (double)spad_max )); goto bail1; }
- if( FD_UNLIKELY( !banks_mem ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(banks) failed (bank_max=%lu fork_max=%lu)", bank_max, fork_max )); goto bail1; }
+ fd_solfuzz_runner_t * runner = fd_wksp_alloc_laddr( wksp, alignof(fd_solfuzz_runner_t), sizeof(fd_solfuzz_runner_t), wksp_tag );
+ void * funk_mem = fd_wksp_alloc_laddr( wksp, fd_funk_align(), fd_funk_footprint( txn_max, rec_max ), wksp_tag );
+ void * pcache_mem = fd_wksp_alloc_laddr( wksp, fd_funk_align(), fd_funk_footprint( txn_max, rec_max ), wksp_tag );
+ uchar * scratch = fd_wksp_alloc_laddr( wksp, FD_PROGCACHE_SCRATCH_ALIGN, FD_PROGCACHE_SCRATCH_FOOTPRINT, wksp_tag );
+ void * spad_mem = fd_wksp_alloc_laddr( wksp, fd_spad_align(), fd_spad_footprint( spad_max ), wksp_tag );
+ void * banks_mem = fd_wksp_alloc_laddr( wksp, fd_banks_align(), fd_banks_footprint( bank_max, fork_max ), wksp_tag );
+ if( FD_UNLIKELY( !runner ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(solfuzz_runner) failed" )); goto bail1; }
+ if( FD_UNLIKELY( !funk_mem ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(funk) failed" )); goto bail1; }
+ if( FD_UNLIKELY( !pcache_mem ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(funk) failed" )); goto bail1; }
+ if( FD_UNLIKELY( !scratch ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(scratch) failed" )); goto bail1; }
+ if( FD_UNLIKELY( !spad_mem ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(spad) failed (spad_max=%g)", (double)spad_max )); goto bail1; }
+ if( FD_UNLIKELY( !banks_mem ) ) { FD_LOG_WARNING(( "fd_wksp_alloc(banks) failed (bank_max=%lu fork_max=%lu)", bank_max, fork_max )); goto bail1; }
/* Create objects */
fd_memset( runner, 0, sizeof(fd_solfuzz_runner_t) );
runner->wksp = wksp;
- void * shfunk = fd_funk_new( funk_mem, wksp_tag, 1UL, txn_max, rec_max );
- if( FD_UNLIKELY( !shfunk ) ) goto bail1;
+
+ void * shfunk = fd_funk_new( funk_mem, wksp_tag, 1UL, txn_max, rec_max );
+ void * shpcache = fd_funk_new( pcache_mem, wksp_tag, 1UL, txn_max, rec_max );
+ if( FD_UNLIKELY( !shfunk ) ) goto bail1;
+ if( FD_UNLIKELY( !shpcache ) ) goto bail1;
+
if( FD_UNLIKELY( !fd_funk_join( runner->funk, funk_mem ) ) ) goto bail2;
+ if( FD_UNLIKELY( !fd_progcache_join( runner->progcache, pcache_mem, scratch, FD_PROGCACHE_SCRATCH_FOOTPRINT ) ) ) goto bail2;
+ if( FD_UNLIKELY( !fd_progcache_admin_join( runner->progcache_admin, pcache_mem ) ) ) goto bail2;
+
runner->spad = fd_spad_join( fd_spad_new( spad_mem, spad_max ) );
if( FD_UNLIKELY( !runner->spad ) ) goto bail2;
runner->banks = fd_banks_join( fd_banks_new( banks_mem, bank_max, fork_max ) );
@@ -109,17 +120,21 @@ fd_solfuzz_runner_new( fd_wksp_t * wksp,
fd_bank_slot_set( runner->bank, 0UL );
runner->enable_vm_tracing = options->enable_vm_tracing;
+ FD_TEST( runner->progcache->funk->shmem );
return runner;
bail2:
- if( runner->spad ) fd_spad_delete( fd_spad_leave( runner->spad ) );
- if( shfunk ) fd_funk_delete( funk_mem ); /* free underlying fd_alloc instance */
- if( runner->banks ) fd_banks_delete( fd_banks_leave( runner->banks ) );
+ if( runner->spad ) fd_spad_delete( fd_spad_leave( runner->spad ) );
+ if( shfunk ) fd_funk_delete( shfunk ); /* free underlying fd_alloc instance */
+ if( shpcache ) fd_funk_delete( shpcache );
+ if( runner->banks ) fd_banks_delete( fd_banks_leave( runner->banks ) );
bail1:
- fd_wksp_free_laddr( funk_mem );
- fd_wksp_free_laddr( spad_mem );
- fd_wksp_free_laddr( banks_mem );
- fd_wksp_free_laddr( runner );
+ fd_wksp_free_laddr( scratch );
+ fd_wksp_free_laddr( pcache_mem );
+ fd_wksp_free_laddr( funk_mem );
+ fd_wksp_free_laddr( spad_mem );
+ fd_wksp_free_laddr( banks_mem );
+ fd_wksp_free_laddr( runner );
FD_LOG_WARNING(( "fd_solfuzz_runner_new failed" ));
return NULL;
}
diff --git a/src/flamenco/runtime/tests/fd_solfuzz.h b/src/flamenco/runtime/tests/fd_solfuzz.h
index b429508bdcd..411f2b17f94 100644
--- a/src/flamenco/runtime/tests/fd_solfuzz.h
+++ b/src/flamenco/runtime/tests/fd_solfuzz.h
@@ -12,6 +12,8 @@
#include "../../capture/fd_solcap_writer.h"
#include "../../../funk/fd_funk.h"
+#include "../../progcache/fd_progcache_admin.h"
+#include "../../progcache/fd_progcache_user.h"
/* A fd_solfuzz_runner_t object processes solfuzz inputs. Can be reused
for different inputs, even of different types. Single-thread per
@@ -27,6 +29,9 @@ struct fd_solfuzz_runner {
fd_banks_t * banks;
fd_bank_t * bank;
+ fd_progcache_t progcache[1];
+ fd_progcache_admin_t progcache_admin[1];
+
fd_solcap_writer_t * solcap;
void * solcap_file; /* FILE * */
diff --git a/src/flamenco/runtime/tests/fd_solfuzz_private.h b/src/flamenco/runtime/tests/fd_solfuzz_private.h
index 9a41cfd97e1..141f894734f 100644
--- a/src/flamenco/runtime/tests/fd_solfuzz_private.h
+++ b/src/flamenco/runtime/tests/fd_solfuzz_private.h
@@ -28,14 +28,6 @@ int
fd_runtime_fuzz_restore_features( fd_features_t * features,
fd_exec_test_feature_set_t const * feature_set );
-void
-fd_runtime_fuzz_refresh_program_cache( fd_bank_t * bank,
- fd_funk_t * funk,
- fd_funk_txn_xid_t const * xid,
- fd_exec_test_acct_state_t const * acct_states,
- ulong acct_states_count,
- fd_spad_t * runtime_spad );
-
typedef ulong( exec_test_run_fn_t )( fd_solfuzz_runner_t *,
void const *,
void **,
diff --git a/src/flamenco/runtime/tests/fd_txn_harness.c b/src/flamenco/runtime/tests/fd_txn_harness.c
index 293f2b2b782..2b763351f61 100644
--- a/src/flamenco/runtime/tests/fd_txn_harness.c
+++ b/src/flamenco/runtime/tests/fd_txn_harness.c
@@ -40,6 +40,7 @@ fd_runtime_fuzz_xid_cancel( fd_solfuzz_runner_t * runner,
fd_funk_txn_xid_t * xid ) {
if( FD_UNLIKELY( !xid ) ) return; // This shouldn't be false either
fd_funk_txn_cancel( runner->funk, xid );
+ fd_progcache_clear( runner->progcache_admin );
}
/* Creates transaction execution context for a single test case. Returns a
@@ -55,7 +56,8 @@ fd_runtime_fuzz_txn_ctx_create( fd_solfuzz_runner_t * runner,
/* Set up the funk transaction */
fd_funk_txn_xid_t xid = { .ul = { slot, slot } };
fd_funk_txn_xid_t parent_xid; fd_funk_txn_xid_set_root( &parent_xid );
- fd_funk_txn_prepare( funk, &parent_xid, &xid );
+ fd_funk_txn_prepare ( funk, &parent_xid, &xid );
+ fd_progcache_txn_prepare( runner->progcache_admin, &parent_xid, &xid );
/* Set up slot context */
fd_banks_clear_bank( runner->banks, runner->bank );
@@ -202,9 +204,6 @@ fd_runtime_fuzz_txn_ctx_create( fd_solfuzz_runner_t * runner,
/* Restore sysvars from account context */
fd_sysvar_cache_restore_fuzz( runner->bank, runner->funk, &xid );
- /* Refresh the program cache */
- fd_runtime_fuzz_refresh_program_cache( runner->bank, runner->funk, &xid, test_ctx->account_shared_data, test_ctx->account_shared_data_count, runner->spad );
-
/* Create the raw txn (https://solana.com/docs/core/transactions#transaction-size) */
fd_txn_p_t * txn = fd_spad_alloc( runner->spad, alignof(fd_txn_p_t), sizeof(fd_txn_p_t) );
ulong msg_sz = fd_runtime_fuzz_serialize_txn( txn->payload, &test_ctx->tx );
@@ -340,10 +339,17 @@ fd_runtime_fuzz_txn_ctx_exec( fd_solfuzz_runner_t * runner,
int * exec_res ) {
/* Setup the spad for account allocation */
- uchar * txn_ctx_mem = fd_spad_alloc( runner->spad, FD_EXEC_TXN_CTX_ALIGN, FD_EXEC_TXN_CTX_FOOTPRINT );
+ uchar * txn_ctx_mem = fd_spad_alloc_check( runner->spad, FD_EXEC_TXN_CTX_ALIGN, FD_EXEC_TXN_CTX_FOOTPRINT );
fd_exec_txn_ctx_t * txn_ctx = fd_exec_txn_ctx_join( fd_exec_txn_ctx_new( txn_ctx_mem ), runner->spad, fd_wksp_containing( runner->spad ) );
txn_ctx->flags = FD_TXN_P_FLAGS_SANITIZE_SUCCESS;
- txn_ctx->funk[0] = *runner->funk;
+ if( FD_UNLIKELY( !fd_funk_join( txn_ctx->funk, runner->funk->shmem ) ) ) {
+ FD_LOG_CRIT(( "fd_funk_join failed" ));
+ }
+ uchar * pc_scratch = fd_spad_alloc_check( runner->spad, FD_PROGCACHE_SCRATCH_ALIGN, FD_PROGCACHE_SCRATCH_FOOTPRINT );
+ txn_ctx->progcache = fd_progcache_join( txn_ctx->_progcache, runner->progcache->funk->shmem, pc_scratch, FD_PROGCACHE_SCRATCH_FOOTPRINT );
+ if( FD_UNLIKELY( !txn_ctx->progcache ) ) {
+ FD_LOG_CRIT(( "fd_progcache_join failed" ));
+ }
txn_ctx->bank_hash_cmp = NULL;
txn_ctx->fuzz_config.enable_vm_tracing = runner->enable_vm_tracing;
txn_ctx->xid[0] = *xid;
diff --git a/src/flamenco/vm/fd_vm.c b/src/flamenco/vm/fd_vm.c
index 6e413b52375..e2ef98f7fb6 100644
--- a/src/flamenco/vm/fd_vm.c
+++ b/src/flamenco/vm/fd_vm.c
@@ -585,7 +585,7 @@ fd_vm_init(
ulong text_off,
ulong text_sz,
ulong entry_pc,
- ulong * calldests,
+ ulong const * calldests,
ulong sbpf_version,
fd_sbpf_syscalls_t * syscalls,
fd_vm_trace_t * trace,
diff --git a/src/flamenco/vm/fd_vm.h b/src/flamenco/vm/fd_vm.h
index f9c029cd8f0..a5e01eb2775 100644
--- a/src/flamenco/vm/fd_vm.h
+++ b/src/flamenco/vm/fd_vm.h
@@ -292,7 +292,7 @@ fd_vm_init(
ulong text_off,
ulong text_sz,
ulong entry_pc,
- ulong * calldests,
+ ulong const * calldests,
ulong sbpf_version,
fd_sbpf_syscalls_t * syscalls,
fd_vm_trace_t * trace,
diff --git a/src/funk/fd_funk.c b/src/funk/fd_funk.c
index ba13a0fc76d..5d1fa53b846 100644
--- a/src/funk/fd_funk.c
+++ b/src/funk/fd_funk.c
@@ -109,6 +109,8 @@ fd_funk_new( void * shmem,
funk->child_head_cidx = fd_funk_txn_cidx( FD_FUNK_TXN_IDX_NULL );
funk->child_tail_cidx = fd_funk_txn_cidx( FD_FUNK_TXN_IDX_NULL );
+ for( ulong i=0UL; iele[ i ].lock );
+
fd_funk_txn_xid_set_root( funk->root );
fd_funk_txn_xid_set_root( funk->last_publish );
diff --git a/src/funk/fd_funk_base.h b/src/funk/fd_funk_base.h
index f1caa79f8b0..59957906c74 100644
--- a/src/funk/fd_funk_base.h
+++ b/src/funk/fd_funk_base.h
@@ -45,6 +45,10 @@
#include "../util/fd_util.h"
#include "../util/valloc/fd_valloc.h"
+#if FD_HAS_X86
+#include
+#endif
+
/* FD_FUNK_SUCCESS is used by various APIs to indicate the operation
successfully completed. This will be 0. FD_FUNK_ERR_* gives a
number of error codes used by fd_funk APIs. These will be negative
@@ -66,7 +70,7 @@
facilitate compile time declarations. */
#define FD_FUNK_REC_KEY_ALIGN (8UL)
-#define FD_FUNK_REC_KEY_FOOTPRINT (40UL) /* 32 byte hash + 8 byte meta */
+#define FD_FUNK_REC_KEY_FOOTPRINT (32UL)
/* A fd_funk_rec_key_t identifies a funk record. Compact binary keys
are encouraged but a cstr can be used so long as it has
@@ -77,8 +81,8 @@
union __attribute__((aligned(FD_FUNK_REC_KEY_ALIGN))) fd_funk_rec_key {
uchar uc[ FD_FUNK_REC_KEY_FOOTPRINT ];
- uint ui[ 10 ];
- ulong ul[ 5 ];
+ uint ui[ 8 ];
+ ulong ul[ 4 ];
};
typedef union fd_funk_rec_key fd_funk_rec_key_t;
@@ -88,7 +92,7 @@ typedef union fd_funk_rec_key fd_funk_rec_key_t;
of 2. FOOTPRINT is a multiple of ALIGN. These are provided to
facilitate compile time declarations. */
-#define FD_FUNK_TXN_XID_ALIGN (8UL)
+#define FD_FUNK_TXN_XID_ALIGN (16UL)
#define FD_FUNK_TXN_XID_FOOTPRINT (16UL)
/* A fd_funk_txn_xid_t identifies a funk transaction currently in
@@ -103,6 +107,12 @@ typedef union fd_funk_rec_key fd_funk_rec_key_t;
union __attribute__((aligned(FD_FUNK_TXN_XID_ALIGN))) fd_funk_txn_xid {
uchar uc[ FD_FUNK_TXN_XID_FOOTPRINT ];
ulong ul[ FD_FUNK_TXN_XID_FOOTPRINT / sizeof(ulong) ];
+#if FD_HAS_INT128
+ uint128 uf[1];
+#endif
+#if FD_HAS_X86
+ __m128i xmm[1];
+#endif
};
typedef union fd_funk_txn_xid fd_funk_txn_xid_t;
@@ -112,8 +122,8 @@ typedef union fd_funk_txn_xid fd_funk_txn_xid_t;
power of 2. FOOTPRINT is a multiple of ALIGN. These are provided to
facilitate compile time declarations. */
-#define FD_FUNK_XID_KEY_PAIR_ALIGN (8UL)
-#define FD_FUNK_XID_KEY_PAIR_FOOTPRINT (56UL)
+#define FD_FUNK_XID_KEY_PAIR_ALIGN (16UL)
+#define FD_FUNK_XID_KEY_PAIR_FOOTPRINT (48UL)
/* A fd_funk_xid_key_pair_t identifies a funk record. It is just
xid and key packed into the same structure. */
@@ -183,7 +193,6 @@ fd_funk_rec_key_hash1( uchar const key[ 32 ],
FD_FN_PURE static inline ulong
fd_funk_rec_key_hash( fd_funk_rec_key_t const * k,
ulong seed ) {
- seed ^= k->ul[4];
/* tons of ILP */
return (fd_ulong_hash( seed ^ (1UL<<0) ^ k->ul[0] ) ^ fd_ulong_hash( seed ^ (1UL<<1) ^ k->ul[1] ) ) ^
(fd_ulong_hash( seed ^ (1UL<<2) ^ k->ul[2] ) ^ fd_ulong_hash( seed ^ (1UL<<3) ^ k->ul[3] ) );
@@ -211,7 +220,7 @@ fd_funk_rec_key_hash1( uchar const key[ 32 ],
FD_FN_PURE static inline ulong
fd_funk_rec_key_hash( fd_funk_rec_key_t const * k,
ulong seed ) {
- return fd_funk_rec_key_hash1( k->uc, k->ul[4], seed );
+ return fd_funk_rec_key_hash1( k->uc, 0UL, seed );
}
#endif /* FD_HAS_INT128 */
@@ -222,10 +231,10 @@ fd_funk_rec_key_hash( fd_funk_rec_key_t const * k,
FD_FN_UNUSED FD_FN_PURE static int /* Workaround -Winline */
fd_funk_rec_key_eq( fd_funk_rec_key_t const * ka,
- fd_funk_rec_key_t const * kb ) {
+ fd_funk_rec_key_t const * kb ) {
ulong const * a = ka->ul;
ulong const * b = kb->ul;
- return !( ((a[0]^b[0]) | (a[1]^b[1])) | ((a[2]^b[2]) | (a[3]^b[3])) | (a[4]^b[4]) ) ;
+ return !( ((a[0]^b[0]) | (a[1]^b[1])) | ((a[2]^b[2]) | (a[3]^b[3])) ) ;
}
/* fd_funk_rec_key_copy copies the key pointed to by ks into the key
@@ -234,10 +243,10 @@ fd_funk_rec_key_eq( fd_funk_rec_key_t const * ka,
static inline fd_funk_rec_key_t *
fd_funk_rec_key_copy( fd_funk_rec_key_t * kd,
- fd_funk_rec_key_t const * ks ) {
+ fd_funk_rec_key_t const * ks ) {
ulong * d = kd->ul;
ulong const * s = ks->ul;
- d[0] = s[0]; d[1] = s[1]; d[2] = s[2]; d[3] = s[3]; d[4] = s[4];
+ d[0] = s[0]; d[1] = s[1]; d[2] = s[2]; d[3] = s[3];
return kd;
}
@@ -250,7 +259,7 @@ fd_funk_rec_key_copy( fd_funk_rec_key_t * kd,
FD_FN_UNUSED FD_FN_PURE static ulong /* Work around -Winline */
fd_funk_txn_xid_hash( fd_funk_txn_xid_t const * x,
- ulong seed ) {
+ ulong seed ) {
return ( fd_ulong_hash( seed ^ (1UL<<0) ^ x->ul[0] ) ^ fd_ulong_hash( seed ^ (1UL<<1) ^ x->ul[1] ) ); /* tons of ILP */
}
@@ -260,7 +269,7 @@ fd_funk_txn_xid_hash( fd_funk_txn_xid_t const * x,
FD_FN_PURE static inline int
fd_funk_txn_xid_eq( fd_funk_txn_xid_t const * xa,
- fd_funk_txn_xid_t const * xb ) {
+ fd_funk_txn_xid_t const * xb ) {
ulong const * a = xa->ul;
ulong const * b = xb->ul;
return !( (a[0]^b[0]) | (a[1]^b[1]) );
@@ -272,13 +281,39 @@ fd_funk_txn_xid_eq( fd_funk_txn_xid_t const * xa,
static inline fd_funk_txn_xid_t *
fd_funk_txn_xid_copy( fd_funk_txn_xid_t * xd,
- fd_funk_txn_xid_t const * xs ) {
+ fd_funk_txn_xid_t const * xs ) {
ulong * d = xd->ul;
ulong const * s = xs->ul;
d[0] = s[0]; d[1] = s[1];
return xd;
}
+static inline fd_funk_txn_xid_t *
+fd_funk_txn_xid_st_atomic( fd_funk_txn_xid_t * xd,
+ fd_funk_txn_xid_t const * xs ) {
+# if FD_HAS_X86
+ FD_VOLATILE( xd->xmm[0] ) = xs->xmm[0];
+# elif FD_HAS_INT128
+ FD_VOLATILE( xd->uf[0] ) = xs->uf[0];
+# else
+ fd_funk_txn_xid_copy( xd, xs );
+# endif
+ return xd;
+}
+
+static inline fd_funk_txn_xid_t *
+fd_funk_txn_xid_ld_atomic( fd_funk_txn_xid_t * xd,
+ fd_funk_txn_xid_t const * xs ) {
+# if FD_HAS_X86
+ xd->xmm[0] = FD_VOLATILE_CONST( xs->xmm[0] );
+# elif FD_HAS_INT128
+ xd->uf[0] = FD_VOLATILE_CONST( xs->uf[0] );
+# else
+ fd_funk_txn_xid_copy( xd, xs );
+# endif
+ return xd;
+}
+
/* fd_funk_txn_xid_eq_root returns 1 if transaction id pointed to by x
is the root transaction. Assumes x is in the caller's address space
and valid. */
diff --git a/src/funk/fd_funk_txn.h b/src/funk/fd_funk_txn.h
index 59a72a7db79..1df25e0b278 100644
--- a/src/funk/fd_funk_txn.h
+++ b/src/funk/fd_funk_txn.h
@@ -11,6 +11,7 @@
threads. */
#include "fd_funk_base.h"
+#include "../flamenco/fd_rwlock.h"
/* FD_FUNK_TXN_{ALIGN,FOOTPRINT} describe the alignment and footprint of
a fd_funk_txn_t. ALIGN will be a power of 2, footprint will be a
@@ -55,6 +56,8 @@ struct __attribute__((aligned(FD_FUNK_TXN_ALIGN))) fd_funk_txn_private {
uint rec_tail_idx; /* " last " */
uint state; /* one of FD_FUNK_TXN_STATE_* */
+
+ fd_rwlock_t lock[1];
};
typedef struct fd_funk_txn_private fd_funk_txn_t;
diff --git a/src/funk/test_funk_base.c b/src/funk/test_funk_base.c
index c9254fc0380..8c3d507fafb 100644
--- a/src/funk/test_funk_base.c
+++ b/src/funk/test_funk_base.c
@@ -10,19 +10,19 @@ FD_STATIC_ASSERT( FD_FUNK_ERR_REC ==-6,
FD_STATIC_ASSERT( FD_FUNK_ERR_MEM ==-7, unit_test );
FD_STATIC_ASSERT( FD_FUNK_REC_KEY_ALIGN ==8UL, unit_test );
-FD_STATIC_ASSERT( FD_FUNK_REC_KEY_FOOTPRINT ==40UL, unit_test );
+FD_STATIC_ASSERT( FD_FUNK_REC_KEY_FOOTPRINT ==32UL, unit_test );
FD_STATIC_ASSERT( FD_FUNK_REC_KEY_ALIGN ==alignof(fd_funk_rec_key_t), unit_test );
FD_STATIC_ASSERT( FD_FUNK_REC_KEY_FOOTPRINT ==sizeof (fd_funk_rec_key_t), unit_test );
-FD_STATIC_ASSERT( FD_FUNK_TXN_XID_ALIGN ==8UL, unit_test );
+FD_STATIC_ASSERT( FD_FUNK_TXN_XID_ALIGN ==16UL, unit_test );
FD_STATIC_ASSERT( FD_FUNK_TXN_XID_FOOTPRINT ==16UL, unit_test );
FD_STATIC_ASSERT( FD_FUNK_TXN_XID_ALIGN ==alignof(fd_funk_txn_xid_t), unit_test );
FD_STATIC_ASSERT( FD_FUNK_TXN_XID_FOOTPRINT ==sizeof (fd_funk_txn_xid_t), unit_test );
-FD_STATIC_ASSERT( FD_FUNK_XID_KEY_PAIR_ALIGN ==8UL, unit_test );
-FD_STATIC_ASSERT( FD_FUNK_XID_KEY_PAIR_FOOTPRINT==56UL, unit_test );
+FD_STATIC_ASSERT( FD_FUNK_XID_KEY_PAIR_ALIGN ==16UL, unit_test );
+FD_STATIC_ASSERT( FD_FUNK_XID_KEY_PAIR_FOOTPRINT==48UL, unit_test );
FD_STATIC_ASSERT( FD_FUNK_XID_KEY_PAIR_ALIGN ==alignof(fd_funk_xid_key_pair_t), unit_test );
FD_STATIC_ASSERT( FD_FUNK_XID_KEY_PAIR_FOOTPRINT==sizeof (fd_funk_xid_key_pair_t), unit_test );
@@ -39,7 +39,6 @@ fd_funk_rec_key_set_unique( fd_funk_rec_key_t * key ) {
# else
key->ul[3] = 0UL;
# endif
- key->ul[4] = ~key->ul[0];
return key;
}
diff --git a/src/funk/test_funk_common.h b/src/funk/test_funk_common.h
index 19261682fb9..3c554fe9a58 100644
--- a/src/funk/test_funk_common.h
+++ b/src/funk/test_funk_common.h
@@ -178,7 +178,6 @@ __attribute__((noinline)) static FD_FN_UNUSED fd_funk_rec_key_t *
key_set( fd_funk_rec_key_t * key,
ulong _key ) {
key->ul[0] = _key; key->ul[1] = _key+_key; key->ul[2] = _key*_key; key->ul[3] = -_key;
- key->ul[4] = _key*3U;
return key;
}