diff --git a/src/flamenco/runtime/fd_txncache.c b/src/flamenco/runtime/fd_txncache.c index f800474cc9..61c185c38a 100644 --- a/src/flamenco/runtime/fd_txncache.c +++ b/src/flamenco/runtime/fd_txncache.c @@ -1,6 +1,7 @@ #include "fd_txncache.h" #include "fd_txncache_private.h" #include "../../util/log/fd_log.h" +#include "program/fd_bpf_loader_program.h" struct blockcache { fd_txncache_blockcache_shmem_t * shmem; @@ -38,8 +39,11 @@ fd_txncache_align( void ) { } FD_FN_CONST ulong -fd_txncache_footprint( ulong max_live_slots ) { - ulong max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots; +fd_txncache_footprint_ext( ulong max_live_slots, + ulong max_blockhash_distance ) { + if( !max_live_slots ) return 0UL; + if( !max_blockhash_distance ) return 0UL; + ulong max_active_slots = max_blockhash_distance+max_live_slots; ulong l; l = FD_LAYOUT_INIT; @@ -151,7 +155,7 @@ fd_txncache_ensure_txnpage( fd_txncache_t * tc, if( FD_UNLIKELY( page_cnt==tc->shmem->txnpages_per_blockhash_max ) ) return NULL; if( FD_LIKELY( FD_ATOMIC_CAS( &blockcache->pages[ page_cnt ], UINT_MAX, UINT_MAX-1UL )==UINT_MAX ) ) { - ulong txnpages_free_cnt = tc->shmem->txnpages_free_cnt; + ulong txnpages_free_cnt = FD_VOLATILE_CONST( tc->shmem->txnpages_free_cnt ); for(;;) { if( FD_UNLIKELY( !txnpages_free_cnt ) ) return NULL; ulong old_txnpages_free_cnt = FD_ATOMIC_CAS( &tc->shmem->txnpages_free_cnt, (ushort)txnpages_free_cnt, (ushort)(txnpages_free_cnt-1UL) ); @@ -171,7 +175,7 @@ fd_txncache_ensure_txnpage( fd_txncache_t * tc, } else { uint txnpage_idx = blockcache->pages[ page_cnt ]; while( FD_UNLIKELY( txnpage_idx>=UINT_MAX-1UL ) ) { - txnpage_idx = blockcache->pages[ page_cnt ]; + txnpage_idx = FD_VOLATILE_CONST( blockcache->pages[ page_cnt ] ); FD_SPIN_PAUSE(); } return &tc->txnpages[ txnpage_idx ]; @@ -187,7 +191,7 @@ fd_txncache_insert_txn( fd_txncache_t * tc, ulong txnpage_idx = (ulong)(txnpage - tc->txnpages); for(;;) { - ushort txnpage_free = txnpage->free; + ushort txnpage_free = FD_VOLATILE_CONST( txnpage->free ); if( FD_UNLIKELY( !txnpage_free ) ) return 0; if( FD_UNLIKELY( FD_ATOMIC_CAS( &txnpage->free, txnpage_free, txnpage_free-1UL )!=txnpage_free ) ) { FD_SPIN_PAUSE(); @@ -202,7 +206,7 @@ fd_txncache_insert_txn( fd_txncache_t * tc, ulong txn_bucket = FD_LOAD( ulong, txnhash+txnhash_offset )%tc->shmem->txn_per_slot_max; for(;;) { - uint head = blockcache->heads[ txn_bucket ]; + uint head = FD_VOLATILE_CONST( blockcache->heads[ txn_bucket ] ); txnpage->txns[ txn_idx ]->blockcache_next = head; FD_COMPILER_MFENCE(); if( FD_LIKELY( FD_ATOMIC_CAS( &blockcache->heads[ txn_bucket ], head, (uint)(FD_TXNCACHE_TXNS_PER_PAGE*txnpage_idx+txn_idx) )==head ) ) break; diff --git a/src/flamenco/runtime/fd_txncache.h b/src/flamenco/runtime/fd_txncache.h index 19a82d27fd..d71ce79fc4 100644 --- a/src/flamenco/runtime/fd_txncache.h +++ b/src/flamenco/runtime/fd_txncache.h @@ -152,7 +152,13 @@ FD_FN_CONST ulong fd_txncache_align( void ); FD_FN_CONST ulong -fd_txncache_footprint( ulong max_live_slots ); +fd_txncache_footprint_ext( ulong max_live_slots, + ulong max_blockhash_distance ); + +FD_FN_CONST static inline ulong +fd_txncache_footprint( ulong max_live_slots ) { + return fd_txncache_footprint_ext( max_live_slots, FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE ); +} void * fd_txncache_new( void * ljoin, diff --git a/src/flamenco/runtime/fd_txncache_private.h b/src/flamenco/runtime/fd_txncache_private.h index 0d72b772fc..1961fafeec 100644 --- a/src/flamenco/runtime/fd_txncache_private.h +++ b/src/flamenco/runtime/fd_txncache_private.h @@ -10,13 +10,7 @@ and returning pages to the pool, but not so high that the memory wasted from blockhashes with only one transaction is significant. */ -#define FD_TXNCACHE_TXNS_PER_PAGE (16384UL) - -/* The maximum distance a transaction blockhash reference can be - (inclusive). For example, if no slots were skipped, and the value is - 151, slot 300 is allowed to reference blockhashes from slots - [149, 300). */ -#define FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE (151UL) +#define FD_TXNCACHE_TXNS_PER_PAGE (4UL) struct fd_txncache_single_txn { uint blockcache_next; /* Pointer to the next element in the blockcache hash chain containing this entry from the pool. */ @@ -118,6 +112,7 @@ struct __attribute__((aligned(FD_TXNCACHE_SHMEM_ALIGN))) fd_txncache_shmem_priva fd_rwlock_t lock[ 1 ] __attribute__((aligned(128UL))); ulong txn_per_slot_max; + ulong blockhash_distance_max; ulong active_slots_max; ushort txnpages_per_blockhash_max; ushort max_txnpages; diff --git a/src/flamenco/runtime/fd_txncache_shmem.c b/src/flamenco/runtime/fd_txncache_shmem.c index 3e393b4cb0..a4c42278d6 100644 --- a/src/flamenco/runtime/fd_txncache_shmem.c +++ b/src/flamenco/runtime/fd_txncache_shmem.c @@ -72,12 +72,15 @@ fd_txncache_shmem_align( void ) { } FD_FN_CONST ulong -fd_txncache_shmem_footprint( ulong max_live_slots, - ulong max_txn_per_slot ) { +fd_txncache_shmem_footprint_ext( ulong max_live_slots, + ulong max_blockhash_distance, + ulong max_txn_per_slot ) { if( FD_UNLIKELY( max_live_slots<1UL ) ) return 0UL; + if( FD_UNLIKELY( max_blockhash_distance<1UL ) ) return 0UL; if( FD_UNLIKELY( max_txn_per_slot<1UL ) ) return 0UL; - ulong max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots; + if( !max_blockhash_distance ) max_blockhash_distance = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE; + ulong max_active_slots = max_blockhash_distance+max_live_slots; ulong blockhash_map_chains = fd_ulong_pow2_up( 2UL*max_active_slots ); /* To save memory, txnpages are referenced as ushort which is enough @@ -102,9 +105,10 @@ fd_txncache_shmem_footprint( ulong max_live_slots, } void * -fd_txncache_shmem_new( void * shmem, - ulong max_live_slots, - ulong max_txn_per_slot ) { +fd_txncache_shmem_new_ext( void * shmem, + ulong max_live_slots, + ulong max_blockhash_distance, + ulong max_txn_per_slot ) { if( FD_UNLIKELY( !shmem ) ) { FD_LOG_WARNING(( "NULL shmem" )); return NULL; @@ -117,8 +121,9 @@ fd_txncache_shmem_new( void * shmem, if( FD_UNLIKELY( !max_live_slots ) ) return NULL; if( FD_UNLIKELY( !max_txn_per_slot ) ) return NULL; + if( !max_blockhash_distance ) max_blockhash_distance = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE; - ulong max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots; + ulong max_active_slots = max_blockhash_distance+max_live_slots; ulong blockhash_map_chains = fd_ulong_pow2_up( 2UL*max_active_slots ); ushort _max_txnpages = fd_txncache_max_txnpages( max_active_slots, max_txn_per_slot ); @@ -149,6 +154,7 @@ fd_txncache_shmem_new( void * shmem, tc->lock->value = 0; tc->txn_per_slot_max = max_txn_per_slot; + tc->blockhash_distance_max = max_blockhash_distance; tc->active_slots_max = max_active_slots; tc->txnpages_per_blockhash_max = _max_txnpages_per_blockhash; tc->max_txnpages = _max_txnpages; diff --git a/src/flamenco/runtime/fd_txncache_shmem.h b/src/flamenco/runtime/fd_txncache_shmem.h index 98a22730c7..edaad4ab70 100644 --- a/src/flamenco/runtime/fd_txncache_shmem.h +++ b/src/flamenco/runtime/fd_txncache_shmem.h @@ -7,6 +7,12 @@ #define FD_TXNCACHE_SHMEM_MAGIC (0xF17EDA2CE58CC4E0) /* FIREDANCE SMCCHE V0 */ +/* The maximum distance a transaction blockhash reference can be + (inclusive). For example, if no slots were skipped, and the value is + 151, slot 300 is allowed to reference blockhashes from slots + [149, 300). */ +#define FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE (151UL) + typedef struct { ushort val; } fd_txncache_fork_id_t; struct fd_txncache_shmem_private; @@ -18,13 +24,28 @@ FD_FN_CONST ulong fd_txncache_shmem_align( void ); FD_FN_CONST ulong +fd_txncache_shmem_footprint_ext( ulong max_live_slots, + ulong max_blockhash_distance, + ulong max_txn_per_slot ); + +FD_FN_CONST static inline ulong fd_txncache_shmem_footprint( ulong max_live_slots, - ulong max_txn_per_slot ); + ulong max_txn_per_slot ) { + return fd_txncache_shmem_footprint_ext( max_live_slots, FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE, max_txn_per_slot ); +} void * +fd_txncache_shmem_new_ext( void * shmem, + ulong max_live_slots, + ulong max_blockhash_distance, + ulong max_txn_per_slot ); + +static inline void * fd_txncache_shmem_new( void * shmem, ulong max_live_slots, - ulong max_txn_per_slot ); + ulong max_txn_per_slot ) { + return fd_txncache_shmem_new_ext( shmem, max_live_slots, FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE, max_txn_per_slot ); +} fd_txncache_shmem_t * fd_txncache_shmem_join( void * shtc ); diff --git a/src/flamenco/runtime/test_txncache_race.c b/src/flamenco/runtime/test_txncache_race.c new file mode 100644 index 0000000000..5565582646 --- /dev/null +++ b/src/flamenco/runtime/test_txncache_race.c @@ -0,0 +1,152 @@ +#include "fd_txncache.h" +#include "fd_txncache_shmem.h" +#include "../../util/racesan/fd_racesan_weave.h" +#include "../../util/fd_util.h" + +#if !FD_HAS_RACESAN +#error "test_txncache_race requires FD_HAS_RACESAN" +#endif + +struct test_env { + fd_txncache_shmem_t * shmem; + fd_txncache_t * tc; +}; +typedef struct test_env test_env_t; + +static test_env_t g_env[1]; + +static fd_wksp_t * g_wksp; + +static uchar wksp_mem[ 1UL<<26 ] __attribute__((aligned(FD_SHMEM_NORMAL_PAGE_SZ))); + +static test_env_t * +test_env_init( void ) { + FD_TEST( !g_wksp ); + ulong part_max = fd_wksp_part_max_est( sizeof(wksp_mem), 64UL<<10 ); + ulong data_max = fd_wksp_data_max_est( sizeof(wksp_mem), part_max ); + fd_wksp_t * wksp = fd_wksp_join( fd_wksp_new( wksp_mem, "funk_test", 1U, part_max, data_max ) ); + FD_TEST( wksp ); + fd_shmem_join_anonymous( "funk_test", FD_SHMEM_JOIN_MODE_READ_WRITE, wksp, wksp_mem, FD_SHMEM_NORMAL_PAGE_SZ, sizeof(wksp_mem)>>FD_SHMEM_NORMAL_LG_PAGE_SZ ); + + ulong wksp_tag = 1UL; + ulong max_live_slots = 1UL; + ulong max_bh_distance = 1UL; + ulong max_txn_per_slot = 64UL; + ulong shmem_fp = fd_txncache_shmem_footprint_ext( max_live_slots, max_bh_distance, max_txn_per_slot ); + FD_TEST( shmem_fp ); + void * shmem = fd_wksp_alloc_laddr( wksp, fd_txncache_shmem_align(), shmem_fp, wksp_tag ); + FD_TEST( shmem ); + FD_TEST( fd_txncache_shmem_new_ext( shmem, max_live_slots, max_bh_distance, max_txn_per_slot ) ); + + memset( g_env, 0, sizeof(test_env_t) ); + g_env->shmem = fd_txncache_shmem_join( shmem ); + FD_TEST( g_env->shmem ); + g_env->tc = fd_txncache_join( fd_txncache_new( fd_wksp_alloc_laddr( wksp, fd_txncache_align(), fd_txncache_footprint_ext( max_live_slots, max_bh_distance ), wksp_tag ), g_env->shmem ) ); + FD_TEST( g_env->tc ); + g_wksp = fd_wksp_join( wksp_mem ); + + return g_env; +} + +static void +test_env_fini( void ) { + FD_TEST( g_wksp ); + + //FD_TEST( fd_wksp_free_laddr( fd_txncache_delete( fd_txncache_leave( g_env->tc ) ) ) ); + fd_wksp_free_laddr( g_env->tc ); + //FD_TEST( fd_wksp_free_laddr( fd_txncache_shmem_delete( fd_txncache_shmem_leave( g_env->shmem ) ) ) ); + fd_wksp_free_laddr( g_env->shmem ); + + /* Check for alloc leaks */ + fd_wksp_usage_t wksp_usage; + FD_TEST( fd_wksp_usage( g_wksp, NULL, 0UL, &wksp_usage ) ); + FD_TEST( wksp_usage.free_cnt==wksp_usage.total_cnt ); + + memset( g_env, 0, sizeof(test_env_t) ); + fd_shmem_leave_anonymous( g_wksp, NULL ); + FD_TEST( fd_wksp_delete( fd_wksp_leave( g_wksp ) ) ); + g_wksp = NULL; +} + +struct async_ctx { + fd_txncache_t * tc; + fd_txncache_fork_id_t fork_id; + uchar blockhash[ 32 ]; + uchar txnhash[ 32 ]; +}; +typedef struct async_ctx async_ctx_t; + +static void +async_insert( void * _ctx ) { + async_ctx_t * ctx = _ctx; + fd_txncache_insert( ctx->tc, ctx->fork_id, ctx->blockhash, ctx->txnhash ); +} + +static void +async_query( void * _ctx ) { + async_ctx_t * ctx = _ctx; + (void)fd_txncache_query( ctx->tc, ctx->fork_id, ctx->blockhash, ctx->txnhash ); +} + +static void +race_insert_query( void ) { + test_env_t * env = test_env_init(); + + fd_racesan_weave_t weave[1]; + fd_racesan_weave_new( weave ); + +# define OP_CNT 33 + async_ctx_t op_ctx[ OP_CNT ]; + for( ulong i=0UL; itc; + memset( op_ctx[ i ].blockhash, 0, 32UL ); + memset( op_ctx[ i ].txnhash, 0, 32UL ); + FD_STORE( ulong, op_ctx[ i ].txnhash, i ); + } + static fd_racesan_async_t insert_async[ OP_CNT ]; + for( ulong i=0UL; itc ); + fd_txncache_fork_id_t null = {USHORT_MAX}; + fd_txncache_fork_id_t root = fd_txncache_attach_child( env->tc, null ); + fd_txncache_finalize_fork( env->tc, root, 0UL, op_ctx[0].blockhash ); + fd_txncache_fork_id_t slot1 = fd_txncache_attach_child( env->tc, root ); + for( ulong i=0UL; itc, slot1, 0UL, op_ctx[0].blockhash ); + for( ulong i=0UL; itc, slot1, op_ctx[ i ].blockhash, op_ctx[ i ].txnhash ) ); + } + } + + fd_racesan_weave_delete( weave ); + for( ulong i=0UL; ihook_ctx = async; fd_racesan_enter( racesan ); async->fn( async->fn_ctx ); - fd_racesan_delete( racesan ); async->done = 1; fd_racesan_async_yield( async ); diff --git a/src/util/racesan/fd_racesan_weave.h b/src/util/racesan/fd_racesan_weave.h index e930190bf2..659565924c 100644 --- a/src/util/racesan/fd_racesan_weave.h +++ b/src/util/racesan/fd_racesan_weave.h @@ -5,7 +5,7 @@ #include "fd_racesan_async.h" -#define FD_RACESAN_WEAVE_MAX (16UL) +#define FD_RACESAN_WEAVE_MAX (256UL) struct fd_racesan_weave { fd_racesan_async_t * async[ FD_RACESAN_WEAVE_MAX ];