txncache: add racesan

src/flamenco/runtime/fd_txncache.c

@@ -1,6 +1,7 @@
 #include "fd_txncache.h"
 #include "fd_txncache_private.h"
 #include "../../util/log/fd_log.h"
+#include "program/fd_bpf_loader_program.h"
 
 struct blockcache {
   fd_txncache_blockcache_shmem_t * shmem;
@@ -38,8 +39,11 @@ fd_txncache_align( void ) {
 }
 
 FD_FN_CONST ulong
-fd_txncache_footprint( ulong max_live_slots ) {
-  ulong max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots;
+fd_txncache_footprint_ext( ulong max_live_slots,
+                           ulong max_blockhash_distance ) {
+  if( !max_live_slots         ) return 0UL;
+  if( !max_blockhash_distance ) return 0UL;
+  ulong max_active_slots = max_blockhash_distance+max_live_slots;
 
   ulong l;
   l = FD_LAYOUT_INIT;
@@ -151,7 +155,7 @@ fd_txncache_ensure_txnpage( fd_txncache_t * tc,
 
   if( FD_UNLIKELY( page_cnt==tc->shmem->txnpages_per_blockhash_max ) ) return NULL;
   if( FD_LIKELY( FD_ATOMIC_CAS( &blockcache->pages[ page_cnt ], UINT_MAX, UINT_MAX-1UL )==UINT_MAX ) ) {
-    ulong txnpages_free_cnt = tc->shmem->txnpages_free_cnt;
+    ulong txnpages_free_cnt = FD_VOLATILE_CONST( tc->shmem->txnpages_free_cnt );
     for(;;) {
       if( FD_UNLIKELY( !txnpages_free_cnt ) ) return NULL;
       ulong old_txnpages_free_cnt = FD_ATOMIC_CAS( &tc->shmem->txnpages_free_cnt, (ushort)txnpages_free_cnt, (ushort)(txnpages_free_cnt-1UL) );
@@ -171,7 +175,7 @@ fd_txncache_ensure_txnpage( fd_txncache_t * tc,
   } else {
     uint txnpage_idx = blockcache->pages[ page_cnt ];
     while( FD_UNLIKELY( txnpage_idx>=UINT_MAX-1UL ) ) {
-      txnpage_idx = blockcache->pages[ page_cnt ];
+      txnpage_idx = FD_VOLATILE_CONST( blockcache->pages[ page_cnt ] );
       FD_SPIN_PAUSE();
     }
     return &tc->txnpages[ txnpage_idx ];
@@ -187,7 +191,7 @@ fd_txncache_insert_txn( fd_txncache_t *         tc,
   ulong txnpage_idx = (ulong)(txnpage - tc->txnpages);
 
   for(;;) {
-    ushort txnpage_free = txnpage->free;
+    ushort txnpage_free = FD_VOLATILE_CONST( txnpage->free );
     if( FD_UNLIKELY( !txnpage_free ) ) return 0;
     if( FD_UNLIKELY( FD_ATOMIC_CAS( &txnpage->free, txnpage_free, txnpage_free-1UL )!=txnpage_free ) ) {
       FD_SPIN_PAUSE();
@@ -202,7 +206,7 @@ fd_txncache_insert_txn( fd_txncache_t *         tc,
 
     ulong txn_bucket = FD_LOAD( ulong, txnhash+txnhash_offset )%tc->shmem->txn_per_slot_max;
     for(;;) {
-      uint head = blockcache->heads[ txn_bucket ];
+      uint head = FD_VOLATILE_CONST( blockcache->heads[ txn_bucket ] );
       txnpage->txns[ txn_idx ]->blockcache_next = head;
       FD_COMPILER_MFENCE();
       if( FD_LIKELY( FD_ATOMIC_CAS( &blockcache->heads[ txn_bucket ], head, (uint)(FD_TXNCACHE_TXNS_PER_PAGE*txnpage_idx+txn_idx) )==head ) ) break;

src/flamenco/runtime/fd_txncache.h

@@ -152,7 +152,13 @@ FD_FN_CONST ulong
 fd_txncache_align( void );
 
 FD_FN_CONST ulong
-fd_txncache_footprint( ulong max_live_slots );
+fd_txncache_footprint_ext( ulong max_live_slots,
+                           ulong max_blockhash_distance );
+
+FD_FN_CONST static inline ulong
+fd_txncache_footprint( ulong max_live_slots ) {
+  return fd_txncache_footprint_ext( max_live_slots, FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE );
+}
 
 void *
 fd_txncache_new( void *                ljoin,

src/flamenco/runtime/fd_txncache_private.h

@@ -10,13 +10,7 @@
    and returning pages to the pool, but not so high that the memory
    wasted from blockhashes with only one transaction is significant. */
 
-#define FD_TXNCACHE_TXNS_PER_PAGE (16384UL)
-
-/* The maximum distance a transaction blockhash reference can be
-   (inclusive).  For example, if no slots were skipped, and the value is
-   151, slot 300 is allowed to reference blockhashes from slots
-   [149, 300). */
-#define FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE (151UL)
+#define FD_TXNCACHE_TXNS_PER_PAGE (4UL)
 
 struct fd_txncache_single_txn {
   uint  blockcache_next; /* Pointer to the next element in the blockcache hash chain containing this entry from the pool. */
@@ -118,6 +112,7 @@ struct __attribute__((aligned(FD_TXNCACHE_SHMEM_ALIGN))) fd_txncache_shmem_priva
   fd_rwlock_t lock[ 1 ] __attribute__((aligned(128UL)));
 
   ulong  txn_per_slot_max;
+  ulong  blockhash_distance_max;
   ulong  active_slots_max;
   ushort txnpages_per_blockhash_max;
   ushort max_txnpages;

src/flamenco/runtime/fd_txncache_shmem.c

@@ -72,12 +72,15 @@ fd_txncache_shmem_align( void ) {
 }
 
 FD_FN_CONST ulong
-fd_txncache_shmem_footprint( ulong max_live_slots,
-                             ulong max_txn_per_slot ) {
+fd_txncache_shmem_footprint_ext( ulong max_live_slots,
+                                 ulong max_blockhash_distance,
+                                 ulong max_txn_per_slot ) {
   if( FD_UNLIKELY( max_live_slots<1UL ) ) return 0UL;
+  if( FD_UNLIKELY( max_blockhash_distance<1UL ) ) return 0UL;
   if( FD_UNLIKELY( max_txn_per_slot<1UL ) ) return 0UL;
 
-  ulong max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots;
+  if( !max_blockhash_distance ) max_blockhash_distance = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE;
+  ulong max_active_slots = max_blockhash_distance+max_live_slots;
   ulong blockhash_map_chains = fd_ulong_pow2_up( 2UL*max_active_slots );
 
   /* To save memory, txnpages are referenced as ushort which is enough
@@ -102,9 +105,10 @@ fd_txncache_shmem_footprint( ulong max_live_slots,
 }
 
 void *
-fd_txncache_shmem_new( void * shmem,
-                       ulong  max_live_slots,
-                       ulong  max_txn_per_slot ) {
+fd_txncache_shmem_new_ext( void * shmem,
+                           ulong  max_live_slots,
+                           ulong  max_blockhash_distance,
+                           ulong  max_txn_per_slot ) {
   if( FD_UNLIKELY( !shmem ) ) {
     FD_LOG_WARNING(( "NULL shmem" ));
     return NULL;
@@ -117,8 +121,9 @@ fd_txncache_shmem_new( void * shmem,
 
   if( FD_UNLIKELY( !max_live_slots ) ) return NULL;
   if( FD_UNLIKELY( !max_txn_per_slot ) ) return NULL;
+  if( !max_blockhash_distance ) max_blockhash_distance = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE;
 
-  ulong max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots;
+  ulong max_active_slots = max_blockhash_distance+max_live_slots;
   ulong blockhash_map_chains = fd_ulong_pow2_up( 2UL*max_active_slots );
 
   ushort _max_txnpages               = fd_txncache_max_txnpages( max_active_slots, max_txn_per_slot );
@@ -149,6 +154,7 @@ fd_txncache_shmem_new( void * shmem,
   tc->lock->value = 0;
 
   tc->txn_per_slot_max           = max_txn_per_slot;
+  tc->blockhash_distance_max     = max_blockhash_distance;
   tc->active_slots_max           = max_active_slots;
   tc->txnpages_per_blockhash_max = _max_txnpages_per_blockhash;
   tc->max_txnpages               = _max_txnpages;

src/flamenco/runtime/fd_txncache_shmem.h

@@ -7,6 +7,12 @@
 
 #define FD_TXNCACHE_SHMEM_MAGIC (0xF17EDA2CE58CC4E0) /* FIREDANCE SMCCHE V0 */
 
+/* The maximum distance a transaction blockhash reference can be
+   (inclusive).  For example, if no slots were skipped, and the value is
+   151, slot 300 is allowed to reference blockhashes from slots
+   [149, 300). */
+#define FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE (151UL)
+
 typedef struct { ushort val; } fd_txncache_fork_id_t;
 
 struct fd_txncache_shmem_private;
@@ -18,13 +24,28 @@ FD_FN_CONST ulong
 fd_txncache_shmem_align( void );
 
 FD_FN_CONST ulong
+fd_txncache_shmem_footprint_ext( ulong max_live_slots,
+                                 ulong max_blockhash_distance,
+                                 ulong max_txn_per_slot );
+
+FD_FN_CONST static inline ulong
 fd_txncache_shmem_footprint( ulong max_live_slots,
-                             ulong max_txn_per_slot );
+                             ulong max_txn_per_slot ) {
+  return fd_txncache_shmem_footprint_ext( max_live_slots, FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE, max_txn_per_slot );
+}
 
 void *
+fd_txncache_shmem_new_ext( void * shmem,
+                           ulong  max_live_slots,
+                           ulong  max_blockhash_distance,
+                           ulong  max_txn_per_slot );
+
+static inline void *
 fd_txncache_shmem_new( void * shmem,
                        ulong  max_live_slots,
-                       ulong  max_txn_per_slot );
+                       ulong  max_txn_per_slot ) {
+  return fd_txncache_shmem_new_ext( shmem, max_live_slots, FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE, max_txn_per_slot );
+}
 
 fd_txncache_shmem_t *
 fd_txncache_shmem_join( void * shtc );

src/flamenco/runtime/test_txncache_race.c

@@ -0,0 +1,152 @@
+#include "fd_txncache.h"
+#include "fd_txncache_shmem.h"
+#include "../../util/racesan/fd_racesan_weave.h"
+#include "../../util/fd_util.h"
+
+#if !FD_HAS_RACESAN
+#error "test_txncache_race requires FD_HAS_RACESAN"
+#endif
+
+struct test_env {
+  fd_txncache_shmem_t * shmem;
+  fd_txncache_t *       tc;
+};
+typedef struct test_env test_env_t;
+
+static test_env_t g_env[1];
+
+static fd_wksp_t * g_wksp;
+
+static uchar wksp_mem[ 1UL<<26 ] __attribute__((aligned(FD_SHMEM_NORMAL_PAGE_SZ)));
+
+static test_env_t *
+test_env_init( void ) {
+  FD_TEST( !g_wksp );
+  ulong part_max = fd_wksp_part_max_est( sizeof(wksp_mem), 64UL<<10 );
+  ulong data_max = fd_wksp_data_max_est( sizeof(wksp_mem), part_max );
+  fd_wksp_t * wksp = fd_wksp_join( fd_wksp_new( wksp_mem, "funk_test", 1U, part_max, data_max ) );
+  FD_TEST( wksp );
+  fd_shmem_join_anonymous( "funk_test", FD_SHMEM_JOIN_MODE_READ_WRITE, wksp, wksp_mem, FD_SHMEM_NORMAL_PAGE_SZ, sizeof(wksp_mem)>>FD_SHMEM_NORMAL_LG_PAGE_SZ );
+
+  ulong  wksp_tag         =  1UL;
+  ulong  max_live_slots   =  1UL;
+  ulong  max_bh_distance  =  1UL;
+  ulong  max_txn_per_slot = 64UL;
+  ulong  shmem_fp = fd_txncache_shmem_footprint_ext( max_live_slots, max_bh_distance, max_txn_per_slot );
+  FD_TEST( shmem_fp );
+  void * shmem = fd_wksp_alloc_laddr( wksp, fd_txncache_shmem_align(), shmem_fp, wksp_tag );
+  FD_TEST( shmem );
+  FD_TEST( fd_txncache_shmem_new_ext( shmem, max_live_slots, max_bh_distance, max_txn_per_slot ) );
+
+  memset( g_env, 0, sizeof(test_env_t) );
+  g_env->shmem = fd_txncache_shmem_join( shmem );
+  FD_TEST( g_env->shmem );
+  g_env->tc = fd_txncache_join( fd_txncache_new( fd_wksp_alloc_laddr( wksp, fd_txncache_align(), fd_txncache_footprint_ext( max_live_slots, max_bh_distance ), wksp_tag ), g_env->shmem ) );
+  FD_TEST( g_env->tc );
+  g_wksp = fd_wksp_join( wksp_mem );
+
+  return g_env;
+}
+
+static void
+test_env_fini( void ) {
+  FD_TEST( g_wksp );
+
+  //FD_TEST( fd_wksp_free_laddr( fd_txncache_delete( fd_txncache_leave( g_env->tc ) ) ) );
+  fd_wksp_free_laddr( g_env->tc );
+  //FD_TEST( fd_wksp_free_laddr( fd_txncache_shmem_delete( fd_txncache_shmem_leave( g_env->shmem ) ) ) );
+  fd_wksp_free_laddr( g_env->shmem );
+
+  /* Check for alloc leaks */
+  fd_wksp_usage_t wksp_usage;
+  FD_TEST( fd_wksp_usage( g_wksp, NULL, 0UL, &wksp_usage ) );
+  FD_TEST( wksp_usage.free_cnt==wksp_usage.total_cnt );
+
+  memset( g_env, 0, sizeof(test_env_t) );
+  fd_shmem_leave_anonymous( g_wksp, NULL );
+  FD_TEST( fd_wksp_delete( fd_wksp_leave( g_wksp ) ) );
+  g_wksp = NULL;
+}
+
+struct async_ctx {
+  fd_txncache_t *       tc;
+  fd_txncache_fork_id_t fork_id;
+  uchar                 blockhash[ 32 ];
+  uchar                 txnhash[ 32 ];
+};
+typedef struct async_ctx async_ctx_t;
+
+static void
+async_insert( void * _ctx ) {
+  async_ctx_t * ctx = _ctx;
+  fd_txncache_insert( ctx->tc, ctx->fork_id, ctx->blockhash, ctx->txnhash );
+}
+
+static void
+async_query( void * _ctx ) {
+  async_ctx_t * ctx = _ctx;
+  (void)fd_txncache_query( ctx->tc, ctx->fork_id, ctx->blockhash, ctx->txnhash );
+}
+
+static void
+race_insert_query( void ) {
+  test_env_t * env = test_env_init();
+
+  fd_racesan_weave_t weave[1];
+  fd_racesan_weave_new( weave );
+
+# define OP_CNT 33
+  async_ctx_t op_ctx[ OP_CNT ];
+  for( ulong i=0UL; i<OP_CNT; i++ ) {
+    op_ctx[ i ].tc = env->tc;
+    memset( op_ctx[ i ].blockhash, 0, 32UL );
+    memset( op_ctx[ i ].txnhash,   0, 32UL );
+    FD_STORE( ulong, op_ctx[ i ].txnhash, i );
+  }
+  static fd_racesan_async_t insert_async[ OP_CNT ];
+  for( ulong i=0UL; i<OP_CNT; i++ ) {
+    FD_TEST( fd_racesan_async_new( &insert_async[i], async_insert, &op_ctx[i] ) );
+    fd_racesan_weave_add( weave, &insert_async[i] );
+  }
+
+  (void)async_query;
+  // static fd_racesan_async_t query_async[ OP_CNT ];
+  // for( ulong i=0UL; i<OP_CNT; i++ ) {
+  //   FD_TEST( fd_racesan_async_new( &query_async[i], async_query, &op_ctx[i] ) );
+  //   fd_racesan_weave_add( weave, &query_async[i] );
+  // }
+
+  ulong iter     = (ulong)1e5;
+  ulong step_max = 1024UL;
+  for( ulong rem=iter; rem; rem-- ) {
+    fd_txncache_reset( env->tc );
+    fd_txncache_fork_id_t null = {USHORT_MAX};
+    fd_txncache_fork_id_t root = fd_txncache_attach_child( env->tc, null );
+    fd_txncache_finalize_fork( env->tc, root, 0UL, op_ctx[0].blockhash );
+    fd_txncache_fork_id_t slot1 = fd_txncache_attach_child( env->tc, root );
+    for( ulong i=0UL; i<OP_CNT; i++ ) op_ctx[i].fork_id = slot1;
+    fd_racesan_weave_exec_rand( weave, rem, step_max );
+    fd_txncache_finalize_fork( env->tc, slot1, 0UL, op_ctx[0].blockhash );
+    for( ulong i=0UL; i<OP_CNT; i++ ) {
+      FD_TEST( fd_txncache_query( env->tc, slot1, op_ctx[ i ].blockhash, op_ctx[ i ].txnhash ) );
+    }
+  }
+
+  fd_racesan_weave_delete( weave );
+  for( ulong i=0UL; i<OP_CNT; i++ ) fd_racesan_async_delete( &insert_async[i] );
+
+  test_env_fini();
+# undef OP_CNT
+}
+
+int
+main( int     argc,
+      char ** argv ) {
+  fd_boot( &argc, &argv );
+
+  race_insert_query();
+
+  FD_LOG_NOTICE(( "pass" ));
+  fd_halt();
+  return 0;
+}

src/util/racesan/fd_racesan_async.c

@@ -30,12 +30,15 @@ fd_racesan_async_hook( void * ctx,
 
 static void
 fd_racesan_async_target( fd_racesan_async_t * async ) {
-  fd_racesan_t racesan[1];
-  fd_racesan_new( racesan, async );
-  fd_racesan_inject_default( racesan, fd_racesan_async_hook );
+  static fd_racesan_t racesan[1];
+  FD_ONCE_BEGIN {
+    fd_racesan_new( racesan, NULL );
+    fd_racesan_inject_default( racesan, fd_racesan_async_hook );
+  }
+  FD_ONCE_END;
+  racesan->hook_ctx = async;
   fd_racesan_enter( racesan );
   async->fn( async->fn_ctx );
-  fd_racesan_delete( racesan );
 
   async->done = 1;
   fd_racesan_async_yield( async );

src/util/racesan/fd_racesan_weave.h

@@ -5,7 +5,7 @@
 
 #include "fd_racesan_async.h"
 
-#define FD_RACESAN_WEAVE_MAX (16UL)
+#define FD_RACESAN_WEAVE_MAX (256UL)
 
 struct fd_racesan_weave {
   fd_racesan_async_t * async[ FD_RACESAN_WEAVE_MAX ];