gui: leader schedule + peer vote info

Author: jherrera-jump

book/api/websocket.md

@@ -257,6 +257,7 @@ fd_topo_initialize( config_t * config ) {
 
   fd_topob_wksp( topo, "shred_out"    );
   fd_topob_wksp( topo, "replay_stake" );
+  fd_topob_wksp( topo, "replay_votes" );
   fd_topob_wksp( topo, "replay_exec"  );
   fd_topob_wksp( topo, "replay_out"   );
   fd_topob_wksp( topo, "tower_out"    );
@@ -344,6 +345,9 @@ fd_topo_initialize( config_t * config ) {
   /**/                 fd_topob_link( topo, "dedup_resolv", "dedup_resolv", 65536UL,                                  FD_TPU_PARSED_MTU,             1UL );
   FOR(resolv_tile_cnt) fd_topob_link( topo, "resolv_pack",  "resolv_pack",  65536UL,                                  FD_TPU_RESOLVED_MTU,           1UL );
   /**/                 fd_topob_link( topo, "replay_stake", "replay_stake", 128UL,                                    FD_STAKE_OUT_MTU,              1UL ); /* TODO: This should be 2 but requires fixing STEM_BURST */
+  if( FD_LIKELY( config->tiles.gui.enabled ) ) { /* the gui, which is optional, is the only consumer of replay_votes */
+                       fd_topob_link( topo, "replay_votes", "replay_votes", 128UL,                                    sizeof(fd_replay_votes_t),     1UL );
+  }
   /**/                 fd_topob_link( topo, "replay_out",   "replay_out",   8192UL,                                   sizeof(fd_replay_message_t),   1UL );
   /**/                 fd_topob_link( topo, "pack_poh",     "pack_poh",     128UL,                                    sizeof(fd_done_packing_t),     1UL );
   /* pack_bank is shared across all banks, so if one bank stalls due to complex transactions, the buffer neeeds to be large so that
@@ -503,6 +507,9 @@ fd_topo_initialize( config_t * config ) {
   /**/                 fd_topob_tile_in (   topo, "replay",  0UL,          "metric_in", "genesi_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_out(   topo, "replay",  0UL,                       "replay_out",   0UL                                                );
   /**/                 fd_topob_tile_out(   topo, "replay",  0UL,                       "replay_stake", 0UL                                                );
+  if( FD_LIKELY( config->tiles.gui.enabled ) ) { /* the gui, which is optional, is the only consumer of replay_votes */
+                       fd_topob_tile_out(   topo, "replay",  0UL,                       "replay_votes", 0UL                                                );
+  }
   /**/                 fd_topob_tile_out(   topo, "replay",  0UL,                       "executed_txn", 0UL                                                );
   /**/                 fd_topob_tile_out(   topo, "replay",  0UL,                       "replay_exec",  0UL                                                );
   /**/                 fd_topob_tile_in (   topo, "replay",  0UL,          "metric_in", "tower_out",    0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
@@ -668,6 +675,9 @@ fd_topo_initialize( config_t * config ) {
     /**/                 fd_topob_tile_in(  topo, "gui",    0UL,           "metric_in", "gossip_net",   0UL,          FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
     /**/                 fd_topob_tile_in(  topo, "gui",    0UL,           "metric_in", "gossip_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
     /**/                 fd_topob_tile_in(  topo, "gui",    0UL,           "metric_in", "replay_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+    /**/                 fd_topob_tile_in ( topo, "gui",    0UL,           "metric_in", "replay_stake", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+    /**/                 fd_topob_tile_in ( topo, "gui",    0UL,           "metric_in", "replay_votes", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+    /**/                 fd_topob_tile_in(  topo, "gui",    0UL,           "metric_in", "tower_out",    0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
 
     if( FD_LIKELY( snapshots_enabled ) )
                          fd_topob_tile_in ( topo, "gui",    0UL,           "metric_in", "snaprd_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );

src/app/firedancer/topology.c

@@ -221,6 +221,16 @@ fd_gui_ws_open( fd_gui_t * gui,
     FD_TEST( !fd_http_server_ws_send( gui->http, ws_conn_id ) );
   }
 
+  /* todo .. temporary workaround to skip the blur until frontend boot
+     screen lands */
+  if( FD_UNLIKELY( gui->summary.is_full_client ) ) {
+    uchar prev = gui->summary.startup_progress.phase;
+    gui->summary.startup_progress.phase = FD_GUI_START_PROGRESS_TYPE_RUNNING;
+    fd_gui_printf_startup_progress( gui );
+    FD_TEST( !fd_http_server_ws_send( gui->http, ws_conn_id ) );
+    gui->summary.startup_progress.phase = prev;
+  }
+
   if( FD_LIKELY( gui->block_engine.has_block_engine ) ) {
     fd_gui_printf_block_engine( gui );
     FD_TEST( !fd_http_server_ws_send( gui->http, ws_conn_id ) );
@@ -632,8 +642,7 @@ fd_gui_run_boot_progress( fd_gui_t * gui, long now ) {
         gui->summary.boot_progress.phase = FD_GUI_BOOT_PROGRESS_TYPE_LOADING_INCREMENTAL_SNAPSHOT;
         gui->summary.boot_progress.loading_snapshot[ FD_GUI_BOOT_PROGRESS_INCREMENTAL_SNAPSHOT_IDX ].sample_time_nanos = now;
       }
-      int slot_turbine_hist_full = gui->summary.slots_max_turbine[ FD_GUI_TURBINE_SLOT_HISTORY_SZ-1 ].slot!=ULONG_MAX;
-      if( FD_UNLIKELY( snapshot_phase == 12UL && slot_turbine_hist_full && gui->summary.slot_completed!=ULONG_MAX ) ) {
+      if( FD_UNLIKELY( snapshot_phase == 12UL && gui->summary.slots_max_turbine[ 0 ].slot!=ULONG_MAX && gui->summary.slot_completed!=ULONG_MAX ) ) {
         gui->summary.boot_progress.phase                           = FD_GUI_BOOT_PROGRESS_TYPE_CATCHING_UP;
         gui->summary.boot_progress.catching_up_time_nanos          = now;
       }
@@ -1344,56 +1353,49 @@ fd_gui_clear_slot( fd_gui_t * gui,
   }
 }
 
-static void
-fd_gui_handle_leader_schedule( fd_gui_t *    gui,
-                               ulong const * msg,
-                               long          now ) {
-  ulong epoch               = msg[ 0 ];
-  ulong staked_cnt          = msg[ 1 ];
-  ulong start_slot          = msg[ 2 ];
-  ulong slot_cnt            = msg[ 3 ];
-  ulong excluded_stake      = msg[ 4 ];
-  ulong vote_keyed_lsched   = msg[ 5 ];
-
-  FD_TEST( staked_cnt<=MAX_STAKED_LEADERS );
-  FD_TEST( slot_cnt<=MAX_SLOTS_PER_EPOCH );
-
-  ulong idx = epoch % 2UL;
+void
+fd_gui_handle_leader_schedule( fd_gui_t *                    gui,
+                               fd_stake_weight_msg_t const * leader_schedule,
+                               long                          now ) {
+  FD_TEST( leader_schedule->staked_cnt<=MAX_STAKED_LEADERS );
+  FD_TEST( leader_schedule->slot_cnt<=MAX_SLOTS_PER_EPOCH );
+
+  ulong idx = leader_schedule->epoch % 2UL;
   gui->epoch.has_epoch[ idx ] = 1;
 
-  gui->epoch.epochs[ idx ].epoch            = epoch;
-  gui->epoch.epochs[ idx ].start_slot       = start_slot;
-  gui->epoch.epochs[ idx ].end_slot         = start_slot + slot_cnt - 1; // end_slot is inclusive.
-  gui->epoch.epochs[ idx ].excluded_stake   = excluded_stake;
+  gui->epoch.epochs[ idx ].epoch            = leader_schedule->epoch;
+  gui->epoch.epochs[ idx ].start_slot       = leader_schedule->start_slot;
+  gui->epoch.epochs[ idx ].end_slot         = leader_schedule->start_slot + leader_schedule->slot_cnt - 1; // end_slot is inclusive.
+  gui->epoch.epochs[ idx ].excluded_stake   = leader_schedule->excluded_stake;
   gui->epoch.epochs[ idx ].my_total_slots   = 0UL;
   gui->epoch.epochs[ idx ].my_skipped_slots = 0UL;
 
   memset( gui->epoch.epochs[ idx ].rankings,    (int)(UINT_MAX), sizeof(gui->epoch.epochs[ idx ].rankings)    );
   memset( gui->epoch.epochs[ idx ].my_rankings, (int)(UINT_MAX), sizeof(gui->epoch.epochs[ idx ].my_rankings) );
 
-  gui->epoch.epochs[ idx ].rankings_slot = start_slot;
+  gui->epoch.epochs[ idx ].rankings_slot = leader_schedule->start_slot;
 
-  fd_vote_stake_weight_t const * stake_weights = fd_type_pun_const( msg+6UL );
-  memcpy( gui->epoch.epochs[ idx ].stakes, stake_weights, staked_cnt*sizeof(fd_vote_stake_weight_t) );
+  fd_vote_stake_weight_t const * stake_weights = leader_schedule->weights;
+  fd_memcpy( gui->epoch.epochs[ idx ].stakes, stake_weights, leader_schedule->staked_cnt*sizeof(fd_vote_stake_weight_t) );
 
   fd_epoch_leaders_delete( fd_epoch_leaders_leave( gui->epoch.epochs[ idx ].lsched ) );
   gui->epoch.epochs[idx].lsched = fd_epoch_leaders_join( fd_epoch_leaders_new( gui->epoch.epochs[ idx ]._lsched,
-                                                                               epoch,
+                                                                               leader_schedule->epoch,
                                                                                gui->epoch.epochs[ idx ].start_slot,
-                                                                               slot_cnt,
-                                                                               staked_cnt,
+                                                                               leader_schedule->slot_cnt,
+                                                                               leader_schedule->staked_cnt,
                                                                                gui->epoch.epochs[ idx ].stakes,
-                                                                               excluded_stake,
-                                                                               vote_keyed_lsched ) );
+                                                                               leader_schedule->excluded_stake,
+                                                                               leader_schedule->vote_keyed_lsched ) );
 
-  if( FD_UNLIKELY( start_slot==0UL ) ) {
+  if( FD_UNLIKELY( leader_schedule->start_slot==0UL ) ) {
     gui->epoch.epochs[ 0 ].start_time = now;
   } else {
     gui->epoch.epochs[ idx ].start_time = LONG_MAX;
 
-    for( ulong i=0UL; i<fd_ulong_min( start_slot-1UL, FD_GUI_SLOTS_CNT ); i++ ) {
-      fd_gui_slot_t * slot = gui->slots[ (start_slot-i) % FD_GUI_SLOTS_CNT ];
-      if( FD_UNLIKELY( slot->slot!=(start_slot-i) ) ) break;
+    for( ulong i=0UL; i<fd_ulong_min( leader_schedule->start_slot-1UL, FD_GUI_SLOTS_CNT ); i++ ) {
+      fd_gui_slot_t * slot = gui->slots[ (leader_schedule->start_slot-i) % FD_GUI_SLOTS_CNT ];
+      if( FD_UNLIKELY( slot->slot!=(leader_schedule->start_slot-i) ) ) break;
       else if( FD_UNLIKELY( slot->skipped ) ) continue;
 
       gui->epoch.epochs[ idx ].start_time = slot->completed_time;
@@ -1773,8 +1775,7 @@ fd_gui_handle_rooted_slot( fd_gui_t * gui,
     if( FD_UNLIKELY( slot->slot==ULONG_MAX) ) break;
 
     if( FD_UNLIKELY( slot->slot!=parent_slot ) ) {
-      FD_LOG_ERR(( "_slot %lu i %lu we expect parent_slot %lu got slot->slot %lu", _slot, i, parent_slot, slot->slot ));
-    }
+      FD_LOG_ERR(( "_slot %lu i %lu we expect parent_slot %lu got slot->slot %lu", _slot, i, parent_slot, slot->slot ));    }
     if( FD_UNLIKELY( slot->level>=FD_GUI_SLOT_LEVEL_ROOTED ) ) break;
 
     slot->level = FD_GUI_SLOT_LEVEL_ROOTED;
@@ -2069,7 +2070,8 @@ fd_gui_handle_tower_update( fd_gui_t *                   gui,
                             long                         now ) {
   (void)now;
 
-  if( FD_LIKELY( gui->summary.slot_rooted!=tower->root_slot ) ) {
+  /* rooted slot */
+  if( FD_LIKELY( tower->root_slot!=ULONG_MAX && gui->summary.slot_rooted!=tower->root_slot ) ) {
     /* Move any shred data up to and including _slot from staging array
        to history array */
     ulong start_slot = fd_ulong_if( gui->shreds.history_slot!=ULONG_MAX, gui->shreds.history_slot+1UL, tower->root_slot );
@@ -2112,11 +2114,71 @@ fd_gui_handle_tower_update( fd_gui_t *                   gui,
        gui->shreds.staged_next_broadcast to keep it in the bounds of the
        ring buffer */
     gui->shreds.staged_next_broadcast = fd_ulong_max( gui->shreds.staged_head, gui->shreds.staged_next_broadcast );
+
+    gui->summary.slot_rooted = tower->root_slot;
+    fd_gui_printf_root_slot( gui );
+    fd_http_server_ws_broadcast( gui->http );
+
+    fd_gui_printf_slot( gui, gui->summary.slot_rooted );
+    fd_http_server_ws_broadcast( gui->http );
   }
 
-  gui->summary.slot_rooted = tower->root_slot;
-  fd_gui_printf_root_slot( gui );
-  fd_http_server_ws_broadcast( gui->http );
+  /* optimisitically confirmed slot */
+  if( FD_UNLIKELY( tower->opt_confirmed_slot!=ULONG_MAX ) ) {
+    /* Slot 0 is always rooted.  No need to iterate all the way back to
+      i==_slot */
+    for( ulong i=0UL; i<fd_ulong_min( tower->opt_confirmed_slot, FD_GUI_SLOTS_CNT ); i++ ) {
+      ulong parent_slot = tower->opt_confirmed_slot - i;
+      ulong parent_idx = parent_slot % FD_GUI_SLOTS_CNT;
+  
+      fd_gui_slot_t * slot = gui->slots[ parent_idx ];
+      if( FD_UNLIKELY( slot->slot==ULONG_MAX) ) break;
+  
+      if( FD_UNLIKELY( slot->slot>parent_slot ) ) {
+        FD_LOG_ERR(( "_slot %lu i %lu we expect parent_slot %lu got slot->slot %lu", tower->opt_confirmed_slot, i, parent_slot, slot->slot ));
+      } else if( FD_UNLIKELY( slot->slot<parent_slot ) ) {
+        /* Slot not even replayed yet ... will come out as optimistically confirmed */
+        continue;
+      }
+      if( FD_UNLIKELY( slot->level>=FD_GUI_SLOT_LEVEL_ROOTED ) ) break;
+  
+      if( FD_LIKELY( slot->level<FD_GUI_SLOT_LEVEL_OPTIMISTICALLY_CONFIRMED ) ) {
+        slot->level = FD_GUI_SLOT_LEVEL_OPTIMISTICALLY_CONFIRMED;
+        fd_gui_printf_slot( gui, parent_slot );
+        fd_http_server_ws_broadcast( gui->http );
+      }
+    }
+  
+    if( FD_UNLIKELY( gui->summary.slot_optimistically_confirmed!=ULONG_MAX && tower->opt_confirmed_slot<gui->summary.slot_optimistically_confirmed ) ) {
+      /* Optimistically confirmed slot went backwards ... mark some slots as no
+         longer optimistically confirmed. */
+      for( ulong i=gui->summary.slot_optimistically_confirmed; i>=tower->opt_confirmed_slot; i-- ) {
+        fd_gui_slot_t * slot = gui->slots[ i % FD_GUI_SLOTS_CNT ];
+        if( FD_UNLIKELY( slot->slot==ULONG_MAX ) ) break;
+        if( FD_LIKELY( slot->slot==i ) ) {
+          /* It's possible for the optimistically confirmed slot to skip
+             backwards between two slots that we haven't yet replayed.  In
+             that case we don't need to change anything, since they will
+             get marked properly when they get completed. */
+          slot->level = FD_GUI_SLOT_LEVEL_COMPLETED;
+          fd_gui_printf_slot( gui, i );
+          fd_http_server_ws_broadcast( gui->http );
+        }
+      }
+    }
+  
+    int slot_turbine_hist_full = gui->summary.slots_max_turbine[ FD_GUI_TURBINE_SLOT_HISTORY_SZ-1UL ].slot!=ULONG_MAX;
+    if( FD_UNLIKELY( gui->summary.slot_caught_up==ULONG_MAX && slot_turbine_hist_full && gui->summary.slots_max_turbine[ 0 ].slot < (tower->opt_confirmed_slot + 3UL) ) ) {
+      gui->summary.slot_caught_up = tower->opt_confirmed_slot + 4UL;
+  
+      fd_gui_printf_slot_caught_up( gui );
+      fd_http_server_ws_broadcast( gui->http );
+    }
+  
+    gui->summary.slot_optimistically_confirmed = tower->opt_confirmed_slot;
+    fd_gui_printf_optimistically_confirmed_slot( gui );
+    fd_http_server_ws_broadcast( gui->http );
+  }
 }
 
 void
@@ -2161,7 +2223,8 @@ fd_gui_plugin_message( fd_gui_t *    gui,
       break;
     }
     case FD_PLUGIN_MSG_LEADER_SCHEDULE: {
-      fd_gui_handle_leader_schedule( gui, (ulong const *)msg, now );
+      FD_STATIC_ASSERT( sizeof(fd_stake_weight_msg_t)==6*sizeof(ulong), "sanity check" );
+      fd_gui_handle_leader_schedule( gui, (fd_stake_weight_msg_t *)msg, now );
       break;
     }
     case FD_PLUGIN_MSG_SLOT_START: {

src/disco/gui/fd_gui.c

@@ -12,6 +12,7 @@
 #include "../../discof/restore/fd_snaprd_tile.h"
 #include "../../discof/replay/fd_replay_tile.h"
 #include "../../discof/tower/fd_tower_tile.h"
+#include "../../choreo/fd_choreo_base.h"
 
 #include "../topo/fd_topo.h"
 
@@ -739,9 +740,19 @@ void
 fd_gui_handle_repair_slot( fd_gui_t * gui, ulong slot, long now );
 
 void
-fd_gui_handle_snapshot_update( fd_gui_t *                 gui,
+fd_gui_handle_snapshot_update( fd_gui_t *           gui,
                                fd_snaprd_update_t * msg );
 
+void
+fd_gui_handle_leader_schedule( fd_gui_t *                    gui,
+                               fd_stake_weight_msg_t const * leader_schedule,
+                               long                          now );
+
+void
+fd_gui_handle_tower_update( fd_gui_t *                   gui,
+                            fd_tower_slot_done_t const * tower,
+                            long                         now );
+
 void
 fd_gui_handle_replay_update( fd_gui_t *                         gui,
                              fd_replay_slot_completed_t const * replay,