From f526e3bd68d26f1e18a412e6f657c9b44bdba69c Mon Sep 17 00:00:00 2001 From: Ishan Bhatt Date: Thu, 9 Oct 2025 13:26:37 +0000 Subject: [PATCH 1/3] a --- src/flamenco/stakes/fd_stakes.c | 88 ++++++++++++++------------------- 1 file changed, 36 insertions(+), 52 deletions(-) diff --git a/src/flamenco/stakes/fd_stakes.c b/src/flamenco/stakes/fd_stakes.c index d59bc31ed5c..1c27158bc1a 100644 --- a/src/flamenco/stakes/fd_stakes.c +++ b/src/flamenco/stakes/fd_stakes.c @@ -25,11 +25,21 @@ fd_stake_weights_by_node( fd_vote_states_t const * vote_states, return weights_cnt; } -static void -compute_stake_delegations( fd_bank_t * bank, - fd_stake_delegations_t const * stake_delegations, - fd_stake_history_t const * history, - ulong * new_rate_activation_epoch ) { +/* Refresh vote accounts. + + This updates the epoch bank stakes vote_accounts cache - that is, the + total amount of delegated stake each vote account has, using the + current delegation values from inside each stake account. Contrary + to the Agave equivalent, it also merges the stakes cache vote + accounts with the new vote account keys from this epoch. + + https://github.com/solana-labs/solana/blob/c091fd3da8014c0ef83b626318018f238f506435/runtime/src/stakes.rs#L562 */ +void +fd_refresh_vote_accounts( fd_bank_t * bank, + fd_stake_delegations_t const * stake_delegations, + fd_stake_history_t const * history, + ulong * new_rate_activation_epoch ) { + ulong epoch = fd_bank_epoch_get( bank ); ulong total_stake = 0UL; @@ -74,41 +84,31 @@ compute_stake_delegations( fd_bank_t * bank, fd_bank_total_epoch_stake_set( bank, total_stake ); + FD_LOG_CRIT(("TOTAL STAKE %lu %lu", fd_bank_epoch_get( bank ), total_stake)); + fd_bank_vote_states_end_locking_modify( bank ); } -/* Refresh vote accounts. - - This updates the epoch bank stakes vote_accounts cache - that is, the total amount - of delegated stake each vote account has, using the current delegation values from inside each - stake account. Contrary to the Agave equivalent, it also merges the stakes cache vote accounts with the - new vote account keys from this epoch. +/* Accumulate stake information for this epoch into a stake history + entry which gets inserted into the stake history sysvar. Unlike in + the Agave client, the total amount of stake on each vote account is + not updated as stake accounts are updated/inserted/removed. Because, + we are already iterating through all of the stake accounts, + update the stake information for each vote accoutn as well. */ - https://github.com/solana-labs/solana/blob/c091fd3da8014c0ef83b626318018f238f506435/runtime/src/stakes.rs#L562 */ void -fd_refresh_vote_accounts( fd_bank_t * bank, - fd_stake_delegations_t const * stake_delegations, - fd_stake_history_t const * history, - ulong * new_rate_activation_epoch ) { - - compute_stake_delegations( - bank, - stake_delegations, - history, - new_rate_activation_epoch ); -} - -static void -accumulate_stake_cache_delegations( fd_stake_delegations_t const * stake_delegations, - fd_stake_history_t const * history, - ulong * new_rate_activation_epoch, - fd_stake_history_entry_t * accumulator, - ulong epoch ) { +fd_accumulate_stake_infos( ulong epoch, + fd_stake_delegations_t const * stake_delegations, + fd_stake_history_t const * history, + ulong * new_rate_activation_epoch, + fd_stake_history_entry_t * accumulator ) { ulong effective = 0UL; ulong activating = 0UL; ulong deactivating = 0UL; + FD_LOG_WARNING(("TIMESTAMP %lu", fd_log_wallclock())); + fd_stake_delegations_iter_t iter_[1]; for( fd_stake_delegations_iter_t * iter = fd_stake_delegations_iter_init( iter_, stake_delegations ); !fd_stake_delegations_iter_done( iter ); @@ -133,29 +133,11 @@ accumulate_stake_cache_delegations( fd_stake_delegations_t const * stake_delegat deactivating += new_entry.deactivating; } + FD_LOG_WARNING(("TIMESTAMP2 %lu", fd_log_wallclock())); + accumulator->effective += effective; accumulator->activating += activating; accumulator->deactivating += deactivating; - -} - -/* Accumulates information about epoch stakes into `temp_info`, which is a temporary cache - used to save intermediate state about stake and vote accounts to avoid them from having to - be recomputed on every access, especially at the epoch boundary. Also collects stats in `accumulator` */ -void -fd_accumulate_stake_infos( ulong epoch, - fd_stake_delegations_t const * stake_delegations, - fd_stake_history_t const * history, - ulong * new_rate_activation_epoch, - fd_stake_history_entry_t * accumulator ) { - - accumulate_stake_cache_delegations( - stake_delegations, - history, - new_rate_activation_epoch, - accumulator, - epoch ); - } /* https://github.com/solana-labs/solana/blob/88aeaa82a856fc807234e7da0b31b89f2dc0e091/runtime/src/stakes.rs#L169 */ @@ -192,14 +174,16 @@ fd_stakes_activate_epoch( fd_bank_t * bank, /* https://github.com/anza-xyz/agave/blob/v2.1.6/runtime/src/stakes.rs#L359 */ fd_epoch_stake_history_entry_pair_t new_elem = { - .epoch = fd_bank_epoch_get( bank ), - .entry = { + .epoch = fd_bank_epoch_get( bank ), + .entry = { .effective = accumulator.effective, .activating = accumulator.activating, .deactivating = accumulator.deactivating } }; + FD_LOG_WARNING(("EFFECTIVE STAKE EFFECTIVE STAKE %lu %lu", fd_bank_epoch_get( bank ), new_elem.entry.effective)); + fd_sysvar_stake_history_update( bank, funk, xid, capture_ctx, &new_elem, runtime_spad ); } From 7c259fb4ed7e0430db2efb3143d1804918539d25 Mon Sep 17 00:00:00 2001 From: Ishan Bhatt Date: Thu, 9 Oct 2025 14:44:30 +0000 Subject: [PATCH 2/3] wip --- src/flamenco/rewards/fd_rewards.c | 5 +- src/flamenco/runtime/fd_runtime.c | 23 ++---- .../runtime/program/fd_stake_program.c | 42 ++++------ src/flamenco/stakes/fd_stakes.c | 81 ++++++++++++------- src/flamenco/stakes/fd_stakes.h | 5 +- 5 files changed, 80 insertions(+), 76 deletions(-) diff --git a/src/flamenco/rewards/fd_rewards.c b/src/flamenco/rewards/fd_rewards.c index 0cd00aac069..73652e5e14a 100644 --- a/src/flamenco/rewards/fd_rewards.c +++ b/src/flamenco/rewards/fd_rewards.c @@ -1241,11 +1241,12 @@ fd_rewards_recalculate_partitioned_rewards( fd_banks_t * banks, } fd_accumulate_stake_infos( - epoch, + bank, stake_delegations, stake_history, new_warmup_cooldown_rate_epoch, - &_accumulator ); + &_accumulator, + 1 /* is_recalculation */ ); /* Make sure is_recalculation is ==1 since we are booting up in the middle of rewards distribution (so we should use the epoch diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index a652160712f..4b9b3053f9e 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -1526,25 +1526,16 @@ fd_runtime_process_new_epoch( fd_banks_t * banks, new_rate_activation_epoch = NULL; } - /* Updates stake history sysvar accumulated values. */ + /* The first thing that needs to be done is to calculate the value of + the stake history entry that needs to be inserted into the stake + history sysvar. At this point, we need to calculate the stakes for + */ fd_stakes_activate_epoch( bank, funk, xid, capture_ctx, stake_delegations, new_rate_activation_epoch, runtime_spad ); - /* Refresh vote accounts in stakes cache using updated stake weights, and merges slot bank vote accounts with the epoch bank vote accounts. - https://github.com/anza-xyz/agave/blob/v2.1.6/runtime/src/stakes.rs#L363-L370 */ - fd_stake_history_t const * history = fd_sysvar_stake_history_read( funk, xid, runtime_spad ); - if( FD_UNLIKELY( !history ) ) { - FD_LOG_ERR(( "StakeHistory sysvar could not be read and decoded" )); - } - /* Now increment the epoch */ fd_bank_epoch_set( bank, fd_bank_epoch_get( bank ) + 1UL ); - fd_refresh_vote_accounts( bank, - stake_delegations, - history, - new_rate_activation_epoch ); - /* Distribute rewards */ fd_hash_t parent_blockhash = {0}; @@ -1555,6 +1546,7 @@ fd_runtime_process_new_epoch( fd_banks_t * banks, parent_blockhash = *bhq_last; } + long e = fd_log_wallclock(); fd_begin_partitioned_rewards( bank, funk, xid, @@ -1563,7 +1555,8 @@ fd_runtime_process_new_epoch( fd_banks_t * banks, &parent_blockhash, parent_epoch, runtime_spad ); - + long f = fd_log_wallclock(); + FD_LOG_NOTICE(("fd_begin_partitioned_rewards took %ld ns", f - e)); /* Update vote_states_prev_prev with vote_states_prev */ @@ -1580,7 +1573,7 @@ fd_runtime_process_new_epoch( fd_banks_t * banks, FD_LOG_NOTICE(( "fd_process_new_epoch end" )); long end = fd_log_wallclock(); - FD_LOG_NOTICE(("fd_process_new_epoch took %ld ns", end - start)); + FD_LOG_CRIT(("fd_process_new_epoch took %ld ns", end - start)); } FD_SPAD_FRAME_END; } diff --git a/src/flamenco/runtime/program/fd_stake_program.c b/src/flamenco/runtime/program/fd_stake_program.c index 1c1c4ac0121..05a879cf19d 100644 --- a/src/flamenco/runtime/program/fd_stake_program.c +++ b/src/flamenco/runtime/program/fd_stake_program.c @@ -564,15 +564,15 @@ stake_and_activating( fd_delegation_t const * self, } // https://github.com/anza-xyz/agave/blob/c8685ce0e1bb9b26014f1024de2cd2b8c308cbde/sdk/program/src/stake/state.rs#L641 -static fd_stake_activation_status_t -stake_activating_and_deactivating( fd_delegation_t const * self, - ulong target_epoch, - fd_stake_history_t const * stake_history, - ulong * new_rate_activation_epoch ) { +fd_stake_activation_status_t +fd_stake_activating_and_deactivating( fd_delegation_t const * delegation, + ulong target_epoch, + fd_stake_history_t const * stake_history, + ulong * new_rate_activation_epoch ) { // https://github.com/anza-xyz/agave/blob/c8685ce0e1bb9b26014f1024de2cd2b8c308cbde/sdk/program/src/stake/state.rs#L648 effective_activating_t effective_activating = - stake_and_activating( self, target_epoch, stake_history, new_rate_activation_epoch ); + stake_and_activating( delegation, target_epoch, stake_history, new_rate_activation_epoch ); ulong effective_stake = effective_activating.effective; ulong activating_stake = effective_activating.activating; @@ -580,7 +580,7 @@ stake_activating_and_deactivating( fd_delegation_t const * self, fd_stake_history_entry_t const * cluster_stake_at_deactivation_epoch = NULL; // https://github.com/anza-xyz/agave/blob/v2.0.1/sdk/program/src/stake/state.rs#L652 - if( target_epochdeactivation_epoch ) { + if( target_epochdeactivation_epoch ) { // if is bootstrap if( activating_stake==0 ) { return ( fd_stake_history_entry_t ){ @@ -589,14 +589,14 @@ stake_activating_and_deactivating( fd_delegation_t const * self, return ( fd_stake_history_entry_t ){ .effective = effective_stake, .deactivating = 0, .activating = activating_stake }; } - } else if( target_epoch==self->deactivation_epoch ) { + } else if( target_epoch==delegation->deactivation_epoch ) { // https://github.com/anza-xyz/agave/blob/be16321eb0db3e12a57a32f59febbf54b92ebb7c/sdk/program/src/stake/state.rs#L662 return ( fd_stake_history_entry_t ){ .effective = effective_stake, .deactivating = effective_stake, .activating = 0 }; } else if( stake_history && - ( cluster_stake_at_deactivation_epoch = fd_stake_history_ele_query_const( stake_history, self->deactivation_epoch ) ) ) { + ( cluster_stake_at_deactivation_epoch = fd_stake_history_ele_query_const( stake_history, delegation->deactivation_epoch ) ) ) { // https://github.com/anza-xyz/agave/blob/be16321eb0db3e12a57a32f59febbf54b92ebb7c/sdk/program/src/stake/state.rs#L665 - ulong prev_epoch = self->deactivation_epoch; + ulong prev_epoch = delegation->deactivation_epoch; fd_stake_history_entry_t const * prev_cluster_stake = cluster_stake_at_deactivation_epoch; ulong current_epoch; @@ -646,8 +646,7 @@ delegation_stake( fd_delegation_t const * self, ulong epoch, fd_stake_history_t const * history, ulong * new_rate_activation_epoch ) { - return stake_activating_and_deactivating( self, epoch, history, new_rate_activation_epoch ) - .effective; + return fd_stake_activating_and_deactivating( self, epoch, history, new_rate_activation_epoch ).effective; } /**********************************************************************/ @@ -842,10 +841,10 @@ get_if_mergeable( fd_exec_instr_ctx_t * invoke_context, // not const to // https://github.com/anza-xyz/agave/blob/c8685ce0e1bb9b26014f1024de2cd2b8c308cbde/programs/stake/src/stake_state.rs#L1108 fd_stake_history_entry_t status = - stake_activating_and_deactivating( &stake->delegation, - clock->epoch, - stake_history, - fd_ptr_if( is_some, &new_rate_activation_epoch, NULL ) ); + fd_stake_activating_and_deactivating( &stake->delegation, + clock->epoch, + stake_history, + fd_ptr_if( is_some, &new_rate_activation_epoch, NULL ) ); // https://github.com/anza-xyz/agave/blob/c8685ce0e1bb9b26014f1024de2cd2b8c308cbde/programs/stake/src/stake_state.rs#L1115 if( status.effective==0 && status.activating==0 && status.deactivating==0 ) { @@ -1160,7 +1159,7 @@ get_stake_status( fd_exec_instr_ctx_t const * invoke_context, if( FD_UNLIKELY( err ) ) return err; fd_stake_history_t const * stake_history = fd_sysvar_cache_stake_history_join_const( sysvar_cache ); - *out = stake_activating_and_deactivating( + *out = fd_stake_activating_and_deactivating( &stake->delegation, clock->epoch, stake_history, @@ -3278,12 +3277,3 @@ fd_stake_get_state( fd_txn_account_t const * self, fd_stake_state_v2_t * out ) { return get_state( self, out ); } - -fd_stake_history_entry_t -fd_stake_activating_and_deactivating( fd_delegation_t const * self, - ulong target_epoch, - fd_stake_history_t const * stake_history, - ulong * new_rate_activation_epoch ) { - return stake_activating_and_deactivating( - self, target_epoch, stake_history, new_rate_activation_epoch ); -} diff --git a/src/flamenco/stakes/fd_stakes.c b/src/flamenco/stakes/fd_stakes.c index 1c27158bc1a..3ac7ebcef80 100644 --- a/src/flamenco/stakes/fd_stakes.c +++ b/src/flamenco/stakes/fd_stakes.c @@ -84,30 +84,36 @@ fd_refresh_vote_accounts( fd_bank_t * bank, fd_bank_total_epoch_stake_set( bank, total_stake ); - FD_LOG_CRIT(("TOTAL STAKE %lu %lu", fd_bank_epoch_get( bank ), total_stake)); - fd_bank_vote_states_end_locking_modify( bank ); } /* Accumulate stake information for this epoch into a stake history entry which gets inserted into the stake history sysvar. Unlike in the Agave client, the total amount of stake on each vote account is - not updated as stake accounts are updated/inserted/removed. Because, - we are already iterating through all of the stake accounts, - update the stake information for each vote accoutn as well. */ + not updated as stake accounts are updated/inserted/removed and is + instead done at the end of an epoch. */ void -fd_accumulate_stake_infos( ulong epoch, +fd_accumulate_stake_infos( fd_bank_t * bank, fd_stake_delegations_t const * stake_delegations, fd_stake_history_t const * history, ulong * new_rate_activation_epoch, - fd_stake_history_entry_t * accumulator ) { + fd_stake_history_entry_t * accumulator, + int is_recalculation ) { - ulong effective = 0UL; - ulong activating = 0UL; - ulong deactivating = 0UL; + /* This represents the total amount of activated stake for the epoch + after the epoch boundary. */ + ulong total_stake = 0UL; - FD_LOG_WARNING(("TIMESTAMP %lu", fd_log_wallclock())); + ulong epoch = fd_bank_epoch_get( bank ); + + fd_vote_states_t * vote_states = NULL; + if( !is_recalculation ) { + vote_states = fd_bank_vote_states_locking_modify( bank ); + if( FD_UNLIKELY( !vote_states ) ) { + FD_LOG_CRIT(( "vote_states is NULL" )); + } + } fd_stake_delegations_iter_t iter_[1]; for( fd_stake_delegations_iter_t * iter = fd_stake_delegations_iter_init( iter_, stake_delegations ); @@ -128,19 +134,34 @@ fd_accumulate_stake_infos( ulong epoch, epoch, history, new_rate_activation_epoch ); - effective += new_entry.effective; - activating += new_entry.activating; - deactivating += new_entry.deactivating; - } - FD_LOG_WARNING(("TIMESTAMP2 %lu", fd_log_wallclock())); + if( !is_recalculation ) { + fd_stake_history_entry_t next_epoch_entry = fd_stake_activating_and_deactivating( + &delegation, + epoch+1UL, + history, + new_rate_activation_epoch ); + + fd_vote_state_ele_t * vote_state = fd_vote_states_query( vote_states, &stake_delegation->vote_account ); + if( FD_LIKELY( vote_state ) ) { + total_stake += next_epoch_entry.effective; + vote_state->stake += next_epoch_entry.effective; + } + } + + accumulator->effective += new_entry.effective; + accumulator->activating += new_entry.activating; + accumulator->deactivating += new_entry.deactivating; + } - accumulator->effective += effective; - accumulator->activating += activating; - accumulator->deactivating += deactivating; + if( !is_recalculation ) { + /* Store the total amount of activated stake for the new epoch.*/ + fd_bank_total_epoch_stake_set( bank, total_stake ); + fd_bank_vote_states_end_locking_modify( bank ); + } } -/* https://github.com/solana-labs/solana/blob/88aeaa82a856fc807234e7da0b31b89f2dc0e091/runtime/src/stakes.rs#L169 */ +/* https://github.com/anza-xyz/agave/blob/v3.0.0/runtime/src/stakes.rs#L280 */ void fd_stakes_activate_epoch( fd_bank_t * bank, fd_funk_t * funk, @@ -150,13 +171,14 @@ fd_stakes_activate_epoch( fd_bank_t * bank, ulong * new_rate_activation_epoch, fd_spad_t * runtime_spad ) { - /* Current stake delegations: list of all current delegations in stake_delegations - https://github.com/solana-labs/solana/blob/88aeaa82a856fc807234e7da0b31b89f2dc0e091/runtime/src/stakes.rs#L180 */ - /* Add a new entry to the Stake History sysvar for the previous epoch - https://github.com/solana-labs/solana/blob/88aeaa82a856fc807234e7da0b31b89f2dc0e091/runtime/src/stakes.rs#L181-L192 */ + /* This function is responsible for updating the stake history sysvar + with the stake information for the current epoch and computing + stake values for the next epoch. */ fd_stake_history_t const * history = fd_sysvar_stake_history_read( funk, xid, runtime_spad ); - if( FD_UNLIKELY( !history ) ) FD_LOG_ERR(( "StakeHistory sysvar is missing from sysvar cache" )); + if( FD_UNLIKELY( !history ) ) { + FD_LOG_ERR(( "StakeHistory sysvar is missing from sysvar cache" )); + } fd_stake_history_entry_t accumulator = { .effective = 0UL, @@ -164,15 +186,14 @@ fd_stakes_activate_epoch( fd_bank_t * bank, .deactivating = 0UL }; - /* Accumulate stats for stake accounts */ fd_accumulate_stake_infos( - fd_bank_epoch_get( bank ), + bank, stake_delegations, history, new_rate_activation_epoch, - &accumulator ); + &accumulator, + 0 /* is_recalculation */ ); - /* https://github.com/anza-xyz/agave/blob/v2.1.6/runtime/src/stakes.rs#L359 */ fd_epoch_stake_history_entry_pair_t new_elem = { .epoch = fd_bank_epoch_get( bank ), .entry = { @@ -182,8 +203,6 @@ fd_stakes_activate_epoch( fd_bank_t * bank, } }; - FD_LOG_WARNING(("EFFECTIVE STAKE EFFECTIVE STAKE %lu %lu", fd_bank_epoch_get( bank ), new_elem.entry.effective)); - fd_sysvar_stake_history_update( bank, funk, xid, capture_ctx, &new_elem, runtime_spad ); } diff --git a/src/flamenco/stakes/fd_stakes.h b/src/flamenco/stakes/fd_stakes.h index fba1bc5be8e..2425b01c1f1 100644 --- a/src/flamenco/stakes/fd_stakes.h +++ b/src/flamenco/stakes/fd_stakes.h @@ -58,11 +58,12 @@ fd_refresh_vote_accounts( fd_bank_t * bank, ulong * new_rate_activation_epoch ); void -fd_accumulate_stake_infos( ulong epoch, +fd_accumulate_stake_infos( fd_bank_t * bank, fd_stake_delegations_t const * stake_delegations, fd_stake_history_t const * history, ulong * new_rate_activation_epoch, - fd_stake_history_entry_t * accumulator ); + fd_stake_history_entry_t * accumulator, + int is_recalculation ); /* fd_store_stake_delegation is used to update fd_stake_delegations_t based on a specific transaction account. If the account is empty or From 1c48d98a787aebb99a33639df098cfe1be12c590 Mon Sep 17 00:00:00 2001 From: Ishan Bhatt Date: Thu, 9 Oct 2025 15:50:13 +0000 Subject: [PATCH 3/3] cleanup --- src/flamenco/rewards/fd_rewards.c | 206 ++++++++++++------ src/flamenco/runtime/fd_runtime.c | 8 +- .../runtime/tests/run_ledger_backtest.sh | 2 +- src/flamenco/stakes/fd_stakes.c | 110 ++++------ src/flamenco/stakes/fd_stakes.h | 8 - 5 files changed, 186 insertions(+), 148 deletions(-) diff --git a/src/flamenco/rewards/fd_rewards.c b/src/flamenco/rewards/fd_rewards.c index 73652e5e14a..32e72798ed1 100644 --- a/src/flamenco/rewards/fd_rewards.c +++ b/src/flamenco/rewards/fd_rewards.c @@ -247,21 +247,71 @@ redeem_rewards( fd_stake_history_t const * stake_history, } /* https://github.com/anza-xyz/agave/blob/cbc8320d35358da14d79ebcada4dfb6756ffac79/programs/stake/src/points.rs#L70 */ -static int +static void calculate_points( fd_stake_delegation_t const * stake, fd_vote_state_ele_t const * vote_state, fd_stake_history_t const * stake_history, ulong * new_rate_activation_epoch, uint128 * result ) { - fd_calculated_stake_points_t stake_point_result; - calculate_stake_points_and_credits( stake_history, - stake, - vote_state, - new_rate_activation_epoch, - &stake_point_result ); - *result = stake_point_result.points; + ulong credits_in_stake = stake->credits_observed; + ulong credits_in_vote = 0UL; + if( FD_LIKELY( vote_state->credits_cnt>0UL ) ) { + credits_in_vote = vote_state->credits[vote_state->credits_cnt-1UL]; + } - return FD_EXECUTOR_INSTR_SUCCESS; + /* If the Vote account has less credits observed than the Stake + account, something is wrong and we need to force an update. + + https://github.com/anza-xyz/agave/blob/cbc8320d35358da14d79ebcada4dfb6756ffac79/programs/stake/src/points.rs#L142 */ + if( FD_UNLIKELY( credits_in_votecredits_cnt; i++ ) { + + ulong final_epoch_credits = vote_state->credits[i]; + ulong initial_epoch_credits = vote_state->prev_credits[i]; + uint128 earned_credits = 0UL; + if( FD_LIKELY( credits_in_stake < initial_epoch_credits ) ) { + earned_credits = (uint128)(final_epoch_credits - initial_epoch_credits); + } else if( FD_UNLIKELY( credits_in_stake < final_epoch_credits ) ) { + earned_credits = (uint128)(final_epoch_credits - new_credits_observed); + } + + new_credits_observed = fd_ulong_max( new_credits_observed, final_epoch_credits ); + + fd_delegation_t delegation = { + .voter_pubkey = stake->vote_account, + .stake = stake->stake, + .activation_epoch = stake->activation_epoch, + .deactivation_epoch = stake->deactivation_epoch, + .warmup_cooldown_rate = stake->warmup_cooldown_rate, + }; + + ulong stake_amount = fd_stake_activating_and_deactivating( + &delegation, + vote_state->epoch[i], + stake_history, + new_rate_activation_epoch ).effective; + + points += (uint128)stake_amount * earned_credits; + + } + *result = points; } /* Returns the length of the given epoch in slots @@ -312,50 +362,47 @@ get_minimum_stake_delegation( fd_bank_t * bank ) { return 1; } -static uint128 -calculate_points_all( fd_bank_t * bank, - fd_stake_delegations_t const * stake_delegations, - fd_stake_history_t const * stake_history, - ulong * new_warmup_cooldown_rate_epoch, - ulong minimum_stake_delegation ) { +// static uint128 +// calculate_points_all( fd_bank_t * bank, +// fd_stake_delegations_t const * stake_delegations, +// fd_stake_history_t const * stake_history, +// ulong * new_warmup_cooldown_rate_epoch, +// ulong minimum_stake_delegation ) { - uint128 total_points = 0; +// uint128 total_points = 0; - fd_vote_states_t const * vote_states = fd_bank_vote_states_locking_query( bank ); +// fd_vote_states_t const * vote_states = fd_bank_vote_states_locking_query( bank ); - fd_stake_delegations_iter_t iter_[1]; - for( fd_stake_delegations_iter_t * iter = fd_stake_delegations_iter_init( iter_, stake_delegations ); - !fd_stake_delegations_iter_done( iter ); - fd_stake_delegations_iter_next( iter ) ) { - fd_stake_delegation_t const * stake_delegation = fd_stake_delegations_iter_ele( iter ); +// fd_stake_delegations_iter_t iter_[1]; +// for( fd_stake_delegations_iter_t * iter = fd_stake_delegations_iter_init( iter_, stake_delegations ); +// !fd_stake_delegations_iter_done( iter ); +// fd_stake_delegations_iter_next( iter ) ) { +// fd_stake_delegation_t const * stake_delegation = fd_stake_delegations_iter_ele( iter ); - if( FD_UNLIKELY( stake_delegation->stakestakevote_account ); - if( FD_UNLIKELY( !vote_state_ele ) ) { - continue; - } +// fd_vote_state_ele_t * vote_state_ele = fd_vote_states_query( vote_states, &stake_delegation->vote_account ); +// if( FD_UNLIKELY( !vote_state_ele ) ) { +// continue; +// } - uint128 account_points; - int err = calculate_points( - stake_delegation, - vote_state_ele, - stake_history, - new_warmup_cooldown_rate_epoch, &account_points ); - if( FD_UNLIKELY( err ) ) { - FD_LOG_DEBUG(( "failed to calculate points" )); - continue; - } +// uint128 account_points; +// calculate_points( +// stake_delegation, +// vote_state_ele, +// stake_history, +// new_warmup_cooldown_rate_epoch, +// &account_points ); - total_points += account_points; - } +// total_points += account_points; +// } - fd_bank_vote_states_end_locking_query( bank ); +// fd_bank_vote_states_end_locking_query( bank ); - return total_points; -} +// return total_points; +// } /* Calculates epoch reward points from stake/vote accounts. https://github.com/anza-xyz/agave/blob/cbc8320d35358da14d79ebcada4dfb6756ffac79/runtime/src/bank/partitioned_epoch_rewards/calculation.rs#L472 */ @@ -381,15 +428,40 @@ calculate_reward_points_partitioned( fd_bank_t * bank, new_warmup_cooldown_rate_epoch = NULL; } - uint128 points = calculate_points_all( - bank, - stake_delegations, - stake_history, - new_warmup_cooldown_rate_epoch, - minimum_stake_delegation ); + uint128 total_points = 0; + + fd_vote_states_t const * vote_states = fd_bank_vote_states_locking_query( bank ); + + fd_stake_delegations_iter_t iter_[1]; + for( fd_stake_delegations_iter_t * iter = fd_stake_delegations_iter_init( iter_, stake_delegations ); + !fd_stake_delegations_iter_done( iter ); + fd_stake_delegations_iter_next( iter ) ) { + fd_stake_delegation_t const * stake_delegation = fd_stake_delegations_iter_ele( iter ); + + if( FD_UNLIKELY( stake_delegation->stake 0 ) { - result->points = points; + fd_vote_state_ele_t * vote_state_ele = fd_vote_states_query( vote_states, &stake_delegation->vote_account ); + if( FD_UNLIKELY( !vote_state_ele ) ) { + continue; + } + + uint128 account_points; + calculate_points( + stake_delegation, + vote_state_ele, + stake_history, + new_warmup_cooldown_rate_epoch, + &account_points ); + + total_points += account_points; + } + + fd_bank_vote_states_end_locking_query( bank ); + + if( FD_LIKELY( total_points>0 ) ) { + result->points = total_points; result->rewards = rewards; } } @@ -669,6 +741,8 @@ calculate_validator_rewards( fd_bank_t * bank, FD_LOG_ERR(( "Unable to read and decode stake history sysvar" )); } + long b = fd_log_wallclock(); + /* Calculate the epoch reward points from stake/vote accounts */ calculate_reward_points_partitioned( bank, @@ -686,7 +760,10 @@ calculate_validator_rewards( fd_bank_t * bank, result->point_value.points ); } - /* Calculate the stake and vote rewards for each account. We want to + long c = fd_log_wallclock(); + FD_LOG_NOTICE(("calculate_reward_points_partitioned took %ld ns", c - b)); + + /* Calculate the stake and vote rewards for each account. We want to use the vote states from the end of the current_epoch. */ calculate_stake_vote_rewards( bank, @@ -698,6 +775,9 @@ calculate_validator_rewards( fd_bank_t * bank, &result->calculate_stake_vote_rewards_result, runtime_spad, 0 ); + + long d = fd_log_wallclock(); + FD_LOG_NOTICE(("calculate_stake_vote_rewards took %ld ns", d - c)); } /* Calculate the number of blocks required to distribute rewards to all stake accounts. @@ -1084,6 +1164,8 @@ fd_distribute_partitioned_epoch_rewards( fd_bank_t * bank, return; } + FD_LOG_WARNING(( "distributing rewards" )); + ulong height = fd_bank_block_height_get( bank ); ulong distribution_starting_block_height = fd_epoch_rewards_get_starting_block_height( epoch_rewards ); ulong distribution_end_exclusive = fd_epoch_rewards_get_exclusive_ending_block_height( epoch_rewards ); @@ -1229,25 +1311,11 @@ fd_rewards_recalculate_partitioned_rewards( fd_banks_t * banks, fd_point_value_t point_value = { .points = epoch_rewards->total_points, .rewards = epoch_rewards->total_rewards }; - fd_stake_history_entry_t _accumulator = { - .effective = 0UL, - .activating = 0UL, - .deactivating = 0UL - }; - fd_stake_delegations_t const * stake_delegations = fd_bank_stake_delegations_frontier_query( banks, bank ); if( FD_UNLIKELY( !stake_delegations ) ) { FD_LOG_CRIT(( "stake_delegations is NULL" )); } - fd_accumulate_stake_infos( - bank, - stake_delegations, - stake_history, - new_warmup_cooldown_rate_epoch, - &_accumulator, - 1 /* is_recalculation */ ); - /* Make sure is_recalculation is ==1 since we are booting up in the middle of rewards distribution (so we should use the epoch stakes for the end of epoch E-1 since we are still distributing @@ -1264,10 +1332,6 @@ fd_rewards_recalculate_partitioned_rewards( fd_banks_t * banks, runtime_spad, 1 /* is_recalculation */ ); - /* The vote reward map isn't actually used in this code path and - will only be freed after rewards have been distributed. */ - - /* Use the epoch rewards sysvar parent_blockhash and num_partitions. https://github.com/anza-xyz/agave/blob/v2.2.14/runtime/src/bank/partitioned_epoch_rewards/calculation.rs#L579 */ hash_rewards_into_partitions( diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index 4b9b3053f9e..eef38a79054 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -1558,6 +1558,8 @@ fd_runtime_process_new_epoch( fd_banks_t * banks, long f = fd_log_wallclock(); FD_LOG_NOTICE(("fd_begin_partitioned_rewards took %ld ns", f - e)); + long g = fd_log_wallclock(); + /* Update vote_states_prev_prev with vote_states_prev */ fd_update_vote_states_prev_prev( bank ); @@ -1573,7 +1575,11 @@ fd_runtime_process_new_epoch( fd_banks_t * banks, FD_LOG_NOTICE(( "fd_process_new_epoch end" )); long end = fd_log_wallclock(); - FD_LOG_CRIT(("fd_process_new_epoch took %ld ns", end - start)); + + long h = fd_log_wallclock(); + FD_LOG_NOTICE(("fd_update_vote_states_prev took %ld ns", h - g)); + + FD_LOG_NOTICE(("fd_process_new_epoch took %ld ns", end - start)); } FD_SPAD_FRAME_END; } diff --git a/src/flamenco/runtime/tests/run_ledger_backtest.sh b/src/flamenco/runtime/tests/run_ledger_backtest.sh index b1e94f1f122..c85b5b2207d 100755 --- a/src/flamenco/runtime/tests/run_ledger_backtest.sh +++ b/src/flamenco/runtime/tests/run_ledger_backtest.sh @@ -214,7 +214,7 @@ echo " max_live_slots = 32 max_fork_width = 4 [log] - level_stderr = \"INFO\" + level_stderr = \"NOTICE\" path = \"$LOG\" [paths] snapshots = \"$DUMP/$LEDGER\" diff --git a/src/flamenco/stakes/fd_stakes.c b/src/flamenco/stakes/fd_stakes.c index 3ac7ebcef80..1cb8c91e19f 100644 --- a/src/flamenco/stakes/fd_stakes.c +++ b/src/flamenco/stakes/fd_stakes.c @@ -91,15 +91,32 @@ fd_refresh_vote_accounts( fd_bank_t * bank, entry which gets inserted into the stake history sysvar. Unlike in the Agave client, the total amount of stake on each vote account is not updated as stake accounts are updated/inserted/removed and is - instead done at the end of an epoch. */ + instead done at the end of an epoch. */ +/* https://github.com/anza-xyz/agave/blob/v3.0.0/runtime/src/stakes.rs#L280 */ void -fd_accumulate_stake_infos( fd_bank_t * bank, - fd_stake_delegations_t const * stake_delegations, - fd_stake_history_t const * history, - ulong * new_rate_activation_epoch, - fd_stake_history_entry_t * accumulator, - int is_recalculation ) { +fd_stakes_activate_epoch( fd_bank_t * bank, + fd_funk_t * funk, + fd_funk_txn_xid_t const * xid, + fd_capture_ctx_t * capture_ctx, + fd_stake_delegations_t const * stake_delegations, + ulong * new_rate_activation_epoch, + fd_spad_t * runtime_spad ) { + + /* This function is responsible for updating the stake history sysvar + with the stake information for the current epoch and computing + stake values for the next epoch. */ + + fd_stake_history_t const * history = fd_sysvar_stake_history_read( funk, xid, runtime_spad ); + if( FD_UNLIKELY( !history ) ) { + FD_LOG_ERR(( "StakeHistory sysvar is missing from sysvar cache" )); + } + + fd_stake_history_entry_t accumulator = { + .effective = 0UL, + .activating = 0UL, + .deactivating = 0UL + }; /* This represents the total amount of activated stake for the epoch after the epoch boundary. */ @@ -107,12 +124,9 @@ fd_accumulate_stake_infos( fd_bank_t * bank, ulong epoch = fd_bank_epoch_get( bank ); - fd_vote_states_t * vote_states = NULL; - if( !is_recalculation ) { - vote_states = fd_bank_vote_states_locking_modify( bank ); - if( FD_UNLIKELY( !vote_states ) ) { - FD_LOG_CRIT(( "vote_states is NULL" )); - } + fd_vote_states_t * vote_states = fd_bank_vote_states_locking_modify( bank ); + if( FD_UNLIKELY( !vote_states ) ) { + FD_LOG_CRIT(( "vote_states is NULL" )); } fd_stake_delegations_iter_t iter_[1]; @@ -135,64 +149,26 @@ fd_accumulate_stake_infos( fd_bank_t * bank, history, new_rate_activation_epoch ); - if( !is_recalculation ) { - fd_stake_history_entry_t next_epoch_entry = fd_stake_activating_and_deactivating( - &delegation, - epoch+1UL, - history, - new_rate_activation_epoch ); - - fd_vote_state_ele_t * vote_state = fd_vote_states_query( vote_states, &stake_delegation->vote_account ); - if( FD_LIKELY( vote_state ) ) { - total_stake += next_epoch_entry.effective; - vote_state->stake += next_epoch_entry.effective; - } - } - - accumulator->effective += new_entry.effective; - accumulator->activating += new_entry.activating; - accumulator->deactivating += new_entry.deactivating; - } - - if( !is_recalculation ) { - /* Store the total amount of activated stake for the new epoch.*/ - fd_bank_total_epoch_stake_set( bank, total_stake ); - fd_bank_vote_states_end_locking_modify( bank ); - } -} - -/* https://github.com/anza-xyz/agave/blob/v3.0.0/runtime/src/stakes.rs#L280 */ -void -fd_stakes_activate_epoch( fd_bank_t * bank, - fd_funk_t * funk, - fd_funk_txn_xid_t const * xid, - fd_capture_ctx_t * capture_ctx, - fd_stake_delegations_t const * stake_delegations, - ulong * new_rate_activation_epoch, - fd_spad_t * runtime_spad ) { + fd_stake_history_entry_t next_epoch_entry = fd_stake_activating_and_deactivating( + &delegation, + epoch+1UL, + history, + new_rate_activation_epoch ); - /* This function is responsible for updating the stake history sysvar - with the stake information for the current epoch and computing - stake values for the next epoch. */ + fd_vote_state_ele_t * vote_state = fd_vote_states_query( vote_states, &stake_delegation->vote_account ); + if( FD_LIKELY( vote_state ) ) { + total_stake += next_epoch_entry.effective; + vote_state->stake += next_epoch_entry.effective; + } - fd_stake_history_t const * history = fd_sysvar_stake_history_read( funk, xid, runtime_spad ); - if( FD_UNLIKELY( !history ) ) { - FD_LOG_ERR(( "StakeHistory sysvar is missing from sysvar cache" )); + accumulator.effective += new_entry.effective; + accumulator.activating += new_entry.activating; + accumulator.deactivating += new_entry.deactivating; } - fd_stake_history_entry_t accumulator = { - .effective = 0UL, - .activating = 0UL, - .deactivating = 0UL - }; - - fd_accumulate_stake_infos( - bank, - stake_delegations, - history, - new_rate_activation_epoch, - &accumulator, - 0 /* is_recalculation */ ); + /* Store the total amount of activated stake for the new epoch.*/ + fd_bank_total_epoch_stake_set( bank, total_stake ); + fd_bank_vote_states_end_locking_modify( bank ); fd_epoch_stake_history_entry_pair_t new_elem = { .epoch = fd_bank_epoch_get( bank ), diff --git a/src/flamenco/stakes/fd_stakes.h b/src/flamenco/stakes/fd_stakes.h index 2425b01c1f1..c3a565e96ab 100644 --- a/src/flamenco/stakes/fd_stakes.h +++ b/src/flamenco/stakes/fd_stakes.h @@ -57,14 +57,6 @@ fd_refresh_vote_accounts( fd_bank_t * bank, fd_stake_history_t const * history, ulong * new_rate_activation_epoch ); -void -fd_accumulate_stake_infos( fd_bank_t * bank, - fd_stake_delegations_t const * stake_delegations, - fd_stake_history_t const * history, - ulong * new_rate_activation_epoch, - fd_stake_history_entry_t * accumulator, - int is_recalculation ); - /* fd_store_stake_delegation is used to update fd_stake_delegations_t based on a specific transaction account. If the account is empty or uninitialized, it is removed from the stake delegation map. */