Proposal for New Positional Consistency and Softmax for distribution

runnable/generate_request_minerstatistics.py

@@ -114,9 +114,14 @@ def generate_request_minerstatistics(time_now:int):
     all_miner_hotkeys = challengeperiod_success_hotkeys + challengeperiod_testing_hotkeys
 
     filtered_ledger = subtensor_weight_setter.filtered_ledger(hotkeys=all_miner_hotkeys)
+    filtered_positions = subtensor_weight_setter.get_all_miner_positions_by_hotkey(
+        challengeperiod_success_hotkeys,
+        sort_positions=True,
+        acceptable_position_end_ms=time_now - ValiConfig.SET_WEIGHT_LOOKBACK_RANGE_MS
+    )
 
     ## Penalties
-    miner_penalties = Scoring.miner_penalties(filtered_ledger)
+    miner_penalties = Scoring.miner_penalties(filtered_ledger, filtered_positions)
     fullpenalty_miners: list[tuple[str, float]] = [ ( miner, 0 ) for miner, penalty in miner_penalties.items() if penalty == 0 ]
 
     consistency_penalties = {}
@@ -172,7 +177,12 @@ def generate_request_minerstatistics(time_now:int):
 
     ## This is when we only want to look at the successful miners
     successful_ledger = subtensor_weight_setter.filtered_ledger(hotkeys=challengeperiod_success_hotkeys)
-    checkpoint_results = Scoring.compute_results_checkpoint(successful_ledger, evaluation_time_ms=time_now, verbose=False)
+    checkpoint_results = Scoring.compute_results_checkpoint(
+        ledger_dict=successful_ledger, 
+        filtered_positions=filtered_positions,
+        evaluation_time_ms=time_now, 
+        verbose=False
+    )
 
     challengeperiod_scores = [ (x, ValiConfig.SET_WEIGHT_MINER_CHALLENGE_PERIOD_WEIGHT) for x in challengeperiod_testing_hotkeys ]
     scoring_results = checkpoint_results + challengeperiod_scores

runnable/run_incentive_review.py

@@ -48,13 +48,19 @@
     )
 
     filtered_ledger = subtensor_weight_setter.filtered_ledger(hotkeys=passing_hotkeys + challengeperiod_success)
+    filtered_positions = subtensor_weight_setter.get_all_miner_positions_by_hotkey(
+        passing_hotkeys + challengeperiod_success,
+        sort_positions=True,
+        acceptable_position_end_ms=current_time - ValiConfig.SET_WEIGHT_LOOKBACK_RANGE_MS
+    )
+
     return_decay_coefficient_short = ValiConfig.HISTORICAL_DECAY_COEFFICIENT_RETURNS_SHORT
     return_decay_coefficient_long = ValiConfig.HISTORICAL_DECAY_COEFFICIENT_RETURNS_LONG
     risk_adjusted_decay_coefficient = ValiConfig.HISTORICAL_DECAY_COEFFICIENT_RISKMETRIC
     return_decay_short_lookback_time_ms = ValiConfig.RETURN_DECAY_SHORT_LOOKBACK_TIME_MS
 
     # Compute miner penalties
-    miner_penalties = Scoring.miner_penalties(filtered_ledger)
+    miner_penalties = Scoring.miner_penalties(filtered_ledger, filtered_positions)
 
     ## Miners with full penalty
     fullpenalty_miner_scores: list[tuple[str, float]] = [ ( miner, 0 ) for miner, penalty in miner_penalties.items() if penalty == 0 ]
@@ -63,11 +69,16 @@
     ## Individual miner penalties
     consistency_penalties = {}
     drawdown_penalties = {}
+    positional_penalties = {}
 
     for miner, ledger in filtered_ledger.items():
         minercps = ledger.cps
         consistency_penalties[miner] = PositionUtils.compute_consistency_penalty_cps(minercps)
         drawdown_penalties[miner] = PositionUtils.compute_drawdown_penalty_cps(minercps)
+        positional_penalties[miner] = PositionUtils.compute_positional_penalty_cps(
+            minercps,
+            filtered_positions[miner]
+        )
 
     # Augmented returns ledgers
     returns_ledger_short = PositionManager.limit_perf_ledger(
@@ -142,7 +153,7 @@
                 combined_scores[miner] = 1
             combined_scores[miner] *= config['weight'] * score + (1 - config['weight'])
 
-    combined_weighed = Scoring.weigh_miner_scores(list(combined_scores.items())) + fullpenalty_miner_scores
+    combined_weighed = Scoring.softmax_scores(list(combined_scores.items())) + fullpenalty_miner_scores
     combined_scores = dict(combined_weighed)
 
     ## Normalize the scores
@@ -168,22 +179,24 @@
         combined_data[miner]['Penalty'] = miner_penalties.get(miner, 0)
         combined_data[miner]['Drawdown Penalty'] = drawdown_penalties.get(miner, 0)
         combined_data[miner]['Consistency Penalty'] = consistency_penalties.get(miner, 0)
+        combined_data[miner]['Positional Penalty'] = positional_penalties.get(miner, 0)
 
     df = pd.DataFrame.from_dict(combined_data, orient='index')
 
-    # printing_columns = [
-    #     'return_cps_short Weighted Score',
-    #     'return_cps_long Weighted Score',
-    #     'Final Normalized Score',
-    #     'Final Rank',
-    #     'Penalty',
-    #     'Drawdown Penalty',
-    #     'Consistency Penalty',
-    # ]
-
-    # df_subset = df[printing_columns].round(3)
-    # df_subset = df_subset.sort_values(by='Final Rank', ascending=True)
-    # # print(df_subset)
+    printing_columns = [
+        'return_cps_short Weighted Score',
+        'return_cps_long Weighted Score',
+        'Final Normalized Score',
+        'Final Rank',
+        'Penalty',
+        'Drawdown Penalty',
+        'Consistency Penalty',
+        'Positional Penalty'
+    ]
+
+    df_subset = df[printing_columns].round(3)
+    df_subset = df_subset.sort_values(by='Final Rank', ascending=True)
+    print(df_subset.head(30))
 
     # Print rankings and original scores for each metric
     for metric, ranks in rankings.items():

vali_config.py

@@ -81,6 +81,10 @@ class ValiConfig:
     SET_WEIGHT_MINIMUM_POSITION_DURATION_MS = 1 * 60 * 1000  # 1 minutes
     SET_WEIGHT_MINIMUM_POSITION_DURATION_TOTAL_MS = SET_WEIGHT_MINIMUM_POSITION_DURATION_MS * SET_WEIGHT_MINIMUM_POSITIONS
     RETURN_DECAY_SHORT_LOOKBACK_TIME_MS = 3 * 24 * 60 * 60 * 1000  # 3 days
+    REALIZED_RETURN_EXPONENT = 4
+    SOFTMAX_TEMPERATURE = 0.5
+    MAX_RETURN_SIGMOID_SPREAD = 15
+    MAX_RETURN_SIGMOID_SHIFT = 0.5
 
     SET_WEIGHT_MINER_CHECKPOINT_CONSISTENCY_DISPLACEMENT = 15
     SET_WEIGHT_CHECKPOINT_CONSISTENCY_TAPER = SET_WEIGHT_MINER_CHECKPOINT_CONSISTENCY_DISPLACEMENT * 1.25
@@ -300,4 +304,4 @@ def __str__(self):
 
 
 TRADE_PAIR_ID_TO_TRADE_PAIR = {x.trade_pair_id: x for x in TradePair}
-TRADE_PAIR_STR_TO_TRADE_PAIR = {x.trade_pair: x for x in TradePair}
+TRADE_PAIR_STR_TO_TRADE_PAIR = {x.trade_pair: x for x in TradePair}

vali_objects/scoring/scoring.py

@@ -13,6 +13,8 @@
 from vali_objects.utils.vali_bkp_utils import ValiBkpUtils
 from vali_objects.utils.vali_utils import ValiUtils
 from vali_objects.vali_dataclasses.perf_ledger import PerfCheckpoint, PerfLedger
+from vali_objects.position import Position
+
 from vali_objects.utils.position_manager import PositionManager
 from time_util.time_util import TimeUtil
 from vali_objects.utils.position_utils import PositionUtils
@@ -63,6 +65,7 @@ class Scoring:
     @staticmethod
     def compute_results_checkpoint(
         ledger_dict: Dict[str, PerfLedger],
+        filtered_positions: Dict[str, list[Position]],
         evaluation_time_ms: int = None,
         verbose=True
     ) -> List[Tuple[str, float]]:
@@ -82,7 +85,7 @@ def compute_results_checkpoint(
         return_decay_short_lookback_time_ms = ValiConfig.RETURN_DECAY_SHORT_LOOKBACK_TIME_MS
 
         # Compute miner penalties
-        miner_penalties = Scoring.miner_penalties(ledger_dict)
+        miner_penalties = Scoring.miner_penalties(ledger_dict, filtered_positions)
 
         ## Miners with full penalty
         fullpenalty_miner_scores: list[tuple[str, float]] = [ ( miner, 0 ) for miner, penalty in miner_penalties.items() if penalty == 0 ]
@@ -134,23 +137,24 @@ def compute_results_checkpoint(
                 combined_scores[miner] *= config['weight'] * score + (1 - config['weight'])
 
         # ## Force good performance of all error metrics
-        combined_weighed = Scoring.weigh_miner_scores(list(combined_scores.items())) + fullpenalty_miner_scores
+        combined_weighed = Scoring.softmax_scores(list(combined_scores.items())) + fullpenalty_miner_scores
         combined_scores = dict(combined_weighed)
 
         ## Normalize the scores
-        normalized_scores = Scoring.normalize_scores(combined_scores)
-        return sorted(normalized_scores.items(), key=lambda x: x[1], reverse=True)
+        return sorted(combined_scores.items(), key=lambda x: x[1], reverse=True)
 
     @staticmethod
-    def miner_penalties(ledger_dict: dict[str, PerfLedger]) -> dict[str, float]:
+    def miner_penalties(ledger_dict: dict[str, PerfLedger], miner_positions: dict[str, list[Position]]) -> dict[str, float]:
         # Compute miner penalties
         miner_penalties = {}
         for miner, perfledger in ledger_dict.items():
             ledgercps = perfledger.cps
 
             consistency_penalty = PositionUtils.compute_consistency_penalty_cps(ledgercps)
             drawdown_penalty = PositionUtils.compute_drawdown_penalty_cps(ledgercps)
-            miner_penalties[miner] = drawdown_penalty * consistency_penalty
+            positional_penalty = PositionUtils.compute_positional_penalty_cps(ledgercps, miner_positions.get(miner, []))
+
+            miner_penalties[miner] = drawdown_penalty * consistency_penalty * positional_penalty
 
         return miner_penalties
 
@@ -438,7 +442,43 @@ def miner_scores_percentiles(miner_scores: list[tuple[str, float]]) -> list[tupl
 
         return miner_percentiles
 
+    @staticmethod
+    def softmax_scores(returns: list[tuple[str, float]]) -> list[tuple[str, float]]:
+        """
+        Assign weights to the returns based on their relative position and apply softmax with a temperature parameter.
+
+        The softmax function is used to convert the scores into probabilities that sum to 1.
+        Subtracting the max value from the scores before exponentiation improves numerical stability.
 
+        Parameters:
+        returns (list[tuple[str, float]]): List of tuples with miner names and their scores.
+        temperature (float): Temperature parameter to control the sharpness of the softmax distribution. Default is 1.0.
+
+        Returns:
+        list[tuple[str, float]]: List of tuples with miner names and their softmax weights.
+        """
+        epsilon = ValiConfig.EPSILON
+        temperature = ValiConfig.SOFTMAX_TEMPERATURE
+
+        if not returns:
+            bt.debug("No returns to score, returning empty list")
+            return []
+
+        if len(returns) == 1:
+            bt.info("Only one miner, returning 1.0 for the solo miner weight")
+            return [(returns[0][0], 1.0)]
+
+        # Extract scores and apply softmax with temperature
+        scores = [score for _, score in returns]
+        max_score = np.max(scores)
+        exp_scores = np.exp((scores - max_score) / temperature)
+        softmax_scores = exp_scores / max(np.sum(exp_scores), epsilon)
+
+        # Combine miners with their respective softmax scores
+        weighted_returns = [(returns[i][0], softmax_scores[i]) for i in range(len(returns))]
+
+        return weighted_returns
+
     @staticmethod
     def weigh_miner_scores(returns: list[tuple[str, float]]) -> list[tuple[str, float]]:
         """