Skip to content

Commit 909cf05

Browse files
igerberclaude
igerber
and
claude
committed
Filter NaN cells in bootstrap aggregation paths, add bootstrap NaN test, extend validator group fields
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 4d54cd7 commit 909cf05

File tree

4 files changed

+270
-25
lines changed

4 files changed

+270
-25
lines changed

‎benchmarks/speed_review/baseline_results.json‎

Lines changed: 168 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -414,15 +414,33 @@
414414
"group_effects": {
415415
"3": {
416416
"effect": 1.9301904330043325,
417-
"se": 0.030438318925920818
417+
"se": 0.030438318925920818,
418+
"t_stat": 63.41317461394397,
419+
"p_value": 0.005,
420+
"conf_int": [
421+
1.8702219368960384,
422+
1.9861420614466454
423+
]
418424
},
419425
"5": {
420426
"effect": 1.9809149332576672,
421-
"se": 0.03170174036975451
427+
"se": 0.03170174036975451,
428+
"t_stat": 62.48599951148382,
429+
"p_value": 0.005,
430+
"conf_int": [
431+
1.9184994180296897,
432+
2.0357338459955687
433+
]
422434
},
423435
"7": {
424436
"effect": 1.9717890565347787,
425-
"se": 0.03344476401759643
437+
"se": 0.03344476401759643,
438+
"t_stat": 58.956584519399,
439+
"p_value": 0.005,
440+
"conf_int": [
441+
1.9113592178120258,
442+
2.0361181246440454
443+
]
426444
}
427445
}
428446
},
@@ -841,15 +859,33 @@
841859
"group_effects": {
842860
"3": {
843861
"effect": 1.9295320405205874,
844-
"se": 0.03044125859655524
862+
"se": 0.03044125859655524,
863+
"t_stat": 63.38542259678234,
864+
"p_value": 0.005,
865+
"conf_int": [
866+
1.8695941117699424,
867+
1.9867835997412717
868+
]
845869
},
846870
"5": {
847871
"effect": 1.980830114174799,
848-
"se": 0.0316586783231788
872+
"se": 0.0316586783231788,
873+
"t_stat": 62.568313621751564,
874+
"p_value": 0.005,
875+
"conf_int": [
876+
1.9193890597593606,
877+
2.0362551053682205
878+
]
849879
},
850880
"7": {
851881
"effect": 1.9724360751947247,
852-
"se": 0.03346783119193577
882+
"se": 0.03346783119193577,
883+
"t_stat": 58.93528217836812,
884+
"p_value": 0.005,
885+
"conf_int": [
886+
1.912440579136104,
887+
2.036883663702173
888+
]
853889
}
854890
}
855891
},
@@ -1268,15 +1304,33 @@
12681304
"group_effects": {
12691305
"3": {
12701306
"effect": 1.9294692278922927,
1271-
"se": 0.030613393479888402
1307+
"se": 0.030613393479888402,
1308+
"t_stat": 63.02696331786496,
1309+
"p_value": 0.005,
1310+
"conf_int": [
1311+
1.8693199149275344,
1312+
1.983526736406974
1313+
]
12721314
},
12731315
"5": {
12741316
"effect": 1.9808312690492889,
1275-
"se": 0.03170591980009046
1317+
"se": 0.03170591980009046,
1318+
"t_stat": 62.47512393706482,
1319+
"p_value": 0.005,
1320+
"conf_int": [
1321+
1.9183357288880067,
1322+
2.036631351864895
1323+
]
12761324
},
12771325
"7": {
12781326
"effect": 1.9738890151454458,
1279-
"se": 0.03333757547853297
1327+
"se": 0.03333757547853297,
1328+
"t_stat": 59.20913524189814,
1329+
"p_value": 0.005,
1330+
"conf_int": [
1331+
1.912612275253619,
1332+
2.036232874071883
1333+
]
12801334
}
12811335
}
12821336
},
@@ -1695,15 +1749,33 @@
16951749
"group_effects": {
16961750
"3": {
16971751
"effect": 1.9295291363580418,
1698-
"se": 0.030432814553369162
1752+
"se": 0.030432814553369162,
1753+
"t_stat": 63.40291440919082,
1754+
"p_value": 0.005,
1755+
"conf_int": [
1756+
1.869928417389561,
1757+
1.9864120974980581
1758+
]
16991759
},
17001760
"5": {
17011761
"effect": 1.9808298091482444,
1702-
"se": 0.03166813271404776
1762+
"se": 0.03166813271404776,
1763+
"t_stat": 62.54962447689763,
1764+
"p_value": 0.005,
1765+
"conf_int": [
1766+
1.9193431683117772,
1767+
2.0362183499128976
1768+
]
17031769
},
17041770
"7": {
17051771
"effect": 1.9724482314870557,
1706-
"se": 0.03348899468270557
1772+
"se": 0.03348899468270557,
1773+
"t_stat": 58.89840080824135,
1774+
"p_value": 0.005,
1775+
"conf_int": [
1776+
1.9113054655520512,
1777+
2.0358791270186636
1778+
]
17071779
}
17081780
}
17091781
},
@@ -2122,15 +2194,33 @@
21222194
"group_effects": {
21232195
"3": {
21242196
"effect": 1.9295131516250037,
2125-
"se": 0.030428651624796504
2197+
"se": 0.030428651624796504,
2198+
"t_stat": 63.41106321164199,
2199+
"p_value": 0.005,
2200+
"conf_int": [
2201+
1.8693318914387955,
2202+
1.9852891590817527
2203+
]
21262204
},
21272205
"5": {
21282206
"effect": 1.980829710949667,
2129-
"se": 0.031711555960950315
2207+
"se": 0.031711555960950315,
2208+
"t_stat": 62.463970969726795,
2209+
"p_value": 0.005,
2210+
"conf_int": [
2211+
1.9183682933830388,
2212+
2.035608652828243
2213+
]
21302214
},
21312215
"7": {
21322216
"effect": 1.972400906646088,
2133-
"se": 0.033466935554797926
2217+
"se": 0.033466935554797926,
2218+
"t_stat": 58.93580855099589,
2219+
"p_value": 0.005,
2220+
"conf_int": [
2221+
1.9108155783646885,
2222+
2.035952061830552
2223+
]
21342224
}
21352225
}
21362226
},
@@ -2549,15 +2639,33 @@
25492639
"group_effects": {
25502640
"3": {
25512641
"effect": 1.9379600572395599,
2552-
"se": 0.027359714676020434
2642+
"se": 0.027359714676020434,
2643+
"t_stat": 70.8326121155823,
2644+
"p_value": 0.005,
2645+
"conf_int": [
2646+
1.8889964928323424,
2647+
1.9847938462368524
2648+
]
25532649
},
25542650
"5": {
25552651
"effect": 1.9776177086894955,
2556-
"se": 0.029511266225510993
2652+
"se": 0.029511266225510993,
2653+
"t_stat": 67.01229603560505,
2654+
"p_value": 0.005,
2655+
"conf_int": [
2656+
1.9223093399752835,
2657+
2.0294569469461097
2658+
]
25572659
},
25582660
"7": {
25592661
"effect": 1.972400906646088,
2560-
"se": 0.033466935554797926
2662+
"se": 0.033466935554797926,
2663+
"t_stat": 58.93580855099589,
2664+
"p_value": 0.005,
2665+
"conf_int": [
2666+
1.9108155783646885,
2667+
2.035952061830552
2668+
]
25612669
}
25622670
}
25632671
},
@@ -2976,15 +3084,33 @@
29763084
"group_effects": {
29773085
"3": {
29783086
"effect": 1.9379732080320693,
2979-
"se": 0.027348748599256394
3087+
"se": 0.027348748599256394,
3088+
"t_stat": 70.86149485043576,
3089+
"p_value": 0.005,
3090+
"conf_int": [
3091+
1.8890625503369387,
3092+
1.9846384878549268
3093+
]
29803094
},
29813095
"5": {
29823096
"effect": 1.9776180319767032,
2983-
"se": 0.02946287209657048
3097+
"se": 0.02946287209657048,
3098+
"t_stat": 67.12237780127691,
3099+
"p_value": 0.005,
3100+
"conf_int": [
3101+
1.9220032490097771,
3102+
2.0295504746249913
3103+
]
29843104
},
29853105
"7": {
29863106
"effect": 1.9724482314870557,
2987-
"se": 0.03348899468270557
3107+
"se": 0.03348899468270557,
3108+
"t_stat": 58.89840080824135,
3109+
"p_value": 0.005,
3110+
"conf_int": [
3111+
1.9113054655520512,
3112+
2.0358791270186636
3113+
]
29883114
}
29893115
}
29903116
},
@@ -3403,15 +3529,33 @@
34033529
"group_effects": {
34043530
"3": {
34053531
"effect": 1.9379731464282752,
3406-
"se": 0.027354393932262182
3532+
"se": 0.027354393932262182,
3533+
"t_stat": 70.84686837614781,
3534+
"p_value": 0.005,
3535+
"conf_int": [
3536+
1.8891430541850542,
3537+
1.9844547989387653
3538+
]
34073539
},
34083540
"5": {
34093541
"effect": 1.977618261825107,
3410-
"se": 0.029456139694155045
3542+
"se": 0.029456139694155045,
3543+
"t_stat": 67.13772688338805,
3544+
"p_value": 0.005,
3545+
"conf_int": [
3546+
1.9219215453418508,
3547+
2.02949447907519
3548+
]
34113549
},
34123550
"7": {
34133551
"effect": 1.9724360751947247,
3414-
"se": 0.03346783119193577
3552+
"se": 0.03346783119193577,
3553+
"t_stat": 58.93528217836812,
3554+
"p_value": 0.005,
3555+
"conf_int": [
3556+
1.912440579136104,
3557+
2.036883663702173
3558+
]
34153559
}
34163560
}
34173561
}

‎benchmarks/speed_review/validate_results.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ def run_estimator(df, estimation_method="reg", covariates=None, control_group="n
107107
ge[str(g_key)] = {
108108
"effect": float(data["effect"]),
109109
"se": float(data["se"]),
110+
"t_stat": float(data["t_stat"]),
111+
"p_value": float(data["p_value"]),
112+
"conf_int": [float(data["conf_int"][0]), float(data["conf_int"][1])],
110113
}
111114
out["group_effects"] = ge
112115

@@ -235,6 +238,13 @@ def compare(label, base_val, cur_val, t):
235238
continue
236239
compare(f"{name}/ge[{g_key}].effect", b_ge["effect"], c_ge["effect"], scenario_tol)
237240
compare(f"{name}/ge[{g_key}].se", b_ge["se"], c_ge["se"], gt_se_tol)
241+
if "t_stat" in b_ge and "t_stat" in c_ge:
242+
compare(f"{name}/ge[{g_key}].t_stat", b_ge["t_stat"], c_ge["t_stat"], gt_se_tol)
243+
if "p_value" in b_ge and "p_value" in c_ge:
244+
compare(f"{name}/ge[{g_key}].p_value", b_ge["p_value"], c_ge["p_value"], 0.02)
245+
if "conf_int" in b_ge and "conf_int" in c_ge:
246+
for i, label in enumerate(["lower", "upper"]):
247+
compare(f"{name}/ge[{g_key}].ci.{label}", b_ge["conf_int"][i], c_ge["conf_int"][i], gt_se_tol)
238248

239249
if failures:
240250
all_failures.extend(failures)

‎diff_diff/staggered_bootstrap.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,15 @@ def _run_multiplier_bootstrap(
204204
], dtype=float)
205205
post_n_treated = all_n_treated[post_treatment_mask]
206206

207+
# Filter out NaN ATT(g,t) cells from overall aggregation (matches analytical path)
208+
post_effects_raw = np.array([
209+
group_time_effects[gt_pairs[i]]['effect'] for i in post_treatment_indices
210+
])
211+
finite_post = np.isfinite(post_effects_raw)
212+
if not np.all(finite_post):
213+
post_treatment_indices = post_treatment_indices[finite_post]
214+
post_n_treated = post_n_treated[finite_post]
215+
207216
# Flag to skip overall ATT aggregation when no post-treatment effects
208217
# But continue bootstrap for per-effect SEs (pre-treatment effects need bootstrap SEs too)
209218
skip_overall_aggregation = False
@@ -297,7 +306,7 @@ def _run_multiplier_bootstrap(
297306
# Use combined IF (standard IF + WIF) for proper bootstrap
298307
post_gt_pairs = [gt_pairs[i] for i in post_treatment_indices]
299308
post_groups = np.array([gt_pairs[i][0] for i in post_treatment_indices])
300-
post_effects = original_atts[post_treatment_mask]
309+
post_effects = original_atts[post_treatment_indices]
301310
overall_combined_if, _ = self._compute_combined_influence_function(
302311
post_gt_pairs, overall_weights_post, post_effects, post_groups,
303312
influence_func_info, df, unit, precomputed,
@@ -503,6 +512,15 @@ def _prepare_event_study_aggregation(
503512
effects = np.array([x[1] for x in effect_list])
504513
n_treated = np.array([x[2] for x in effect_list], dtype=float)
505514

515+
# Exclude NaN effects (matches analytical aggregation path)
516+
finite_mask = np.isfinite(effects)
517+
if not np.all(finite_mask):
518+
indices = indices[finite_mask]
519+
effects = effects[finite_mask]
520+
n_treated = n_treated[finite_mask]
521+
if len(effects) == 0:
522+
continue
523+
506524
weights = n_treated / np.sum(n_treated)
507525
agg_effect = np.sum(weights * effects)
508526

@@ -553,6 +571,14 @@ def _prepare_group_aggregation(
553571
indices = np.array([x[0] for x in group_data])
554572
effects = np.array([x[1] for x in group_data])
555573

574+
# Exclude NaN effects (matches analytical aggregation path)
575+
finite_mask = np.isfinite(effects)
576+
if not np.all(finite_mask):
577+
indices = indices[finite_mask]
578+
effects = effects[finite_mask]
579+
if len(effects) == 0:
580+
continue
581+
556582
# Equal weights across time periods
557583
weights = np.ones(len(effects)) / len(effects)
558584
agg_effect = np.sum(weights * effects)

0 commit comments

Comments
 (0)