Skip to content

Commit 11d49c0

Browse files
thebigjcthebigjc
committed
Merge branch 'clean-find-best-feature' into thewarwithin
2 parents e40f81d + 7abe007 commit 11d49c0

File tree

7 files changed

+527
-3
lines changed

7 files changed

+527
-3
lines changed

‎engine/report/json/report_json.cpp‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1022,6 +1022,24 @@ void profileset_json2( const profileset::profilesets_t& profileset, const sim_t&
10221022

10231023
obj[ "iterations" ] = as<uint64_t>( result.iterations() );
10241024

1025+
if ( sim.profileset_cull.enabled )
1026+
{
1027+
obj[ "profileset_cull_best" ] = ( sim.profileset_cull.best_name == profileset->name() );
1028+
if ( sim.profileset_cull.best_name == profileset->name() )
1029+
{
1030+
obj[ "profileset_cull_best_error" ] = sim.profileset_cull.best_error;
1031+
}
1032+
if ( profileset->culled() )
1033+
{
1034+
obj[ "culled" ] = true;
1035+
obj[ "culled_reason" ] = profileset->culled_reason();
1036+
obj[ "culled_iterations" ] = profileset->culled_iterations();
1037+
obj[ "culled_mean" ] = profileset->culled_mean();
1038+
obj[ "culled_error" ] = profileset->culled_error();
1039+
obj[ "culled_error_type" ] = profileset->culled_error_type_cstr();
1040+
}
1041+
}
1042+
10251043
if ( profileset->results() > 1 )
10261044
{
10271045
auto results2 = obj[ "additional_metrics" ].make_array();
@@ -1068,6 +1086,25 @@ void profileset_json3( const profileset::profilesets_t& profilesets, const sim_t
10681086
obj[ "name" ] = profileset->name();
10691087
auto results_obj = obj[ "metrics" ].make_array();
10701088

1089+
// Profileset culling metadata at profileset level
1090+
if ( sim.profileset_cull.enabled )
1091+
{
1092+
obj[ "profileset_cull_best" ] = ( sim.profileset_cull.best_name == profileset->name() );
1093+
if ( sim.profileset_cull.best_name == profileset->name() )
1094+
{
1095+
obj[ "profileset_cull_best_error" ] = sim.profileset_cull.best_error;
1096+
}
1097+
if ( profileset->culled() )
1098+
{
1099+
obj[ "culled" ] = true;
1100+
obj[ "culled_reason" ] = profileset->culled_reason();
1101+
obj[ "culled_iterations" ] = profileset->culled_iterations();
1102+
obj[ "culled_mean" ] = profileset->culled_mean();
1103+
obj[ "culled_error" ] = profileset->culled_error();
1104+
obj[ "culled_error_type" ] = profileset->culled_error_type_cstr();
1105+
}
1106+
}
1107+
10711108
for ( size_t midx = 0; midx < sim.profileset_metric.size(); ++midx )
10721109
{
10731110
const auto& result = profileset->result( sim.profileset_metric[ midx ] );
@@ -1231,6 +1268,19 @@ void to_json( const ::report::json::report_configuration_t& report_configuration
12311268
options_root[ "profileset_metric" ] = util::scale_metric_type_abbrev( sim.profileset_metric.front() );
12321269
options_root[ "profileset_multiactor_base_name" ] = sim.profileset_multiactor_base_name;
12331270

1271+
if ( sim.profileset_cull.enabled )
1272+
{
1273+
auto cull = options_root[ "profileset_cull" ];
1274+
cull[ "enabled" ] = true;
1275+
cull[ "method" ] = sim.profileset_cull.method_name();
1276+
cull[ "min_iterations" ] = sim.profileset_cull.min_iterations;
1277+
if ( sim.profileset_cull.method == sim_t::profileset_cull_state_t::T_TEST )
1278+
cull[ "alpha" ] = sim.profileset_cull.alpha;
1279+
else
1280+
cull[ "margin" ] = sim.profileset_cull.margin;
1281+
cull[ "metric" ] = util::scale_metric_type_abbrev( sim.profileset_cull.metric );
1282+
}
1283+
12341284
to_json( options_root[ "dbc" ], *sim.dbc );
12351285

12361286
if ( sim.scaling->calculate_scale_factors )

‎engine/report/report_html_sim.cpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1351,6 +1351,43 @@ void print_profilesets( std::ostream& out, const profileset::profilesets_t& prof
13511351
out << "<h2 class=\"toggle open\">Profile sets</h2>\n";
13521352
out << "<div class=\"toggle-content\">\n";
13531353

1354+
// Profileset culling indicator and culled list
1355+
if ( sim.profileset_cull.enabled )
1356+
{
1357+
out << "<div class=\"note\" style=\"margin:6px 0;\">";
1358+
out << "Profileset culling enabled: method="
1359+
<< ( sim.profileset_cull.method == sim_t::profileset_cull_state_t::T_TEST ? "t_test" : "ci" )
1360+
<< ", min_iters=" << sim.profileset_cull.min_iterations;
1361+
if ( sim.profileset_cull.method == sim_t::profileset_cull_state_t::T_TEST )
1362+
out << ", alpha=" << sim.profileset_cull.alpha;
1363+
else
1364+
out << ", margin=" << sim.profileset_cull.margin;
1365+
out << "</div>\n";
1366+
1367+
// List culled profiles if any
1368+
bool any_culled = false;
1369+
for ( const auto& pset : profilesets.profilesets() )
1370+
{
1371+
if ( pset->culled() ) { any_culled = true; break; }
1372+
}
1373+
if ( any_culled )
1374+
{
1375+
out << "<div class=\"note\" style=\"margin:6px 0;\"><strong>Culled profiles:</strong><ul>";
1376+
for ( const auto& pset : profilesets.profilesets() )
1377+
{
1378+
if ( !pset->culled() ) continue;
1379+
out << "<li>" << util::encode_html( pset->name() )
1380+
<< ": " << util::encode_html( pset->culled_reason() )
1381+
<< " (iters=" << pset->culled_iterations()
1382+
<< ", mean=" << util::round( pset->culled_mean(), 2 )
1383+
<< ", error=" << util::round( pset->culled_error(), 4 )
1384+
<< ", type=" << pset->culled_error_type_cstr() << ")";
1385+
out << "</li>";
1386+
}
1387+
out << "</ul></div>\n";
1388+
}
1389+
}
1390+
13541391
print_profilesets_chart( out, sim );
13551392

13561393
out << "</div>";

‎engine/sc_main.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,10 @@ int sim_t::main( const std::vector<std::string>& args )
359359
plot->analyze();
360360
reforge_plot->analyze();
361361

362+
if ( profileset_cull.enabled ) {
363+
seed_profileset_cull_from_baseline();
364+
}
365+
362366
if ( canceled == 0 && !profilesets->iterate( this ) )
363367
{
364368
canceled = true;

‎engine/sim/profileset.cpp‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ void simulate_profileset( sim_t* parent, profileset::profile_set_t& set, sim_t*&
110110
// Reset random seed for the profileset sims
111111
profile_sim -> seed = 0;
112112
profile_sim -> profileset_enabled = true;
113+
profile_sim -> profileset_current_name = set.name();
113114
profile_sim -> report_details = 0;
114115
if ( parent -> profileset_work_threads > 0 )
115116
{
@@ -137,6 +138,10 @@ void simulate_profileset( sim_t* parent, profileset::profile_set_t& set, sim_t*&
137138

138139
if ( !ret || profile_sim -> is_canceled() )
139140
{
141+
if ( profile_sim->culled )
142+
{
143+
fmt::print( stderr, "\nProfileset '{}' culled: {}\n", set.name(), profile_sim->culled_reason );
144+
}
140145
return;
141146
}
142147

@@ -156,6 +161,22 @@ void simulate_profileset( sim_t* parent, profileset::profile_set_t& set, sim_t*&
156161
.stddev( data.std_dev )
157162
.mean_stddev( data.mean_std_dev )
158163
.iterations( progress.current_iterations );
164+
165+
// If culled, persist snapshot information for JSON/HTML reporting on primary metric only
166+
if ( profile_sim->culled && metric == parent->profileset_metric.front() )
167+
{
168+
// error to record depends on method: CI mode wants half-width, t-test wants SE
169+
auto etype = ( parent->profileset_cull.prefers_standard_error() ) ?
170+
profileset::profile_set_t::cull_error_type_e::STANDARD_ERROR :
171+
profileset::profile_set_t::cull_error_type_e::CI_HALF_WIDTH;
172+
double err_val = parent->profileset_cull.select_error(data.mean_std_dev * parent->confidence_estimator, data.mean_std_dev / sqrt(parent->iterations) );
173+
set.set_culled( true,
174+
profile_sim->culled_reason,
175+
progress.current_iterations,
176+
data.mean,
177+
err_val,
178+
etype );
179+
}
159180
} );
160181

161182
if ( ! parent -> profileset_output_data.empty() )
@@ -174,6 +195,11 @@ void simulate_profileset( sim_t* parent, profileset::profile_set_t& set, sim_t*&
174195
parent -> event_mgr.total_events_processed += profile_sim -> event_mgr.total_events_processed;
175196

176197
set.cleanup_options();
198+
199+
if ( profile_sim->culled )
200+
{
201+
fmt::print( stderr, "\nProfileset '{}' culled: {}\n", set.name(), profile_sim->culled_reason );
202+
}
177203
}
178204

179205
// Figure out if the option defines new actor(s) with their own scope

‎engine/sim/profileset.hpp‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,19 @@ class profile_set_t
384384
std::vector<profile_result_t> m_results;
385385
std::unique_ptr<profile_output_data_t> m_output_data;
386386

387+
// Culled metadata (set when profileset culling terminates a run early)
388+
bool m_culled = false;
389+
std::string m_culled_reason;
390+
uint64_t m_culled_iterations = 0;
391+
double m_culled_mean = 0.0;
392+
double m_culled_error = 0.0;
393+
// CI half-width or standard error depending on cull method
394+
395+
public:
396+
enum class cull_error_type_e { NONE = 0, CI_HALF_WIDTH, STANDARD_ERROR };
397+
private:
398+
cull_error_type_e m_culled_error_type = cull_error_type_e::NONE;
399+
387400
public:
388401
profile_set_t( std::string name, sim_control_t* opts, bool has_output );
389402

@@ -415,6 +428,35 @@ class profile_set_t
415428

416429
return *m_output_data;
417430
}
431+
432+
// Culled metadata accessors
433+
bool culled() const { return m_culled; }
434+
const std::string& culled_reason() const { return m_culled_reason; }
435+
uint64_t culled_iterations() const { return m_culled_iterations; }
436+
double culled_mean() const { return m_culled_mean; }
437+
double culled_error() const { return m_culled_error; }
438+
profile_set_t::cull_error_type_e culled_error_type() const { return m_culled_error_type; }
439+
const char* culled_error_type_cstr() const {
440+
switch ( m_culled_error_type ) {
441+
case profile_set_t::cull_error_type_e::CI_HALF_WIDTH: return "ci_half_width";
442+
case profile_set_t::cull_error_type_e::STANDARD_ERROR: return "standard_error";
443+
default: return "none";
444+
}
445+
}
446+
void set_culled( bool culled,
447+
std::string reason,
448+
uint64_t iterations,
449+
double mean,
450+
double error,
451+
profile_set_t::cull_error_type_e etype )
452+
{
453+
m_culled = culled;
454+
m_culled_reason = std::move( reason );
455+
m_culled_iterations = iterations;
456+
m_culled_mean = mean;
457+
m_culled_error = error;
458+
m_culled_error_type = etype;
459+
}
418460
};
419461

420462
class worker_t

0 commit comments

Comments
 (0)