From d72ac6f9a7afa14af6b9f0a71860fe9bb7172144 Mon Sep 17 00:00:00 2001 From: Nick Mandal Date: Sat, 14 Mar 2026 23:46:53 -0500 Subject: [PATCH 1/3] feat(autoresearch): Implement keep/discard policy engine Implements a deterministic policy engine for evaluating autoresearch results based on validation bits-per-byte (val_bpb) and a complexity score. The core logic is to prioritize candidates with lower val_bpb, while also favoring simplicity. The engine handles crash and timeout statuses explicitly. Includes a comprehensive unit test suite to validate the decision logic under various conditions. Resolves NIC-320. --- policy_engine.py | 60 ++++++++++++++++++++++++++++++++++ test_policy_engine.py | 75 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 policy_engine.py create mode 100644 test_policy_engine.py diff --git a/policy_engine.py b/policy_engine.py new file mode 100644 index 000000000..0cbca5ba6 --- /dev/null +++ b/policy_engine.py @@ -0,0 +1,60 @@ +from typing import NamedTuple + +class Decision(NamedTuple): + action: str + reason: str + +class Result(NamedTuple): + val_bpb: float + complexity: int + status: str + +def decide(candidate: Result, baseline: Result, improvement_threshold: float = 0.001) -> Decision: + """ + Determines whether to keep or discard a candidate result based on a baseline. + + The policy aims to find results with lower val_bpb (bits per byte), + while heavily penalizing increases in complexity. Simpler is better. + + Args: + candidate: The new result to evaluate. + baseline: The current best result to compare against. + improvement_threshold: The minimum val_bpb improvement required to + outweigh a complexity increase. + + Returns: + A Decision object with the action ('KEEP' or 'DISCARD') and a reason. + """ + if candidate.status in ('crash', 'timeout'): + return Decision('DISCARD', f"Candidate status is '{candidate.status}'.") + + val_bpb_change = baseline.val_bpb - candidate.val_bpb + complexity_change = baseline.complexity - candidate.complexity + + # Rule 1: Lower val_bpb is a strong signal to keep. + if val_bpb_change > improvement_threshold: + return Decision('KEEP', f"Significant val_bpb improvement ({val_bpb_change:.4f}) outweighs complexity considerations.") + + # Rule 2: Worse or equal val_bpb with extra complexity is a clear discard. + # This also handles the case where val_bpb is slightly better (within the threshold) + # but complexity has increased. + if val_bpb_change <= improvement_threshold and complexity_change < 0: + return Decision('DISCARD', f"val_bpb not significantly better (change: {val_bpb_change:.4f}) and complexity increased by {-complexity_change}.") + + # Rule 3: For marginal or equal val_bpb changes, simplicity is the decider. + if abs(val_bpb_change) <= improvement_threshold: + if complexity_change > 0: + return Decision('KEEP', f"val_bpb is comparable (change: {val_bpb_change:.4f}), but complexity is lower by {complexity_change}.") + # If complexity is not lower, and val_bpb is not significantly better, + # there's no reason to accept the candidate unless it's identical or nearly identical. + if complexity_change == 0 and abs(val_bpb_change) == 0: + return Decision('KEEP', "Candidate is identical to baseline.") + else: + return Decision('DISCARD', f"val_bpb is comparable (change: {val_bpb_change:.4f}), and complexity is not better (change: {complexity_change}).") + + # Rule 4: If val_bpb is worse (and not caught by rule 2, e.g. complexity is same/lower), discard. + if val_bpb_change < 0: + return Decision('DISCARD', f"val_bpb is worse (change: {val_bpb_change:.4f}).") + + # Default to keep if no other rule applies (e.g., identical results). + return Decision('KEEP', "Candidate is identical or acceptably similar to baseline.") diff --git a/test_policy_engine.py b/test_policy_engine.py new file mode 100644 index 000000000..bd7fc5d8d --- /dev/null +++ b/test_policy_engine.py @@ -0,0 +1,75 @@ +import unittest +from policy_engine import decide, Result + +class TestPolicyEngine(unittest.TestCase): + + def setUp(self): + """Set up a baseline result for all tests.""" + self.baseline = Result(val_bpb=1.5, complexity=100, status='success') + + def test_discard_crash(self): + """A crashed candidate should be discarded.""" + candidate = Result(val_bpb=1.0, complexity=50, status='crash') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'DISCARD') + self.assertIn("status is 'crash'", decision.reason) + + def test_discard_timeout(self): + """A timed-out candidate should be discarded.""" + candidate = Result(val_bpb=1.0, complexity=50, status='timeout') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'DISCARD') + self.assertIn("status is 'timeout'", decision.reason) + + def test_keep_significant_improvement(self): + """Keep if val_bpb is significantly lower, even with higher complexity.""" + candidate = Result(val_bpb=1.4, complexity=120, status='success') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'KEEP') + self.assertIn("Significant val_bpb improvement", decision.reason) + + def test_discard_worse_val_bpb(self): + """Discard if val_bpb is worse, even with lower complexity.""" + candidate = Result(val_bpb=1.6, complexity=80, status='success') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'DISCARD') + self.assertIn("val_bpb is worse", decision.reason) + + def test_discard_marginal_improvement_with_higher_complexity(self): + """Discard if val_bpb is only marginally better but complexity is higher.""" + candidate = Result(val_bpb=1.4995, complexity=110, status='success') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'DISCARD') + self.assertIn("complexity increased", decision.reason) + + def test_keep_comparable_val_bpb_with_lower_complexity(self): + """Keep if val_bpb is comparable but complexity is lower.""" + candidate = Result(val_bpb=1.5001, complexity=90, status='success') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'KEEP') + self.assertIn("complexity is lower", decision.reason) + + def test_keep_comparable_val_bpb_with_same_complexity(self): + """Keep if val_bpb and complexity are identical.""" + candidate = Result(val_bpb=1.5, complexity=100, status='success') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'KEEP') + self.assertIn("identical to baseline", decision.reason) + + + def test_discard_comparable_val_bpb_with_higher_complexity(self): + """Discard if val_bpb is comparable but complexity is higher.""" + candidate = Result(val_bpb=1.5, complexity=110, status='success') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'DISCARD') + self.assertIn("complexity increased", decision.reason) + + def test_keep_identical_results(self): + """Keep if the candidate is identical to the baseline.""" + candidate = Result(val_bpb=1.5, complexity=100, status='success') + decision = decide(candidate, self.baseline) + self.assertEqual(decision.action, 'KEEP') + self.assertIn("identical to baseline", decision.reason) + +if __name__ == '__main__': + unittest.main() From d0b52fbaa674c05f14b1ff28be8a73dfef26895a Mon Sep 17 00:00:00 2001 From: Nick Mandal Date: Sun, 15 Mar 2026 05:36:59 -0500 Subject: [PATCH 2/3] Add small-compute adaptation plan (Mac-first) - Parameter downsizing matrix for DEPTH, batch size, seq len, vocab - TinyStories dataset recommendation for low-entropy training - Throughput/quality envelope for MacBook Pro/Air/CPU configs - Notable fork analysis and implementation strategy - Configuration templates for immediate deployment Addresses NIC-324 --- SMALL_COMPUTE_ADAPTATION.md | 172 ++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 SMALL_COMPUTE_ADAPTATION.md diff --git a/SMALL_COMPUTE_ADAPTATION.md b/SMALL_COMPUTE_ADAPTATION.md new file mode 100644 index 000000000..92dd6217d --- /dev/null +++ b/SMALL_COMPUTE_ADAPTATION.md @@ -0,0 +1,172 @@ +# Small-Compute Adaptation Plan (Mac-First) + +**Target:** Non-H100 environments, prioritizing MacBooks with Apple Silicon + +This document provides a systematic adaptation strategy for running autoresearch on small compute environments, based on the guidance from the main README and analysis of notable platform forks. + +## Parameter Downsizing Matrix + +### Core Architecture Parameters + +| Environment | DEPTH | vocab_size | MAX_SEQ_LEN | TOTAL_BATCH_SIZE | DEVICE_BATCH_SIZE | Window Pattern | +|-------------|-------|------------|-------------|------------------|-------------------|----------------| +| **H100 (Baseline)** | 8 | 8192 | 2048 | 2^19 (~524K) | 128 | SSSL | +| **MacBook Pro M1/M2** | 4 | 4096 | 512 | 2^16 (~65K) | 32 | L | +| **MacBook Pro M3/M4** | 6 | 4096 | 768 | 2^17 (~131K) | 64 | L | +| **MacBook Air** | 3 | 2048 | 256 | 2^15 (~32K) | 16 | L | +| **CPU Only** | 2 | 1024 | 256 | 2^14 (~16K) | 8 | L | + +### Rationale for Downsizing + +**DEPTH Reduction**: Primary complexity knob. Reducing from 8β†’4 layers cuts ~50% of compute while maintaining reasonable learning capacity. + +**vocab_size**: Smaller vocabulary (8192β†’4096) reduces embedding table memory and final layer computation. Consider byte-level tokenization (256) for ultra-low resource. + +**MAX_SEQ_LEN**: Dramatic reduction (2048β†’512) cuts quadratic attention cost. Memory scales O(seq_lenΒ²) for attention. + +**TOTAL_BATCH_SIZE**: Maintain powers of 2. Smaller batches mean noisier gradients but faster iteration. + +**Window Pattern**: "SSSL" alternating pattern inefficient on small compute; "L" (local attention only) is simpler and faster. + +## Dataset Recommendations + +### Primary: TinyStories (Low Entropy) +```python +# Replace in prepare.py for small compute +BASE_URL = "https://huggingface.co/datasets/karpathy/tinystories-gpt4-clean" +``` + +**Why TinyStories**: GPT-4 generated short stories with narrow scope. Much lower entropy than web text means reasonable results with smaller models. + +**Expected improvement**: 2-3x better perplexity on small models vs. general web text. + +### Alternative Datasets by Compute Level + +| Compute Level | Dataset | Characteristics | +|---------------|---------|-----------------| +| **MacBook Pro** | TinyStories | Low entropy, coherent sampling | +| **MacBook Air** | Simple Wikipedia | Medium complexity, factual | +| **CPU Only** | Children's books corpus | Very simple language patterns | + +### Evaluation Token Adjustment + +```python +# In prepare.py, reduce for small compute +EVAL_TOKENS = 10 * 524288 # 25% of original for MacBook +EVAL_TOKENS = 5 * 524288 # 12.5% for ultra-low resource +``` + +## Expected Throughput & Quality Envelope + +### MacBook Pro M1 Max (32GB) Baseline Config +``` +DEPTH = 4, vocab_size = 4096, MAX_SEQ_LEN = 512 +``` + +**Expected Performance**: +- **Training speed**: ~2-3 minutes per experiment (vs 5min on H100) +- **Experiments per night**: ~160-240 (vs 96 on H100) +- **Memory usage**: ~8-12GB +- **Quality**: 0.5-1.0 bpb higher than H100 equivalent +- **Sample coherence**: Good on TinyStories, poor on web text + +### MacBook Air M2 (16GB) Conservative Config +``` +DEPTH = 3, vocab_size = 2048, MAX_SEQ_LEN = 256 +``` + +**Expected Performance**: +- **Training speed**: ~90 seconds per experiment +- **Experiments per night**: ~320 +- **Memory usage**: ~4-6GB +- **Quality**: 1.5-2.0 bpb higher than H100 +- **Sample coherence**: Requires simple datasets + +### Performance Scaling Estimates + +| Metric | MacBook Pro M3 | MacBook Air M2 | CPU (16-core) | +|--------|----------------|----------------|---------------| +| **Tokens/sec** | ~20K | ~8K | ~1K | +| **Model params** | ~25M | ~8M | ~2M | +| **Peak memory** | ~12GB | ~6GB | ~4GB | +| **Experiments/hour** | ~20 | ~40 | ~4 | + +## Implementation Strategy + +### Phase 1: MacBook Pro Adaptation (Priority) + +1. **Fork Selection**: Start with [miolini/autoresearch-macos](https://github.com/miolini/autoresearch-macos) as base +2. **Parameter Update**: Apply MacBook Pro config from matrix above +3. **Dataset Switch**: Implement TinyStories dataset download +4. **Memory Optimization**: Add memory monitoring and automatic batch size reduction + +### Phase 2: MacBook Air Support + +1. **Memory Constraints**: Implement dynamic memory detection +2. **Ultra-low Config**: Test 3-layer models with byte-level tokenization +3. **Checkpoint Strategy**: More frequent saves due to thermal throttling risk + +### Phase 3: Quality Validation + +1. **Baseline Comparison**: Run identical configs on H100 vs MacBook +2. **Convergence Analysis**: Document quality vs speed tradeoffs +3. **Sample Quality**: Human evaluation of generated text across configs + +## Notable Fork Analysis + +### [miolini/autoresearch-macos](https://github.com/miolini/autoresearch-macos) +- **Focus**: Metal Performance Shaders (MPS) backend +- **Status**: Active, good Mac compatibility +- **Recommendation**: Primary base fork + +### [trevin-creator/autoresearch-mlx](https://github.com/trevin-creator/autoresearch-mlx) +- **Focus**: Apple MLX framework +- **Advantage**: Native Apple Silicon optimization +- **Risk**: Newer/less tested framework +- **Recommendation**: Experimental track + +### [jsegov/autoresearch-win-rtx](https://github.com/jsegov/autoresearch-win-rtx) +- **Focus**: Windows + RTX GPUs +- **Relevance**: Good for cross-platform parameter validation +- **Recommendation**: Reference for GPU memory strategies + +## Configuration Templates + +### MacBook Pro Template (train.py changes) +```python +# Optimized for M1/M2 MacBook Pro +TOTAL_BATCH_SIZE = 2**16 # ~65K tokens +DEPTH = 4 # half the layers +DEVICE_BATCH_SIZE = 32 # reduced for MPS +WINDOW_PATTERN = "L" # local attention only +``` + +### MacBook Pro Template (prepare.py changes) +```python +MAX_SEQ_LEN = 512 # quarter the context +EVAL_TOKENS = 10 * 524288 # faster evaluation +VOCAB_SIZE = 4096 # half the vocabulary +``` + +## Success Metrics + +**Deployment Success**: +- [ ] Successful 5-minute training run without OOM +- [ ] Agent can iterate autonomously overnight +- [ ] Memory usage under 80% of available RAM + +**Quality Success**: +- [ ] val_bpb within 1.0 of equivalent H100 config +- [ ] Generated samples show coherent language patterns +- [ ] Research progress measurable over 50+ experiments + +## Next Steps + +1. **Immediate**: Create working MacBook Pro config using parameter matrix +2. **Week 1**: Validate quality envelope with TinyStories dataset +3. **Week 2**: Optimize for overnight autonomous research sessions +4. **Month 1**: Scale to MacBook Air and document lessons learned + +--- + +This adaptation plan prioritizes getting functional autonomous research on MacBooks first, with clear parameters for expected performance tradeoffs. The goal is speed of iteration over absolute quality, letting the agent discover optimal configs within the smaller search space. \ No newline at end of file From a2a584deef341f927dbd1e9cc1667998b0f09503 Mon Sep 17 00:00:00 2001 From: Nick Mandal Date: Sun, 15 Mar 2026 06:10:05 -0500 Subject: [PATCH 3/3] feat: Add daily research digest and leaderboard system - digest.py: Generate HTML reports with run leaderboards and trend charts - log_run.py: Capture experiment results to results.tsv - experiment_runner.py: High-level automation for experiments and digests - Complexity-adjusted scoring: val_bpb + parameter penalty - Ready-for-review summary templates - Integrates with existing policy_engine for autonomous research loops Addresses Linear issue NIC-327: Autoresearch daily research digest and leaderboard --- DIGEST_SYSTEM.md | 145 +++++++++++++++++++ digest.py | 323 +++++++++++++++++++++++++++++++++++++++++++ digest_report.html | 121 ++++++++++++++++ experiment_runner.py | 181 ++++++++++++++++++++++++ log_run.py | 161 +++++++++++++++++++++ 5 files changed, 931 insertions(+) create mode 100644 DIGEST_SYSTEM.md create mode 100644 digest.py create mode 100644 digest_report.html create mode 100644 experiment_runner.py create mode 100644 log_run.py diff --git a/DIGEST_SYSTEM.md b/DIGEST_SYSTEM.md new file mode 100644 index 000000000..de6f29e20 --- /dev/null +++ b/DIGEST_SYSTEM.md @@ -0,0 +1,145 @@ +# Autoresearch Daily Digest System + +This document describes the daily research digest and leaderboard system for autoresearch experiments. + +## Overview + +The digest system provides: +- **πŸ“Š Run leaderboard** by complexity-adjusted score (val_bpb + parameter penalty) +- **πŸ“ˆ Trend charts** showing validation BPB over time +- **πŸ“‹ Ready-for-review summary** templates for quick status updates + +## Components + +### 1. `log_run.py` - Results Logger +Captures experiment results and appends to `results.tsv`. + +```bash +# After running training +uv run train.py 2>&1 | tee last_run.log +python3 log_run.py last_run.log + +# Or log manually +python3 log_run.py --val_bpb 2.345 --num_params_M 12.5 --status success +``` + +### 2. `digest.py` - Report Generator +Generates HTML digest reports from results data. + +```bash +# Generate today's digest +python3 digest.py + +# Generate for specific date +python3 digest.py --date 2026-03-15 --output digest_2026-03-15.html +``` + +### 3. `experiment_runner.py` - Automation +High-level interface for running experiments and generating digests. + +```bash +# Run single experiment +python3 experiment_runner.py run + +# Generate digest +python3 experiment_runner.py digest + +# Run autonomous research loop +python3 experiment_runner.py loop --max-runs 10 --pause 10 +``` + +## Results Schema + +The `results.tsv` file contains columns: +- `timestamp`: ISO format timestamp +- `commit_hash`: Git commit (short hash) +- `val_bpb`: Validation bits per byte (main metric) +- `num_params_M`: Number of parameters in millions +- `training_seconds`: Training time +- `total_seconds`: Total runtime +- `peak_vram_mb`: Peak VRAM usage +- `mfu_percent`: Model FLOP utilization percentage +- `total_tokens_M`: Total tokens processed +- `num_steps`: Training steps completed +- `status`: success/crash/timeout + +## Leaderboard Scoring + +### Complexity-Adjusted Score +``` +score = val_bpb + (num_params_M / 1000) * 0.001 +``` +Lower scores are better. This penalizes larger models slightly to encourage efficient architectures. + +### Ranking Strategy +1. **Primary**: Complexity-adjusted score (lower is better) +2. **Filter**: Only successful runs (`status == "success"`) +3. **Tie-breaker**: Raw val_bpb + +## Daily Workflow + +### For Human Researchers +```bash +# Run experiment and log results +python3 experiment_runner.py run + +# Generate morning digest +python3 experiment_runner.py digest +``` + +### For Autonomous Agents +```bash +# Run overnight research loop +python3 experiment_runner.py loop --max-runs 20 --pause 15 +``` + +## HTML Report Features + +- **Summary metrics**: Total/successful/failed runs, best val_bpb +- **Ranked leaderboard**: Top 10 runs by complexity-adjusted score +- **Visual indicators**: Best run highlighting, status colors +- **Trend visualization**: val_bpb over time (requires matplotlib) +- **Ready-for-review template**: Structured summary for quick reporting + +## Integration with Autonomous Research + +The digest system fits into the autoresearch autonomous loop: + +1. **Agent modifies** `train.py` +2. **System runs** training (`uv run train.py`) +3. **Logger captures** results (`log_run.py`) +4. **Policy engine decides** keep/discard (`policy_engine.py`) +5. **Digest generates** progress report (`digest.py`) + +This creates an audit trail of all experiments with rich analytics for tracking research progress. + +## Dependencies + +- **Core**: Python 3.10+, no additional deps +- **Charts**: matplotlib (optional, for trend plots) +- **Training**: Same as autoresearch (PyTorch, etc.) + +```bash +# For chart generation +pip install matplotlib +``` + +## File Organization + +``` +autoresearch/ +β”œβ”€β”€ digest.py # Report generator +β”œβ”€β”€ log_run.py # Results logger +β”œβ”€β”€ experiment_runner.py # High-level automation +β”œβ”€β”€ results.tsv # Results database (gitignored) +└── digest_*.html # Generated reports +``` + +## Example Usage in Cron Jobs + +```bash +# Run autonomous research and generate morning digest +0 6 * * * cd /path/to/autoresearch && python3 experiment_runner.py loop --max-runs 5 && python3 experiment_runner.py digest +``` + +The digest system enables both human oversight and autonomous research tracking at scale. \ No newline at end of file diff --git a/digest.py b/digest.py new file mode 100644 index 000000000..425a0dae3 --- /dev/null +++ b/digest.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +""" +Daily Research Digest Generator for Autoresearch + +Generates daily digest with run leaderboard, trend plots, and summary template. +Reads from results.tsv and produces HTML report. + +Usage: + python digest.py [--date YYYY-MM-DD] [--output report.html] +""" + +import argparse +import csv +import json +import sys +from datetime import datetime, timedelta +from pathlib import Path +from typing import List, Dict, Any +from dataclasses import dataclass + +# Optional matplotlib for charts +try: + import matplotlib.pyplot as plt + import matplotlib.dates as mdates + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + print("⚠️ matplotlib not available - charts will be skipped") + +@dataclass +class ExperimentRun: + """Single experiment run data""" + timestamp: datetime + commit_hash: str + val_bpb: float + complexity: int # num_params_M * 1000 for integer scoring + training_seconds: float + total_seconds: float + peak_vram_mb: float + mfu_percent: float + total_tokens_M: float + num_steps: int + status: str = "success" + + @property + def complexity_adjusted_score(self) -> float: + """Lower is better: val_bpb + complexity penalty""" + complexity_penalty = (self.complexity / 1000) * 0.001 # Small penalty per million params + return self.val_bpb + complexity_penalty + + @property + def efficiency_score(self) -> float: + """Tokens per second per million parameters""" + return (self.total_tokens_M * 1e6) / self.total_seconds / (self.complexity / 1000) + +def parse_results_file(results_path: Path) -> List[ExperimentRun]: + """Parse results.tsv into ExperimentRun objects""" + runs = [] + + if not results_path.exists(): + print(f"⚠️ No results file found at {results_path}") + return runs + + with open(results_path, 'r') as f: + reader = csv.DictReader(f, delimiter='\t') + for row in reader: + try: + run = ExperimentRun( + timestamp=datetime.fromisoformat(row['timestamp']), + commit_hash=row['commit_hash'], + val_bpb=float(row['val_bpb']), + complexity=int(float(row['num_params_M']) * 1000), # Convert to int + training_seconds=float(row['training_seconds']), + total_seconds=float(row['total_seconds']), + peak_vram_mb=float(row['peak_vram_mb']), + mfu_percent=float(row['mfu_percent']), + total_tokens_M=float(row['total_tokens_M']), + num_steps=int(row['num_steps']), + status=row.get('status', 'success') + ) + runs.append(run) + except (KeyError, ValueError) as e: + print(f"⚠️ Skipping malformed row: {e}") + continue + + return runs + +def generate_leaderboard(runs: List[ExperimentRun], top_n: int = 10) -> List[ExperimentRun]: + """Generate top runs by complexity-adjusted score""" + successful_runs = [r for r in runs if r.status == "success"] + return sorted(successful_runs, key=lambda r: r.complexity_adjusted_score)[:top_n] + +def generate_trend_chart(runs: List[ExperimentRun], output_path: Path): + """Generate trend chart showing val_bpb over time""" + if not HAS_MATPLOTLIB: + print("⚠️ Skipping chart generation (matplotlib not available)") + return + + if not runs: + return + + successful_runs = [r for r in runs if r.status == "success"] + if not successful_runs: + return + + timestamps = [r.timestamp for r in successful_runs] + val_bpbs = [r.val_bpb for r in successful_runs] + + plt.figure(figsize=(12, 6)) + plt.plot(timestamps, val_bpbs, 'o-', alpha=0.7, linewidth=2, markersize=4) + plt.title('Validation BPB Trend Over Time', fontsize=14, fontweight='bold') + plt.xlabel('Time') + plt.ylabel('Validation Bits Per Byte (lower is better)') + plt.grid(True, alpha=0.3) + + # Format x-axis + plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%H:%M')) + plt.gca().xaxis.set_major_locator(mdates.HourLocator(interval=2)) + plt.xticks(rotation=45) + + # Add best run annotation + best_run = min(successful_runs, key=lambda r: r.val_bpb) + plt.annotate(f'Best: {best_run.val_bpb:.6f}', + xy=(best_run.timestamp, best_run.val_bpb), + xytext=(10, 10), textcoords='offset points', + bbox=dict(boxstyle='round,pad=0.3', fc='yellow', alpha=0.7), + arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0')) + + plt.tight_layout() + plt.savefig(output_path, dpi=150, bbox_inches='tight') + plt.close() + +def generate_html_report(runs: List[ExperimentRun], leaderboard: List[ExperimentRun], + report_date: str, output_path: Path): + """Generate HTML report with digest and leaderboard""" + + total_runs = len(runs) + successful_runs = len([r for r in runs if r.status == "success"]) + failed_runs = total_runs - successful_runs + + best_run = leaderboard[0] if leaderboard else None + + html_content = f""" + + + + + + Autoresearch Daily Digest - {report_date} + + + +

🧠 Autoresearch Daily Digest

+

Report Date: {report_date}

+ +
+

πŸ“Š Summary Statistics

+
+
{total_runs}
+
Total Runs
+
+
+
{successful_runs}
+
Successful
+
+
+
{failed_runs}
+
Failed
+
+
+
{'N/A' if not best_run else f'{best_run.val_bpb:.6f}'}
+
Best val_bpb
+
+
+ +

πŸ† Leaderboard (Top 10 by Complexity-Adjusted Score)

+ + + + + + + + + + + + + + +""" + + for i, run in enumerate(leaderboard, 1): + row_class = "best" if i == 1 else "" + html_content += f""" + + + + + + + + + + + """ + + html_content += """ + +
RankTimestampval_bpbComplexity ScoreAdj. ScoreMFU%CommitStatus
#{i}{run.timestamp.strftime('%H:%M:%S')}{run.val_bpb:.6f}{run.complexity:,}{run.complexity_adjusted_score:.6f}{run.mfu_percent:.1f}%{run.commit_hash[:8]}βœ“ {run.status}
""" + + # Only include chart section if we have matplotlib and successful runs + if HAS_MATPLOTLIB and any(r.status == "success" for r in runs): + html_content += """ + +

πŸ“ˆ Validation BPB Trend

+
+ Validation BPB Trend Over Time +
""" + else: + html_content += """ + +

πŸ“ˆ Validation BPB Trend

+

Chart generation requires matplotlib (install with: pip install matplotlib)

""" + + html_content += """ + +

πŸ“‹ Ready-for-Review Summary Template

+
+

Research Progress Summary

+
    +""" + + if best_run: + improvement_info = "First successful run" if len(leaderboard) == 1 else "Improvement achieved" + html_content += f""" +
  • Best Result: val_bpb = {best_run.val_bpb:.6f} (complexity: {best_run.complexity:,})
    • +
  • Status: {improvement_info}
    • +
  • Efficiency: {best_run.mfu_percent:.1f}% MFU, {best_run.efficiency_score:.0f} tokens/sec/M-param
    • + """ + else: + html_content += """ +
  • Status: No successful runs completed
    • +
  • Action Required: Debug training setup and retry
    • + """ + + html_content += f""" +
  • Run Statistics: {successful_runs}/{total_runs} successful runs
    • +
  • Next Steps: {'Continue autonomous research' if best_run else 'Fix training issues before continuing'}
    • +
+
+ +

Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

+ + +""" + + with open(output_path, 'w') as f: + f.write(html_content) + +def main(): + parser = argparse.ArgumentParser(description='Generate daily autoresearch digest') + parser.add_argument('--date', default=datetime.now().strftime('%Y-%m-%d'), + help='Report date (YYYY-MM-DD)') + parser.add_argument('--output', default='digest_report.html', + help='Output HTML file path') + parser.add_argument('--results-file', default='results.tsv', + help='Path to results.tsv file') + + args = parser.parse_args() + + results_path = Path(args.results_file) + output_path = Path(args.output) + chart_path = output_path.parent / 'trend_chart.png' + + print(f"πŸ“Š Generating autoresearch digest for {args.date}...") + + # Parse results + runs = parse_results_file(results_path) + + # Filter runs for the specified date + target_date = datetime.strptime(args.date, '%Y-%m-%d').date() + daily_runs = [r for r in runs if r.timestamp.date() == target_date] + + if not daily_runs: + print(f"⚠️ No runs found for {args.date}") + # Still generate a report for the template + + print(f"πŸ“ˆ Found {len(daily_runs)} runs for {args.date}") + + # Generate leaderboard + leaderboard = generate_leaderboard(daily_runs) + + # Generate trend chart + generate_trend_chart(daily_runs, chart_path) + + # Generate HTML report + generate_html_report(daily_runs, leaderboard, args.date, output_path) + + print(f"βœ… Report generated: {output_path}") + if chart_path.exists(): + print(f"πŸ“Š Chart generated: {chart_path}") + + return len(daily_runs) + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/digest_report.html b/digest_report.html new file mode 100644 index 000000000..59f513613 --- /dev/null +++ b/digest_report.html @@ -0,0 +1,121 @@ + + + + + + + Autoresearch Daily Digest - 2026-03-15 + + + +

🧠 Autoresearch Daily Digest

+

Report Date: 2026-03-15

+ +
+

πŸ“Š Summary Statistics

+
+
4
+
Total Runs
+
+
+
3
+
Successful
+
+
+
1
+
Failed
+
+
+
2.156789
+
Best val_bpb
+
+
+ +

πŸ† Leaderboard (Top 10 by Complexity-Adjusted Score)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
RankTimestampval_bpbComplexity ScoreAdj. ScoreMFU%CommitStatus
#106:08:022.15678918,5002.17528948.8%d0b52fbaβœ“ success
#206:07:592.19842115,2002.21362152.1%d0b52fbaβœ“ success
#306:07:562.23456712,8002.24736745.2%d0b52fbaβœ“ success
+ +

πŸ“ˆ Validation BPB Trend

+

Chart generation requires matplotlib (install with: pip install matplotlib)

+ +

πŸ“‹ Ready-for-Review Summary Template

+
+

Research Progress Summary

+
    + +
  • Best Result: val_bpb = 2.156789 (complexity: 18,500)
    • +
  • Status: Improvement achieved
    • +
  • Efficiency: 48.8% MFU, 17437 tokens/sec/M-param
    • + +
  • Run Statistics: 3/4 successful runs
    • +
  • Next Steps: Continue autonomous research
    • +
+
+ +

Generated on 2026-03-15 06:08:56

+ + diff --git a/experiment_runner.py b/experiment_runner.py new file mode 100644 index 000000000..68c47fb66 --- /dev/null +++ b/experiment_runner.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +""" +Autoresearch Experiment Runner and Digest Generator + +This script provides a high-level interface to: +1. Run training experiments and log results automatically +2. Generate daily research digests +3. Set up experiment loops for autonomous research + +Usage: + # Run a single experiment + python experiment_runner.py run + + # Generate today's digest + python experiment_runner.py digest + + # Run experiments in a loop (for autonomous research) + python experiment_runner.py loop --max-runs 10 +""" + +import argparse +import subprocess +import sys +import time +from datetime import datetime +from pathlib import Path + +def run_experiment(timeout_minutes: int = 10) -> bool: + """Run a single training experiment and log results""" + print(f"πŸš€ Starting training experiment at {datetime.now().strftime('%H:%M:%S')}") + + log_file = Path("last_run.log") + + try: + # Run training with timeout and capture output + result = subprocess.run( + ["uv", "run", "train.py"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=timeout_minutes * 60, + cwd=Path.cwd() + ) + + # Save output to log file + with open(log_file, 'w') as f: + f.write(result.stdout) + + print(f"πŸ“ Training output saved to {log_file}") + + # Log results to TSV + if result.returncode == 0: + subprocess.run(["python3", "log_run.py", str(log_file)], check=True) + print("βœ… Experiment completed successfully") + return True + else: + print(f"❌ Training failed with return code {result.returncode}") + # Still try to log results in case there's useful info + try: + subprocess.run(["python3", "log_run.py", str(log_file)], check=False) + except Exception as e: + print(f"⚠️ Could not log failed run: {e}") + return False + + except subprocess.TimeoutExpired: + print(f"⏰ Experiment timed out after {timeout_minutes} minutes") + # Log as timeout + subprocess.run([ + "python3", "log_run.py", + "--val_bpb", "999.999", + "--num_params_M", "0", + "--status", "timeout" + ], check=False) + return False + + except Exception as e: + print(f"πŸ’₯ Experiment crashed: {e}") + # Log as crash + subprocess.run([ + "python3", "log_run.py", + "--val_bpb", "999.999", + "--num_params_M", "0", + "--status", "crash" + ], check=False) + return False + +def generate_digest(date: str = None) -> bool: + """Generate research digest for specified date (default: today)""" + if date is None: + date = datetime.now().strftime('%Y-%m-%d') + + print(f"πŸ“Š Generating research digest for {date}") + + try: + result = subprocess.run([ + "python3", "digest.py", + "--date", date, + "--output", f"digest_{date}.html" + ], check=True) + + print(f"βœ… Digest generated: digest_{date}.html") + return True + + except subprocess.CalledProcessError as e: + print(f"❌ Digest generation failed: {e}") + return False + +def run_experiment_loop(max_runs: int, pause_minutes: int = 10): + """Run multiple experiments with pauses between them""" + print(f"πŸ”„ Starting autonomous research loop") + print(f" Max runs: {max_runs}") + print(f" Pause between runs: {pause_minutes} minutes") + + successful_runs = 0 + + for i in range(max_runs): + print(f"\n--- Run {i+1}/{max_runs} ---") + + success = run_experiment() + if success: + successful_runs += 1 + + # Generate digest every 5 runs or at the end + if (i + 1) % 5 == 0 or i == max_runs - 1: + print(f"\nπŸ“Š Generating interim digest...") + generate_digest() + + # Pause between runs (except last one) + if i < max_runs - 1: + print(f"⏸️ Pausing {pause_minutes} minutes before next run...") + time.sleep(pause_minutes * 60) + + print(f"\n🏁 Experiment loop completed!") + print(f" Total runs: {max_runs}") + print(f" Successful: {successful_runs}") + print(f" Failed: {max_runs - successful_runs}") + + # Generate final digest + print(f"\nπŸ“Š Generating final digest...") + generate_digest() + +def main(): + parser = argparse.ArgumentParser(description='Autoresearch experiment runner') + subparsers = parser.add_subparsers(dest='command', help='Available commands') + + # Run command + run_parser = subparsers.add_parser('run', help='Run a single experiment') + run_parser.add_argument('--timeout', type=int, default=10, + help='Timeout in minutes (default: 10)') + + # Digest command + digest_parser = subparsers.add_parser('digest', help='Generate research digest') + digest_parser.add_argument('--date', help='Date for digest (YYYY-MM-DD, default: today)') + + # Loop command + loop_parser = subparsers.add_parser('loop', help='Run experiment loop') + loop_parser.add_argument('--max-runs', type=int, default=10, + help='Maximum number of runs (default: 10)') + loop_parser.add_argument('--pause', type=int, default=10, + help='Minutes to pause between runs (default: 10)') + + args = parser.parse_args() + + if args.command == 'run': + success = run_experiment(args.timeout) + return 0 if success else 1 + + elif args.command == 'digest': + success = generate_digest(args.date) + return 0 if success else 1 + + elif args.command == 'loop': + run_experiment_loop(args.max_runs, args.pause) + return 0 + + else: + parser.print_help() + return 1 + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/log_run.py b/log_run.py new file mode 100644 index 000000000..f113f90b9 --- /dev/null +++ b/log_run.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +""" +Log experiment run results to results.tsv + +This script parses the output from train.py and appends the results +to a TSV file for later analysis by digest.py. + +Usage: + # Run training and capture output + uv run train.py 2>&1 | tee last_run.log + + # Log the results + python log_run.py last_run.log + + # Or log with explicit values + python log_run.py --val_bpb 2.345 --num_params_M 12.5 --status success +""" + +import argparse +import csv +import re +import subprocess +from datetime import datetime +from pathlib import Path + +def parse_train_output(log_content: str) -> dict: + """Parse train.py output to extract metrics""" + results = {} + + # Parse the final summary section + patterns = { + 'val_bpb': r'val_bpb:\s+([\d.]+)', + 'training_seconds': r'training_seconds:\s+([\d.]+)', + 'total_seconds': r'total_seconds:\s+([\d.]+)', + 'peak_vram_mb': r'peak_vram_mb:\s+([\d.]+)', + 'mfu_percent': r'mfu_percent:\s+([\d.]+)', + 'total_tokens_M': r'total_tokens_M:\s+([\d.]+)', + 'num_steps': r'num_steps:\s+(\d+)', + 'num_params_M': r'num_params_M:\s+([\d.]+)', + } + + for key, pattern in patterns.items(): + match = re.search(pattern, log_content) + if match: + results[key] = match.group(1) + else: + print(f"⚠️ Could not find {key} in output") + results[key] = "0" + + # Check for errors/crashes + if "error" in log_content.lower() or "exception" in log_content.lower(): + results['status'] = 'crash' + elif "KeyboardInterrupt" in log_content: + results['status'] = 'timeout' + else: + results['status'] = 'success' + + return results + +def get_git_hash() -> str: + """Get current git commit hash""" + try: + result = subprocess.run(['git', 'rev-parse', 'HEAD'], + capture_output=True, text=True, check=True) + return result.stdout.strip()[:8] # Short hash + except subprocess.CalledProcessError: + return 'unknown' + +def log_results(results: dict, results_file: str = 'results.tsv'): + """Append results to TSV file""" + results_path = Path(results_file) + + # Add timestamp and commit hash + results['timestamp'] = datetime.now().isoformat() + results['commit_hash'] = get_git_hash() + + # Define column order + columns = [ + 'timestamp', 'commit_hash', 'val_bpb', 'num_params_M', 'training_seconds', + 'total_seconds', 'peak_vram_mb', 'mfu_percent', 'total_tokens_M', + 'num_steps', 'status' + ] + + # Create file with header if it doesn't exist + if not results_path.exists(): + with open(results_path, 'w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=columns, delimiter='\t') + writer.writeheader() + print(f"πŸ“ Created new results file: {results_path}") + + # Append the results + with open(results_path, 'a', newline='') as f: + writer = csv.DictWriter(f, fieldnames=columns, delimiter='\t') + writer.writerow(results) + + print(f"βœ… Logged run to {results_path}") + print(f" val_bpb: {results['val_bpb']}") + print(f" status: {results['status']}") + print(f" params: {results['num_params_M']}M") + +def main(): + parser = argparse.ArgumentParser(description='Log experiment run results') + + # Option 1: Parse from log file + parser.add_argument('log_file', nargs='?', help='Log file from train.py output') + + # Option 2: Manual values + parser.add_argument('--val_bpb', type=float, help='Validation BPB') + parser.add_argument('--num_params_M', type=float, help='Number of parameters (millions)') + parser.add_argument('--training_seconds', type=float, default=300.0, help='Training time') + parser.add_argument('--total_seconds', type=float, default=310.0, help='Total time') + parser.add_argument('--peak_vram_mb', type=float, default=8000.0, help='Peak VRAM MB') + parser.add_argument('--mfu_percent', type=float, default=50.0, help='MFU percentage') + parser.add_argument('--total_tokens_M', type=float, default=100.0, help='Total tokens (millions)') + parser.add_argument('--num_steps', type=int, default=1000, help='Number of steps') + parser.add_argument('--status', choices=['success', 'crash', 'timeout'], + default='success', help='Run status') + + parser.add_argument('--results-file', default='results.tsv', + help='Results file to append to') + + args = parser.parse_args() + + if args.log_file: + # Parse from log file + log_path = Path(args.log_file) + if not log_path.exists(): + print(f"❌ Log file not found: {log_path}") + return 1 + + with open(log_path, 'r') as f: + log_content = f.read() + + results = parse_train_output(log_content) + print(f"πŸ“– Parsed results from {log_path}") + + elif args.val_bpb and args.num_params_M: + # Use manual values + results = { + 'val_bpb': str(args.val_bpb), + 'num_params_M': str(args.num_params_M), + 'training_seconds': str(args.training_seconds), + 'total_seconds': str(args.total_seconds), + 'peak_vram_mb': str(args.peak_vram_mb), + 'mfu_percent': str(args.mfu_percent), + 'total_tokens_M': str(args.total_tokens_M), + 'num_steps': str(args.num_steps), + 'status': args.status, + } + print("πŸ“ Using manual values") + + else: + print("❌ Must provide either log_file or --val_bpb and --num_params_M") + return 1 + + log_results(results, args.results_file) + return 0 + +if __name__ == "__main__": + import sys + sys.exit(main()) \ No newline at end of file