From 0e100b95cb527eea74c68894b59ffbc9646a5f80 Mon Sep 17 00:00:00 2001 From: napinoco Date: Sun, 20 Jul 2025 22:40:04 +0900 Subject: [PATCH 01/28] [Refactor] Streamline documentation to eliminate redundancy and improve information architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Transform CLAUDE.md from detailed design document to focused dispatch hub - Remove redundant design philosophy, system capabilities, and extension points sections - Add clear navigation to dedicated design documents with enhanced references - Dramatically streamline README.md from 327 to 118 lines with research focus - Streamline basic_design.md from 254 to 192 lines removing enterprise language - Remove obsolete design files (current_design.md, current_design_ja.md) - Add GitHub Pages link (https://napinoco.github.io/optimization-solver-benchmark/) to README.md - Update development status messaging for proper research tool context - Fix detail_design.md status value case consistency (UPPERCASE vs lowercase) - Remove unused problem_types arrays from MATLAB solver configurations - Strengthen design document references throughout CLAUDE.md 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- CLAUDE.md | 100 +-- README.md | 343 ++------- current_design.md | 384 ----------- current_design_ja.md | 384 ----------- docs/development/basic_design.md | 265 +++---- docs/development/detail_design.md | 1072 +++++++++++------------------ 6 files changed, 613 insertions(+), 1935 deletions(-) delete mode 100644 current_design.md delete mode 100644 current_design_ja.md diff --git a/CLAUDE.md b/CLAUDE.md index ed59759..483cab0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,11 +19,14 @@ This is an automated benchmark system for optimization solvers (LP, QP, SOCP, SD ## 🔗 Essential Documentation Links +> **⚠️ DISPATCH HUB NOTICE:** +> **CLAUDE.md serves as a navigation hub. All detailed design content is in dedicated documents below.** + ### **REQUIRED READING** (Read these documents carefully before any development): -1. **[📋 Basic Design](docs/development/basic_design.md)** - High-level concepts, project vision, and development roadmap -2. **[🏗️ Detailed Design](docs/development/detail_design.md)** - Complete technical architecture and implementation specifications @docs/development/detail_design.md -3. **[📝 Development Conventions](docs/development/conventions.md)** - Coding standards, git protocols, and engineering guidelines @docs/development/conventions.md +1. **[📋 Basic Design](docs/development/basic_design.md)** - Project vision, design philosophy, and system overview +2. **[🏗️ Detailed Design](docs/development/detail_design.md)** - Complete technical architecture, implementation specifications, and code examples +3. **[📝 Development Conventions](docs/development/conventions.md)** - Coding standards, git protocols, and engineering guidelines 4. **[✅ Current Tasks](docs/development/tasks.md)** - Active development tasks for current phase 5. **[📚 Development History](docs/development/history.md)** - Complete record of completed development phases @@ -31,15 +34,17 @@ This is an automated benchmark system for optimization solvers (LP, QP, SOCP, SD - **[🚀 Setup Guides](docs/guides/)** - Installation and configuration guides - **[📊 README.md](README.md)** - Project overview and quick start +> **📖 For system architecture, solver configurations, and implementation details, always refer to the dedicated design documents above rather than this file.** + --- ## 🎯 Current Development Status -**Phase**: Production Ready ✅ | All Sprints Completed Successfully -**Achievement**: Meaningful Public Reporting System Complete -**Status**: 139 problems (DIMACS + SDPLIB + Internal) with 5 major solvers +**Phase**: Research Tool Complete ✅ +**Achievement**: Comprehensive Benchmark System with External Libraries +**Status**: 139 problems (DIMACS + SDPLIB) with 11 solvers (9 Python + 2 MATLAB) -**Key Achievement**: Complete transformation from basic benchmark tool to production-ready public reporting platform with external problem libraries, comprehensive solver version tracking, and professional-grade HTML reports suitable for research publication. +**Key Achievement**: Comprehensive optimization solver benchmarking platform with external problem libraries, complete version tracking, and research-grade HTML reports with data export capabilities. --- @@ -56,34 +61,28 @@ This is an automated benchmark system for optimization solvers (LP, QP, SOCP, SD ├── config/ # YAML configuration files ├── scripts/ # Core system implementation │ ├── benchmark/ # Benchmark execution engine -│ ├── solvers/ # Solver implementations (Python) -│ ├── external/ # External library loaders (DIMACS, SDPLIB) +│ ├── solvers/ # Solver implementations +│ │ ├── python/ # Python solver interfaces +│ │ └── matlab_octave/ # MATLAB/Octave interfaces +│ ├── data_loaders/ # Problem format loaders +│ │ ├── python/ # Python loaders (MAT/DAT) +│ │ └── matlab_octave/ # MATLAB loaders │ ├── utils/ # Problem structure analysis, version detection │ ├── database/ # Data models and storage │ └── reporting/ # HTML generation and data publishing ├── problems/ # Benchmark problem files │ ├── DIMACS/ # External DIMACS library (47 problems) │ └── SDPLIB/ # External SDPLIB library (92 problems) +├── database/ # SQLite database files +│ └── results.db # Benchmark results storage ├── docs/ # Generated reports (GitHub Pages) └── requirements.txt # Python dependencies (single file) ``` -### Current System Capabilities -``` -Problem Type | Total Results | Success Rate | Solver Coverage -LP | 12 results | 100% | SciPy + CLARABEL + SCS + ECOS + OSQP -QP | 6 results | 100% | SciPy + CLARABEL + SCS + ECOS + OSQP -SOCP | 31 results | ~43% | CLARABEL + SCS + ECOS -SDP | 38 results | ~29% | CLARABEL + SCS -``` +### Current Capabilities +**System Status**: Research Tool Complete with **11 solvers** (9 Python + 2 MATLAB) across **139 problems** (DIMACS + SDPLIB) -### External Problem Libraries -``` -Library | Problems | Format | Source -DIMACS | 47 | SeDuMi .mat | Optimization challenges -SDPLIB | 92 | SDPA .dat-s | Semidefinite programming -Total | 139 | Mixed | Complete coverage -``` +**For detailed solver coverage and problem statistics, see [basic_design.md](docs/development/basic_design.md).** --- @@ -111,6 +110,9 @@ python main.py --report # Install dependencies pip install -r requirements.txt + +# Setup MATLAB solvers (optional) +matlab -batch "setup_matlab_solvers" # or octave --eval "setup_matlab_solvers" ``` --- @@ -134,17 +136,9 @@ pip install -r requirements.txt ## 🎯 Design Philosophy -### Fair Baseline Benchmarking -- **Minimal Configuration**: Use solver defaults to avoid optimization bias -- **Transparent Comparison**: Prevent parameter tuning favoring specific solvers -- **Open Data**: Publish results as accessible JSON/CSV for research use -- **Reproducible Results**: Standardized environments ensure consistency +**For complete design principles and technical philosophy, see [basic_design.md](docs/development/basic_design.md).** -### Technical Principles -- **Modular Design**: Independent solver/problem additions -- **Configuration-Driven**: YAML-based management -- **Error Resilience**: Continue despite individual failures -- **Automated Operation**: GitHub Actions enables hands-off execution +Core principles: Fair baseline benchmarking with minimal configuration, reproducible results, and modular architecture for research applications. --- @@ -184,31 +178,23 @@ pip install -r requirements.txt ## 🛠️ Extension Points -### Adding New Solvers -1. Implement `SolverInterface` abstract base class -2. Add configuration to `config/solvers.yaml` -3. Add dependencies to appropriate requirements file -4. Test with validation framework +**For complete implementation guidance on adding new solvers and problems, see [detail_design.md](docs/development/detail_design.md).** -### Adding New Problems -1. **Problem Extension**: Extend existing DIMACS/SDPLIB collections or add new external libraries -2. **External Libraries**: Add git submodule or extend existing DIMACS/SDPLIB -3. Update `problems/problem_registry.yaml` with problem metadata -4. Implement loader in `scripts/external/` if new format required -5. Validate with `python main.py --validate` +Quick reference: Python solvers via `python_interface.py`, MATLAB solvers via `{solver}_runner.m`, new problems via `problem_registry.yaml`. --- ## 📋 For LLM Agents: Pre-Development Checklist +> **🚨 CRITICAL: CLAUDE.md is a dispatch hub only. All design content is in dedicated documents.** + **BEFORE starting any coding task, confirm you have:** -- [ ] Read [basic_design.md](docs/development/basic_design.md) for project vision and goals -- [ ] Read [detail_design.md](docs/development/detail_design.md) for technical architecture -- [ ] Read [conventions.md](docs/development/conventions.md) for coding standards and protocols +- [ ] Read [@docs/development/basic_design.md](docs/development/basic_design.md) for project vision, design philosophy, and system overview +- [ ] Read [@docs/development/detail_design.md](docs/development/detail_design.md) for complete technical architecture and implementation details +- [ ] Read [@docs/development/conventions.md](docs/development/conventions.md) for coding standards and protocols - [ ] Reviewed [tasks.md](docs/development/tasks.md) for current development context -- [ ] Understood the fair benchmarking philosophy and minimal configuration approach -- [ ] Familiarized yourself with the modular architecture and extension points +- [ ] Understood that all design decisions and technical details are documented in the linked design files, not in CLAUDE.md **🚨 CRITICAL CONSTRAINTS CHECKLIST (MUST VERIFY):** @@ -222,18 +208,6 @@ pip install -r requirements.txt --- -## 🤝 Integration Context - -This system prioritizes: -- **Fair, unbiased solver comparison** through minimal configuration and real-world problems -- **Meaningful public reporting** suitable for research publication and external evaluation -- **External library integration** with DIMACS and SDPLIB optimization problem sets -- **Comprehensive metadata tracking** including solver versions and Git commit recording -- **Professional-grade reporting** with problem structure analysis and library attribution -- **Open data publishing** with complete JSON/CSV exports for research community use -- **Production reliability** with comprehensive testing across 139 problems - ---- ## 📝 Development Memories @@ -242,4 +216,4 @@ This system prioritizes: *This dispatch document provides entry point context only. All implementation details, coding standards, and development protocols are documented in the linked files above.* -*Last Updated: June 2025 - Production Ready Implementation Complete* \ No newline at end of file +*Last Updated: July 2025* \ No newline at end of file diff --git a/README.md b/README.md index 44c9326..abe2c11 100644 --- a/README.md +++ b/README.md @@ -1,327 +1,118 @@ # Optimization Solver Benchmark System -A comprehensive open-source platform for benchmarking optimization solvers across multiple problem types (LP, QP, SOCP, SDP). Automated execution via GitHub Actions with interactive HTML reports published to GitHub Pages. +A research tool for benchmarking optimization solvers across LP, QP, SOCP, and SDP problems using external problem libraries (DIMACS, SDPLIB) with automated execution and HTML report generation. -**Project Vision**: Create the leading open-source platform for benchmarking optimization solvers across multiple problem types, providing researchers and practitioners with comprehensive performance insights through automated execution, transparent reporting, and fair baseline comparisons. +Benchmark results are continuously published at: https://napinoco.github.io/optimization-solver-benchmark/ -**Core Mission**: "Regularly benchmark publicly available solvers and publish the results as data" +> **⚠️ Development Status**: This system is currently under active development. Some solvers and problem classes have not yet been properly measured, and comprehensive benchmarking coverage across all problem types is still being completed. -## External Problem Libraries +## Overview -This benchmark system uses the following external problem libraries: +**Purpose**: Compare optimization solver performance using standardized problem sets and minimal solver configuration for unbiased evaluation. -### DIMACS Library -- **Repository**: [https://github.com/vsdp/DIMACS.git](https://github.com/vsdp/DIMACS.git) -- **Current Commit**: `a1a14d1` - FILTER/filtinf1.mat.gz: Fix problem vector 'c'. -- **Problems**: 47 optimization challenge problems in SeDuMi .mat format +**Problem Libraries**: +- **DIMACS**: 47 problems in SeDuMi .mat format +- **SDPLIB**: 92+ problems in SDPA .dat-s format +- **Total**: 139+ optimization problems -### SDPLIB Library -- **Repository**: [https://github.com/vsdp/SDPLIB.git](https://github.com/vsdp/SDPLIB.git) -- **Current Commit**: `2e568e1` - README.md: Remove dead link. -- **Problems**: 92+ semidefinite programming problems in SDPA .dat-s format +**Supported Solvers**: +- **Python (9)**: SciPy, CVXPY backends (CLARABEL, SCS, ECOS, OSQP, CVXOPT, SDPA, SCIP, HIGHS) +- **MATLAB/Octave (2)**: SeDuMi, SDPT3 -Both libraries are included as fixed snapshots (committed on June 25, 2025) for reproducibility and automatically loaded by the benchmark system. This ensures consistent benchmarking results over time. - -## 🎯 Current Status: Production Ready - -- ✅ **Complete External Library Integration**: 139+ problems (DIMACS + SDPLIB) -- ✅ **Multi-Language Solver Support**: 11 solvers (9 Python + 2 MATLAB solvers) -- ✅ **Python Solvers**: SciPy, CLARABEL, SCS, ECOS, OSQP, CVXOPT, SDPA, SCIP, HiGHS via CVXPY -- ✅ **MATLAB/Octave Solvers**: SeDuMi, SDPT3 with unified interface architecture -- ✅ **Unified ProblemData Architecture**: SeDuMi-like format for all problem types -- ✅ **Professional Reporting**: Interactive HTML dashboards with problem structure analysis -- ✅ **GitHub Actions CI/CD**: Automated benchmarking and report publishing - -## 🚀 Quick Start - -### Prerequisites -- Python 3.12+ -- Git and GitHub account -- Virtual environment recommended +## Quick Start ### Installation ```bash git clone --recursive https://github.com/napinoco/optimization-solver-benchmark.git cd optimization-solver-benchmark python -m venv venv -source venv/bin/activate # On Windows: venv\Scripts\activate +source venv/bin/activate # Windows: venv\Scripts\activate pip install -r requirements.txt -# For MATLAB/Octave solver support (optional) -# Install Octave (MATLAB-compatible environment) +# For MATLAB/Octave solvers (optional) # Ubuntu/Debian: sudo apt-get install octave # macOS: brew install octave -# Windows: Download from https://octave.org/download + +# Setup MATLAB solvers - compiles MEX files for SeDuMi and SDPT3 +matlab -batch "setup_matlab_solvers" # or use octave --eval "setup_matlab_solvers" ``` -### Run Benchmarks +### Usage ```bash -# Validate environment and external libraries +# Validate environment python main.py --validate -# Complete benchmark with all libraries +# Run all benchmarks python main.py --all -# Benchmark specific libraries +# Specific libraries python main.py --benchmark --problem-set dimacs python main.py --benchmark --problem-set sdplib -# Generate reports only +# Generate reports only python main.py --report -# Dry-run mode for testing -python main.py --benchmark --problems nb --dry-run - -# Test MATLAB/Octave solvers specifically -python main.py --benchmark --problems nb --solvers matlab_sedumi,matlab_sdpt3 +# Test mode (no database changes) +python main.py --benchmark --dry-run ``` -## 🎯 Design Philosophy: Fair Baseline Benchmarking - -This system prioritizes **unbiased solver comparison** through minimal configuration: - -- **Solver Defaults**: Uses each solver's default parameters to avoid optimization bias -- **Baseline Performance**: Establishes genuine "out of the box" capabilities -- **Transparent Comparison**: Prevents inadvertent parameter tuning favoring specific solvers -- **Reproducible Results**: Standardized environments ensure consistent benchmarking -- **Open Data**: Results published as accessible JSON/CSV for research use - -## 📊 Current System Capabilities - -### Supported Solvers ✅ -#### Python Solvers (9 total) -- **SciPy**: Linear programming solver -- **CVXPY Multi-Backend**: - - CLARABEL (modern Rust-based solver) - - SCS (Splitting Conic Solver) - - ECOS (Embedded Conic Solver) - - OSQP (Operator Splitting QP) - - CVXOPT (Convex optimization) - - SDPA (Semidefinite programming) - - SCIP (Mixed-integer programming) - - HiGHS (Linear/mixed-integer programming) - -#### MATLAB/Octave Solvers (2 total) -- **SeDuMi**: Interior-point method for conic optimization -- **SDPT3**: Semidefinite-quadratic-linear programming solver - -### Problem Type Coverage ✅ -- **LP**: Linear Programming (12+ results, 100% success rate) -- **QP**: Quadratic Programming (6+ results, 100% success rate) -- **SOCP**: Second-Order Cone Programming (31+ results, ~43% success rate) -- **SDP**: Semidefinite Programming (38+ results, ~29% success rate) - -### Multi-Language Architecture ✅ -- **Unified Interface Design**: Symmetrical Python/MATLAB solver interfaces -- **Seamless Integration**: Python-MATLAB bridge via JSON data exchange -- **Fair Comparison**: Standardized metrics calculation across all solvers -- **Dynamic Solver Selection**: Runtime solver selection via configuration - -### Key Features ✅ -- **External Library Integration**: DIMACS (47 problems) + SDPLIB (92+ problems) -- **Unified Problem Format**: SeDuMi-like architecture for all problem types -- **Comprehensive Version Tracking**: Solver backend versions and Git commit recording -- **Interactive Reports**: Bootstrap 5 dashboards with Chart.js visualizations -- **Problem Structure Analysis**: Automatic variable/constraint counting and classification -- **GitHub Pages Deployment**: Automatic report publishing with data exports -- **Validation Framework**: Data quality checks and error detection -- **Dry-Run Mode**: Testing without database pollution - -## 📁 Project Structure +## Design Philosophy -``` -optimization-solver-benchmark/ -├── 📋 Documentation -│ ├── README.md # This file -│ ├── CLAUDE.md # AI development context -│ └── docs/ -│ ├── guides/ # Setup and usage guides -│ └── development/ # Development documentation -│ -├── ⚙️ Configuration -│ └── config/ -│ ├── site_config.yaml # Site and overview settings -│ ├── solver_registry.yaml # Solver display names -│ └── problem_registry.yaml # Problem metadata -│ -├── 🧪 Core System -│ ├── main.py # Main entry point -│ └── scripts/ -│ ├── benchmark/ # Benchmark execution engine -│ ├── solvers/ -│ │ ├── python/ # Python solver implementations -│ │ └── matlab_octave/ # MATLAB/Octave solver integration -│ ├── data_loaders/ # Problem loading (MAT/DAT) -│ │ ├── problem_loader.py # Unified ProblemData class -│ │ ├── python/ # Python format loaders -│ │ └── matlab_octave/ # MATLAB format loaders -│ ├── external/ # External library loaders -│ ├── utils/ # Problem structure analysis -│ ├── database/ # SQLite data models -│ └── reporting/ # HTML report generation -│ -├── 📊 Problems & Results -│ ├── problems/ -│ │ ├── DIMACS/ # External DIMACS library (47 problems) -│ │ ├── SDPLIB/ # External SDPLIB library (92+ problems) -│ ├── database/ # SQLite results with version tracking -│ └── docs/ # Generated reports and data exports -│ -├── 🔧 Dependencies -│ └── requirements.txt -│ -└── 🚀 CI/CD - └── .github/ - └── workflows/ # GitHub Actions and documentation -``` +**Fair Baseline Benchmarking**: Uses solver default parameters to avoid optimization bias and provide genuine "out of the box" performance comparison. -## 📈 Performance Results +**Reproducible Results**: Problem libraries are pinned to specific commit hashes via git submodules, and solver versions are fixed in requirements.txt to ensure identical benchmarking environments across all executions. -### Current Success Rates -``` -Problem Type | Total Results | Success Rate | Top Performers -LP | 12 results | 100% | SciPy + CLARABEL + SCS + ECOS + OSQP -QP | 6 results | 100% | SciPy + CLARABEL + SCS + ECOS + OSQP -SOCP | 31 results | ~43% | CLARABEL + SCS + ECOS + SeDuMi + SDPT3 -SDP | 38 results | ~29% | CLARABEL + SCS + SeDuMi + SDPT3 -``` +**Complete Version Tracking**: All benchmark results are stored in a SQLite database with complete version information (solver versions, Git commit hashes, environment details) to enable historical analysis and ensure full reproducibility of any reported result. + +## Project Structure -### External Library Performance ``` -Library | Problems | Format | Success Rate | Notes -DIMACS | 47 | SeDuMi .mat | Variable | Mixed problem types -SDPLIB | 92+ | SDPA .dat-s | ~30% | Challenging SDP problems -Total | 139+ | Mixed | ~37% | System average +├── main.py # Entry point +├── config/ # YAML configuration files +├── scripts/ +│ ├── benchmark/ # Execution engine +│ ├── solvers/ # Python/MATLAB solver interfaces +│ ├── data_loaders/ # Problem format loaders (MAT/DAT) +│ ├── database/ # SQLite storage +│ └── reporting/ # HTML generation +├── problems/ +│ ├── DIMACS/ # External library (git submodule) +│ └── SDPLIB/ # External library (git submodule) +├── database/ # SQLite database files +│ └── results.db # Benchmark results storage +└── docs/ # Generated reports and documentation ``` -## 🔧 GitHub Actions Integration - -### Automatic Execution -- **Push to main**: Triggers comprehensive benchmarking -- **Pull requests**: Validation and testing -- **Manual triggers**: Custom solver/problem selection - -### Manual Trigger Options -Access via GitHub Actions tab → "Run workflow": -- **Problem Set**: Choose from external, dimacs, or sdplib -- **Solvers**: Select specific solvers to benchmark -- **Library Names**: Filter by library (DIMACS, SDPLIB) -- **Dry Run**: Test without database updates +## Architecture -## 🛠️ Configuration +**Modular Design**: +- **Loaders**: Convert problem files → standardized ProblemData +- **Runners**: Execute solvers on ProblemData → SolverResult +- **Interfaces**: Coordinate problem loading and solver execution -### Core Configuration Files -- **`config/site_config.yaml`**: Site metadata and overview content -- **`config/solver_registry.yaml`**: Solver display names for reports -- **`config/problem_registry.yaml`**: Problem metadata and classification +**Multi-Language Support**: Unified Python/MATLAB integration via JSON data exchange and standardized result format. -### Main Entry Point Options -```bash -# Validation -python main.py --validate # Verify environment and libraries +## Adding Components -# Benchmarking -python main.py --all # Complete benchmark -python main.py --benchmark --problem-set external # External libraries only -python main.py --benchmark --library_names DIMACS # DIMACS problems only -python main.py --benchmark --problems nb,arch0 # Specific problems +### New Solvers +**Python**: Add configuration to `PYTHON_SOLVER_CONFIGS` in `python_interface.py` +**MATLAB**: Create `{solver}_runner.m` and add to `MATLAB_SOLVER_CONFIGS` in `matlab_interface.py` -# Reporting -python main.py --report # Generate HTML reports +### New Problems +Add external libraries as git submodules in `problems/` and extend loaders in `scripts/data_loaders/` -# Testing -python main.py --benchmark --dry-run # Test without database changes -``` +## Documentation -## 🧪 Development - -### Architecture Highlights -- **Unified ProblemData**: SeDuMi-like format with first-class cone structure support -- **Multi-Language Support**: Symmetrical Python/MATLAB solver interfaces with unified result format -- **Modular Component Design**: - - **Loaders**: Convert problem_name → ProblemData (format-specific parsing) - - **Runners**: Convert ProblemData → SolverResult (solver-specific execution) - - **Interfaces**: Orchestrate problem_name → SolverResult (loader + runner coordination) -- **CVXPY Integration**: Unified solver interface supporting multiple backends -- **Version Tracking**: Complete solver backend and Git commit recording -- **Error Resilience**: Continues benchmarking despite individual solver failures - -### Adding New Solvers -#### Python Solvers -1. Extend the CVXPY runner with new backend -2. Add solver configuration to `config/solver_registry.yaml` -3. Test with validation framework -4. Update documentation - -#### MATLAB/Octave Solvers -1. Create solver-specific runner: `{solver}_runner.m` (ProblemData → SolverResult) -2. Add solver configuration to MATLAB_SOLVER_CONFIGS in `matlab_interface.py` -3. Test with validation framework -4. Update documentation - -### Adding New Problems -1. **External Libraries**: Add as Git submodules in `problems/` -2. **Format Support**: Extend loaders in `scripts/data_loaders/python/` -3. **Validation**: Test with `python main.py --validate` - -## 📊 System Achievements - -### Technical Accomplishments ✅ -- **Unified Architecture**: Single ProblemData format for all optimization types -- **Multi-Language Integration**: 11 solvers (9 Python + 2 MATLAB) with symmetrical interfaces -- **External Library Integration**: Professional-grade problem sets (DIMACS + SDPLIB) -- **Production-Ready Reporting**: Interactive dashboards suitable for research publication -- **Fair Baseline Benchmarking**: Minimal configuration for unbiased comparison - -### Real-World Impact -- **Research-Ready Data**: Complete JSON/CSV exports for external analysis -- **Reproducible Results**: Version tracking and standardized environments -- **Open Science**: Public reporting with comprehensive problem attribution -- **Community Resource**: Accessible benchmarking platform for optimization research - -## 📚 Documentation - -### User Guides -- **[CLAUDE.md](CLAUDE.md)**: AI development context and project instructions -- **[docs/guides/](docs/guides/)**: Setup, usage, and integration guides - -### Development Documentation -- **[docs/development/basic_design.md](docs/development/basic_design.md)**: Project vision and architecture -- **[docs/development/detail_design.md](docs/development/detail_design.md)**: Technical implementation details -- **[docs/development/conventions.md](docs/development/conventions.md)**: Development standards and protocols -- **[docs/development/tasks.md](docs/development/tasks.md)**: Current development tasks -- **[docs/development/history.md](docs/development/history.md)**: Complete development history - -## 🏆 Production Ready Status - -### System Completeness ✅ -- **139+ External Problems**: Real-world optimization challenges -- **11 Major Solvers**: Multi-language coverage (9 Python + 2 MATLAB) across optimization types -- **Professional Reporting**: Publication-ready HTML dashboards -- **CI/CD Integration**: Automated benchmarking and deployment -- **Fair Benchmarking Philosophy**: Unbiased solver comparison using defaults - -### Quality Assurance -- **Validation Framework**: Data quality checks and error detection -- **Version Tracking**: Complete reproducibility with Git commit recording -- **Testing Infrastructure**: Dry-run mode and comprehensive validation -- **Documentation**: Complete development history and usage guides - -## 📄 License - -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. - -## 🙏 Acknowledgments +- **[CLAUDE.md](CLAUDE.md)**: Development context and instructions +- **[docs/development/](docs/development/)**: Technical design and implementation details -- **External Problem Libraries**: VSDP team for DIMACS and SDPLIB repositories -- **Open-Source Solvers**: SciPy, CVXPY, CLARABEL, SCS, ECOS, OSQP, CVXOPT, SDPA, SCIP, HiGHS, SeDuMi, SDPT3 -- **Web Technologies**: Bootstrap 5, Chart.js for interactive visualizations -- **CI/CD Platform**: GitHub Actions for automated benchmarking and deployment +## License ---- +MIT License - see [LICENSE](LICENSE) file. -**Status**: Production Ready ✅ | Fair Baseline Benchmarking System Complete ✅ -**Last Updated**: June 2025 -**Achievement**: Comprehensive optimization solver benchmarking platform with unified architecture +## Acknowledgments -For questions, issues, or contributions, please visit our [GitHub repository](https://github.com/napinoco/optimization-solver-benchmark) or check the documentation links above. \ No newline at end of file +- **External Problem Libraries**: VSDP team for DIMACS and SDPLIB repositories +- **MATLAB Solvers**: SQLP team for maintaining Git repositories of SeDuMi and SDPT3 solvers +- **Open-Source Solvers**: SciPy, CVXPY, CLARABEL, SCS, ECOS, OSQP, CVXOPT, SDPA, SCIP, HIGHS communities +- **Development Support**: Claude Code for coding assistance and development guidance \ No newline at end of file diff --git a/current_design.md b/current_design.md deleted file mode 100644 index 0fa7ebf..0000000 --- a/current_design.md +++ /dev/null @@ -1,384 +0,0 @@ -# MATLAB Integration Current Design Documentation - -> **Document Purpose**: This document explains the current functionalities and data-process-flow of MATLAB-related code components in the optimization solver benchmark system. - ---- - -## Table of Contents - -1. [Architecture Overview](#architecture-overview) -2. [Core Components](#core-components) -3. [Data Flow Pipeline](#data-flow-pipeline) -4. [Integration Mechanisms](#integration-mechanisms) -5. [Problem Resolution System](#problem-resolution-system) -6. [Result Processing Chain](#result-processing-chain) -7. [Error Handling Strategy](#error-handling-strategy) -8. [Performance Considerations](#performance-considerations) - ---- - -## Architecture Overview - -The MATLAB integration provides a complete bridge between MATLAB optimization solvers (SeDuMi, SDPT3) and the Python-based benchmark system. The architecture follows a **pipeline design** with clear separation of concerns: - -``` -Python System ↔ matlab_solver.py ↔ MATLAB Scripts ↔ Solver Backends -``` - -### Key Design Principles - -- **SolverInterface Compliance**: Full adherence to the Python `SolverInterface` specification -- **Problem Registry Integration**: Seamless resolution of DIMACS/SDPLIB problems via YAML configuration -- **Fault Tolerance**: Comprehensive error handling with meaningful fallbacks at every layer -- **Data Standardization**: Consistent JSON interchange format between Python and MATLAB -- **Temporary File Management**: Safe handling of intermediate files with automatic cleanup - ---- - -## Core Components - -### 1. Python Layer: `matlab_solver.py` - -**Purpose**: Primary Python interface implementing `SolverInterface` for MATLAB solvers. - -**Key Classes**: -- `MatlabSolver`: Base class with full SolverInterface compliance -- `SeDuMiSolver`: Convenience wrapper for SeDuMi -- `SDPT3Solver`: Convenience wrapper for SDPT3 - -**Core Functionality**: -- **Environment Verification**: Validates MATLAB/Octave availability and startup performance -- **Version Detection**: Dynamic detection and caching of MATLAB and solver versions -- **Problem Validation**: Registry-based problem compatibility checking -- **Subprocess Management**: Safe MATLAB execution with timeout handling -- **Temporary File Management**: UUID-based temporary files with automatic cleanup - -### 2. MATLAB Orchestrator: `matlab_runner.m` - -**Purpose**: Consolidated MATLAB entry point containing all utility functions and coordinating problem loading, solver execution, and result saving. - -**Execution Flow**: -1. **Configuration Loading**: Uses integrated YAML reader for problem metadata -2. **Problem Loading**: Dispatches to appropriate loader based on file type -3. **Solver Execution**: Calls specific solver runner (SeDuMi/SDPT3) -4. **Metrics Calculation**: Uses integrated metrics calculator for objective values and feasibility measures -5. **Result Serialization**: Uses integrated JSON formatter and file saver - -**Integrated Architecture**: All utility functions are embedded as nested functions within the main orchestrator, eliminating external dependencies and simplifying deployment to a single 1,035-line file. - -### 3. Solver Implementations - -#### `sedumi_runner.m` -- **Configuration**: Minimal solver options for fair benchmarking (`fid=0` for silent mode) -- **Validation**: Comprehensive input validation and cone structure normalization -- **Error Handling**: Graceful failure with structured error result creation -- **Version Detection**: Multi-stage SeDuMi version identification - -#### `sdpt3_runner.m` -- **Configuration**: Default SDPT3 parameters with verbose output disabled -- **Preprocessing**: Input validation and problem structure verification -- **Result Mapping**: Translation of SDPT3 status codes to standard format -- **Performance Monitoring**: Solve time measurement and iteration counting - -### 4. Integrated Utility Functions - -All utility functions have been consolidated into `matlab_runner.m` as nested functions for simplified deployment and reduced dependencies: - -#### Core Utilities (Integrated) -- **`calculate_solver_metrics()`**: Objective computation and feasibility measures -- **`read_problem_registry()`**: YAML parsing for problem configuration -- **`format_result_to_json()`**: MATLAB-to-JSON conversion with type safety -- **`save_json_safely()`**: Robust file I/O with error recovery -- **`save_solutions_if_needed()`**: Optional solution vector persistence -- **`detect_versions()`**: Comprehensive environment and solver version detection - -#### Supporting Functions (Integrated) -- **`calculate_dual_cone_violation()`**: Cone-specific dual infeasibility calculation -- **`proj_onto_soc()`**: Second-order cone projection -- **`convert_to_json_compatible()`**: Data type conversion for JSON serialization -- **`get_field_safe()`**: Safe struct field access with defaults -- **`detect_sedumi_version()`** / **`detect_sdpt3_version()`**: Solver-specific version detection - -**Benefits of Consolidation**: -- **Single File Deployment**: All MATLAB functionality in one file -- **Reduced Path Dependencies**: No need to manage multiple script paths -- **Atomic Operations**: All functionality accessible without external dependencies -- **Simplified Maintenance**: Single file to update and version control - ---- - -## Data Flow Pipeline - -### Phase 1: Initialization and Validation - -``` -Python Request → Problem Data Validation → Registry Lookup → Solver Compatibility Check -``` - -1. **Python Layer**: Validates `ProblemData` structure and required fields -2. **Registry Resolution**: Looks up problem in `problem_registry.yaml` to get file path and type -3. **Compatibility Check**: Verifies solver supports problem type (MAT/DAT-S files) -4. **Environment Check**: Confirms MATLAB availability and solver installation - -### Phase 2: MATLAB Execution - -``` -Temporary File Creation → MATLAB Command Construction → Subprocess Execution -``` - -1. **Temp File Management**: Creates UUID-based JSON result file -2. **Command Construction**: Builds safe MATLAB command with proper path handling: - ```matlab - addpath('matlab_script_dir'); matlab_runner('problem_name', 'solver_name', 'result_file') - ``` -3. **Subprocess Execution**: Runs MATLAB with timeout and captures stdout/stderr - -### Phase 3: Problem Resolution and Loading - -``` -Problem Registry → File Path Resolution → Format-Specific Loading -``` - -1. **Registry Parsing**: Integrated `read_problem_registry()` extracts problem metadata -2. **Path Resolution**: Converts relative paths to absolute paths -3. **Format Dispatch**: Routes to `mat_loader()` or `dat_loader()` based on file type -4. **Data Extraction**: Returns matrices `A`, `b`, `c` and cone structure `K` - -### Phase 4: Solver Execution - -``` -Input Validation → Solver Configuration → Problem Solving → Status Mapping -``` - -1. **Input Validation**: Checks matrix dimensions and cone structure consistency -2. **Solver Setup**: Applies minimal configuration for fair benchmarking -3. **Solve Process**: Executes optimization with timing measurement -4. **Result Processing**: Maps solver-specific status codes to standard format - -### Phase 5: Metrics Calculation - -``` -Solution Vectors → Objective Calculation → Feasibility Assessment → Gap Computation -``` - -1. **Objective Values** (via integrated `calculate_solver_metrics()`): - - Primal: `c' * x` - - Dual: `b' * y` -2. **Infeasibility Measures**: - - Primal: `||A*x - b|| / (1 + ||b||)` - - Dual: `sqrt(cone_violation(c - A'*y)) / (1 + ||c||^2)` -3. **Duality Gap**: `|primal_objective - dual_objective|` - -### Phase 6: Result Serialization and Return - -``` -MATLAB Result → JSON Conversion → File Writing → Python Reading → SolverResult Creation -``` - -1. **JSON Formatting**: Integrated `format_result_to_json()` converts MATLAB struct to JSON string -2. **File Writing**: Integrated `save_json_safely()` writes JSON with error recovery -3. **Python Reading**: `json.load()` parses result file -4. **Result Conversion**: Maps to `SolverResult` object with metadata enhancement - ---- - -## Integration Mechanisms - -### Problem Registry Integration - -The system uses a centralized YAML configuration to map problem names to file locations: - -```yaml -problem_libraries: - nb: - display_name: "Network Design" - file_path: "problems/DIMACS/data/ANTENNA/nb.mat.gz" - file_type: "mat" - library_name: "DIMACS" - - arch0: - display_name: "Architecture 0" - file_path: "problems/SDPLIB/data/arch0.dat-s" - file_type: "dat-s" - library_name: "SDPLIB" -``` - -**Resolution Process**: -1. Python `ProblemData.name` → Registry lookup -2. Extract `file_path` and `file_type` -3. Pass to MATLAB for format-specific loading -4. Load problem data using appropriate loader - -### Temporary File Management - -**UUID-Based Naming**: Each solver execution creates unique temporary files: -``` -/tmp/matlab_sedumi_result_.json -``` - -**Automatic Cleanup**: Context managers ensure cleanup even on exceptions: -```python -with temp_file_context(".json") as result_file: - # Execute MATLAB solver - # File automatically cleaned up -``` - -**Orphan Detection**: Periodic cleanup of abandoned temporary files older than 1 hour. - -### Version Detection and Caching - -**Multi-Stage Detection**: -1. **Function Existence**: Check if solver functions are available -2. **Version Functions**: Call solver-specific version functions -3. **File Parsing**: Parse version from installation files -4. **Functionality Test**: Verify solver works with minimal problem - -**Caching Strategy**: Version information cached during initialization to avoid repeated detection overhead. - ---- - -## Result Processing Chain - -### MATLAB Result Structure - -```matlab -result = struct(); -result.solver_name = 'SeDuMi'; -result.solver_version = 'SeDuMi-1.3.7'; -result.status = 'optimal'; -result.solve_time = 0.245; -result.primal_objective = -4.567; -result.dual_objective = -4.567; -result.gap = 1.234e-10; -result.primal_infeasibility = 2.345e-12; -result.dual_infeasibility = 3.456e-11; -result.iterations = 15; -``` - -### JSON Conversion Process - -**Type Safety**: Integrated `format_result_to_json()` handles MATLAB-specific types: -- `NaN` → `null` (empty array in JSON) -- `±Inf` → `±1e308` -- Empty arrays → `null` -- Ensure numeric precision preservation - -### Python SolverResult Mapping - -```python -SolverResult( - solve_time=matlab_result['solve_time'], - status=matlab_result['status'].upper(), - primal_objective_value=safe_float(matlab_result['primal_objective_value']), - dual_objective_value=safe_float(matlab_result['dual_objective_value']), - duality_gap=safe_float(matlab_result['duality_gap']), - primal_infeasibility=safe_float(matlab_result['primal_infeasibility']), - dual_infeasibility=safe_float(matlab_result['dual_infeasibility']), - iterations=safe_int(matlab_result['iterations']), - solver_name='matlab_sedumi', - solver_version='SeDuMi-1.3.7 (MATLAB R2023b)', - additional_info={ - 'matlab_output': matlab_result, - 'solver_backend': 'sedumi', - 'execution_environment': 'matlab' - } -) -``` - ---- - -## Error Handling Strategy - -### Multi-Layer Error Recovery - -#### 1. Python Layer -- **Subprocess Failures**: Capture stderr/stdout for diagnostic information -- **Timeout Handling**: Return structured timeout result -- **File I/O Errors**: Handle missing/corrupt result files -- **JSON Parsing**: Graceful handling of malformed JSON - -#### 2. MATLAB Layer -- **Solver Errors**: Create error result structure with diagnostic info -- **Problem Loading**: Handle missing files or format errors -- **Metrics Calculation**: Safe handling of NaN/Inf in calculations -- **File Writing**: Atomic operations with rollback on failure - -### Error Result Structure - -```python -SolverResult.create_error_result( - error_message="MATLAB execution failed: solver not found", - solve_time=actual_time_spent, - solver_name=self.solver_name, - solver_version=self.get_version() -) -``` - -### Diagnostic Information - -All error results include: -- **Error Classification**: Timeout, solver error, I/O error, etc. -- **Execution Context**: MATLAB version, solver availability, file paths -- **Timing Information**: Time spent before failure -- **Raw Output**: MATLAB stdout/stderr for debugging - ---- - -## Performance Considerations - -### Startup Optimization - -**MATLAB Initialization**: Cold MATLAB startup can take 5-15 seconds -- **Timeout Adjustment**: Add 15s buffer to solve timeout for startup -- **Startup Warning**: Alert when MATLAB startup > 10s -- **Pre-warming**: Consider pre-warmed MATLAB sessions for production - -### Memory Management - -**Temporary Files**: -- UUID-based naming prevents conflicts -- Automatic cleanup prevents disk space issues -- Orphan detection handles interrupted executions - -**MATLAB Memory**: -- Release large matrices after use -- Clear temporary variables in MATLAB workspace -- Monitor memory usage in long-running sessions - -### Execution Efficiency - -**Path Management**: -- Add MATLAB script paths once per session -- Use absolute paths to avoid working directory issues -- Cache problem registry to avoid repeated YAML parsing - -**Subprocess Optimization**: -- Single MATLAB command per solve (no multiple calls) -- Batch multiple problems when possible -- Minimize data serialization overhead - ---- - -## Integration Points Summary - -### With Python Benchmark System - -1. **SolverInterface Compliance**: Drop-in replacement for other solvers -2. **Problem Registry**: Seamless DIMACS/SDPLIB problem resolution -3. **Database Integration**: Standard SolverResult format for storage -4. **Report Generation**: Consistent metadata for reporting pipeline - -### With External Libraries - -1. **DIMACS Problems**: Native MAT file support via `mat_loader.m` -2. **SDPLIB Problems**: DAT-S file support via `dat_loader.m` -3. **Problem Metadata**: Registry-driven problem classification and metadata - -### With MATLAB Ecosystem - -1. **Solver Detection**: Dynamic discovery of installed solvers -2. **Version Tracking**: Comprehensive version metadata collection -3. **Error Propagation**: Meaningful error messages from MATLAB to Python -4. **Solution Storage**: Optional MAT file output for detailed analysis - -This design provides a robust, production-ready integration that maintains the fair benchmarking philosophy while delivering comprehensive MATLAB solver support within the existing Python architecture. \ No newline at end of file diff --git a/current_design_ja.md b/current_design_ja.md deleted file mode 100644 index 4540d8e..0000000 --- a/current_design_ja.md +++ /dev/null @@ -1,384 +0,0 @@ -# MATLAB統合 現行設計書 - -> **文書の目的**: この文書は、最適化ソルバーベンチマークシステムにおけるMATLAB関連コードコンポーネントの現在の機能とデータ処理フローを説明します。 - ---- - -## 目次 - -1. [アーキテクチャ概要](#アーキテクチャ概要) -2. [コアコンポーネント](#コアコンポーネント) -3. [データフローパイプライン](#データフローパイプライン) -4. [統合メカニズム](#統合メカニズム) -5. [問題解決システム](#問題解決システム) -6. [結果処理チェーン](#結果処理チェーン) -7. [エラーハンドリング戦略](#エラーハンドリング戦略) -8. [パフォーマンス考慮事項](#パフォーマンス考慮事項) - ---- - -## アーキテクチャ概要 - -MATLAB統合は、MATLABの最適化ソルバー（SeDuMi、SDPT3）とPythonベースのベンチマークシステム間の完全なブリッジを提供します。アーキテクチャは明確な関心の分離を持つ**パイプライン設計**に従っています： - -``` -Pythonシステム ↔ matlab_solver.py ↔ MATLABスクリプト ↔ ソルバーバックエンド -``` - -### 主要設計原則 - -- **SolverInterface準拠**: Python `SolverInterface`仕様への完全な準拠 -- **問題レジストリ統合**: YAML設定によるDIMACS/SDPLIB問題のシームレスな解決 -- **フォルトトレランス**: すべてのレイヤーで意味のあるフォールバックを持つ包括的なエラーハンドリング -- **データ標準化**: PythonとMATLAB間の一貫したJSON交換フォーマット -- **一時ファイル管理**: 自動クリーンアップによる中間ファイルの安全な処理 - ---- - -## コアコンポーネント - -### 1. Pythonレイヤー: `matlab_solver.py` - -**目的**: MATLABソルバー用の`SolverInterface`を実装する主要なPythonインターフェース。 - -**主要クラス**: -- `MatlabSolver`: 完全なSolverInterface準拠を持つベースクラス -- `SeDuMiSolver`: SeDuMi用の便利ラッパー -- `SDPT3Solver`: SDPT3用の便利ラッパー - -**コア機能**: -- **環境検証**: MATLAB/Octaveの可用性と起動パフォーマンスを検証 -- **バージョン検出**: MATLABとソルバーバージョンの動的検出とキャッシュ -- **問題検証**: レジストリベースの問題互換性チェック -- **サブプロセス管理**: タイムアウト処理を持つ安全なMATLAB実行 -- **一時ファイル管理**: 自動クリーンアップ付きのUUIDベース一時ファイル - -### 2. MATLABオーケストレーター: `matlab_runner.m` - -**目的**: すべてのユーティリティ関数を含む統合MATLABエントリーポイントで、問題読み込み、ソルバー実行、結果保存を調整。 - -**実行フロー**: -1. **設定読み込み**: 統合されたYAMLリーダーを使用して問題メタデータを取得 -2. **問題読み込み**: ファイルタイプに基づいて適切なローダーにディスパッチ -3. **ソルバー実行**: 特定のソルバーランナー（SeDuMi/SDPT3）を呼び出し -4. **メトリクス計算**: 統合されたメトリクス計算機を使用して目的値と実行可能性測定値を計算 -5. **結果シリアライゼーション**: 統合されたJSONフォーマッターとファイル保存機能を使用 - -**統合アーキテクチャ**: すべてのユーティリティ関数がメインオーケストレーター内にネストされた関数として埋め込まれ、外部依存関係を排除し、1,035行の単一ファイルへのデプロイメントを簡素化。 - -### 3. ソルバー実装 - -#### `sedumi_runner.m` -- **設定**: 公正なベンチマークのための最小限のソルバーオプション（サイレントモード用`fid=0`） -- **検証**: 包括的な入力検証と錐構造正規化 -- **エラーハンドリング**: 構造化されたエラー結果作成による優雅な失敗 -- **バージョン検出**: 多段階SeDuMiバージョン識別 - -#### `sdpt3_runner.m` -- **設定**: 詳細出力無効化によるデフォルトSDPT3パラメータ -- **前処理**: 入力検証と問題構造検証 -- **結果マッピング**: SDPT3ステータスコードの標準形式への変換 -- **パフォーマンス監視**: 解時間測定と反復回数カウント - -### 4. 統合ユーティリティ関数 - -すべてのユーティリティ関数は、デプロイメントの簡素化と依存関係の削減のため、`matlab_runner.m`内にネストされた関数として統合されました： - -#### コアユーティリティ（統合済み） -- **`calculate_solver_metrics()`**: 目的計算と実行可能性測定 -- **`read_problem_registry()`**: 問題設定用のYAML解析 -- **`format_result_to_json()`**: 型安全性を持つMATLAB-JSON変換 -- **`save_json_safely()`**: エラー回復を持つ堅牢なファイルI/O -- **`save_solutions_if_needed()`**: オプションの解ベクトル永続化 -- **`detect_versions()`**: 包括的な環境とソルバーバージョン検出 - -#### サポート関数（統合済み） -- **`calculate_dual_cone_violation()`**: 錐固有の双対非実行可能性計算 -- **`proj_onto_soc()`**: 二次錐射影 -- **`convert_to_json_compatible()`**: JSONシリアライゼーション用データ型変換 -- **`get_field_safe()`**: デフォルト付き安全な構造体フィールドアクセス -- **`detect_sedumi_version()`** / **`detect_sdpt3_version()`**: ソルバー固有バージョン検出 - -**統合の利点**: -- **単一ファイルデプロイメント**: すべてのMATLAB機能が一つのファイルに -- **パス依存関係の削減**: 複数のスクリプトパス管理が不要 -- **アトミック操作**: 外部依存関係なしでアクセス可能なすべての機能 -- **簡素化されたメンテナンス**: 更新とバージョン管理が単一ファイルで完結 - ---- - -## データフローパイプライン - -### フェーズ1: 初期化と検証 - -``` -Pythonリクエスト → 問題データ検証 → レジストリ検索 → ソルバー互換性チェック -``` - -1. **Pythonレイヤー**: `ProblemData`構造と必要フィールドを検証 -2. **レジストリ解決**: `problem_registry.yaml`で問題を検索してファイルパスとタイプを取得 -3. **互換性チェック**: ソルバーが問題タイプ（MAT/DAT-Sファイル）をサポートすることを検証 -4. **環境チェック**: MATLABの可用性とソルバーインストールを確認 - -### フェーズ2: MATLAB実行 - -``` -一時ファイル作成 → MATLABコマンド構築 → サブプロセス実行 -``` - -1. **一時ファイル管理**: UUIDベースのJSON結果ファイルを作成 -2. **コマンド構築**: 適切なパス処理で安全なMATLABコマンドを構築: - ```matlab - addpath('matlab_script_dir'); matlab_runner('problem_name', 'solver_name', 'result_file') - ``` -3. **サブプロセス実行**: タイムアウト付きでMATLABを実行し、stdout/stderrをキャプチャ - -### フェーズ3: 問題解決と読み込み - -``` -問題レジストリ → ファイルパス解決 → 形式固有読み込み -``` - -1. **レジストリ解析**: 統合された`read_problem_registry()`が問題メタデータを抽出 -2. **パス解決**: 相対パスを絶対パスに変換 -3. **形式ディスパッチ**: ファイルタイプに基づいて`mat_loader()`または`dat_loader()`にルーティング -4. **データ抽出**: 行列`A`、`b`、`c`と錐構造`K`を返す - -### フェーズ4: ソルバー実行 - -``` -入力検証 → ソルバー設定 → 問題解決 → ステータスマッピング -``` - -1. **入力検証**: 行列次元と錐構造の一貫性をチェック -2. **ソルバー設定**: 公正なベンチマークのための最小限の設定を適用 -3. **解決プロセス**: タイミング測定付きで最適化を実行 -4. **結果処理**: ソルバー固有のステータスコードを標準形式にマップ - -### フェーズ5: メトリクス計算 - -``` -解ベクトル → 目的計算 → 実行可能性評価 → ギャップ計算 -``` - -1. **目的値**（統合された`calculate_solver_metrics()`経由）: - - 主問題: `c' * x` - - 双対問題: `b' * y` -2. **非実行可能性測定**: - - 主問題: `||A*x - b|| / (1 + ||b||)` - - 双対問題: `sqrt(cone_violation(c - A'*y)) / (1 + ||c||^2)` -3. **双対性ギャップ**: `|primal_objective - dual_objective|` - -### フェーズ6: 結果シリアライゼーションと返却 - -``` -MATLAB結果 → JSON変換 → ファイル書き込み → Python読み取り → SolverResult作成 -``` - -1. **JSON形式化**: 統合された`format_result_to_json()`がMATLAB構造体をJSON文字列に変換 -2. **ファイル書き込み**: 統合された`save_json_safely()`がエラー回復付きでJSONを書き込み -3. **Python読み取り**: `json.load()`が結果ファイルを解析 -4. **結果変換**: メタデータ拡張付きで`SolverResult`オブジェクトにマップ - ---- - -## 統合メカニズム - -### 問題レジストリ統合 - -システムは問題名をファイルの場所にマップするために集中化されたYAML設定を使用： - -```yaml -problem_libraries: - nb: - display_name: "Network Design" - file_path: "problems/DIMACS/data/ANTENNA/nb.mat.gz" - file_type: "mat" - library_name: "DIMACS" - - arch0: - display_name: "Architecture 0" - file_path: "problems/SDPLIB/data/arch0.dat-s" - file_type: "dat-s" - library_name: "SDPLIB" -``` - -**解決プロセス**: -1. Python `ProblemData.name` → レジストリ検索 -2. `file_path`と`file_type`を抽出 -3. 形式固有読み込みのためにMATLABに渡す -4. 適切なローダーを使用して問題データを読み込み - -### 一時ファイル管理 - -**UUIDベースネーミング**: 各ソルバー実行は一意の一時ファイルを作成： -``` -/tmp/matlab_sedumi_result_.json -``` - -**自動クリーンアップ**: コンテキストマネージャーが例外時でもクリーンアップを保証： -```python -with temp_file_context(".json") as result_file: - # MATLABソルバーを実行 - # ファイルは自動的にクリーンアップされる -``` - -**孤児検出**: 1時間以上古い放置された一時ファイルの定期的なクリーンアップ。 - -### バージョン検出とキャッシュ - -**多段階検出**: -1. **関数存在**: ソルバー関数が利用可能かチェック -2. **バージョン関数**: ソルバー固有のバージョン関数を呼び出し -3. **ファイル解析**: インストールファイルからバージョンを解析 -4. **機能テスト**: 最小問題でソルバーが動作することを検証 - -**キャッシュ戦略**: 繰り返し検出オーバーヘッドを避けるために初期化時にバージョン情報をキャッシュ。 - ---- - -## 結果処理チェーン - -### MATLAB結果構造 - -```matlab -result = struct(); -result.solver_name = 'SeDuMi'; -result.solver_version = 'SeDuMi-1.3.7'; -result.status = 'optimal'; -result.solve_time = 0.245; -result.primal_objective = -4.567; -result.dual_objective = -4.567; -result.gap = 1.234e-10; -result.primal_infeasibility = 2.345e-12; -result.dual_infeasibility = 3.456e-11; -result.iterations = 15; -``` - -### JSON変換プロセス - -**型安全性**: 統合された`format_result_to_json()`がMATLAB固有型を処理： -- `NaN` → `null`（JSONで空配列） -- `±Inf` → `±1e308` -- 空配列 → `null` -- 数値精度保持を保証 - -### Python SolverResultマッピング - -```python -SolverResult( - solve_time=matlab_result['solve_time'], - status=matlab_result['status'].upper(), - primal_objective_value=safe_float(matlab_result['primal_objective_value']), - dual_objective_value=safe_float(matlab_result['dual_objective_value']), - duality_gap=safe_float(matlab_result['duality_gap']), - primal_infeasibility=safe_float(matlab_result['primal_infeasibility']), - dual_infeasibility=safe_float(matlab_result['dual_infeasibility']), - iterations=safe_int(matlab_result['iterations']), - solver_name='matlab_sedumi', - solver_version='SeDuMi-1.3.7 (MATLAB R2023b)', - additional_info={ - 'matlab_output': matlab_result, - 'solver_backend': 'sedumi', - 'execution_environment': 'matlab' - } -) -``` - ---- - -## エラーハンドリング戦略 - -### 多層エラー回復 - -#### 1. Pythonレイヤー -- **サブプロセス失敗**: 診断情報のためにstderr/stdoutをキャプチャ -- **タイムアウト処理**: 構造化されたタイムアウト結果を返す -- **ファイルI/Oエラー**: 欠損/破損結果ファイルを処理 -- **JSON解析**: 不正なJSONの優雅な処理 - -#### 2. MATLABレイヤー -- **ソルバーエラー**: 診断情報付きエラー結果構造を作成 -- **問題読み込み**: 欠損ファイルまたは形式エラーを処理 -- **メトリクス計算**: 計算でのNaN/Infの安全な処理 -- **ファイル書き込み**: 失敗時のロールバック付きアトミック操作 - -### エラー結果構造 - -```python -SolverResult.create_error_result( - error_message="MATLAB実行失敗: ソルバーが見つかりません", - solve_time=actual_time_spent, - solver_name=self.solver_name, - solver_version=self.get_version() -) -``` - -### 診断情報 - -すべてのエラー結果には以下が含まれます： -- **エラー分類**: タイムアウト、ソルバーエラー、I/Oエラーなど -- **実行コンテキスト**: MATLABバージョン、ソルバー可用性、ファイルパス -- **タイミング情報**: 失敗前に費やした時間 -- **生出力**: デバッグ用のMATLAB stdout/stderr - ---- - -## パフォーマンス考慮事項 - -### 起動最適化 - -**MATLAB初期化**: コールドMATLAB起動は5-15秒かかる可能性 -- **タイムアウト調整**: 起動用に解タイムアウトに15秒バッファを追加 -- **起動警告**: MATLAB起動が10秒以上の場合にアラート -- **事前ウォーミング**: 本番環境では事前ウォーミングされたMATLABセッションを検討 - -### メモリ管理 - -**一時ファイル**: -- UUIDベースネーミングが競合を防止 -- 自動クリーンアップがディスク容量問題を防止 -- 孤児検出が中断された実行を処理 - -**MATLABメモリ**: -- 使用後に大きな行列を解放 -- MATLABワークスペースの一時変数をクリア -- 長時間実行セッションでメモリ使用量を監視 - -### 実行効率 - -**パス管理**: -- セッションごとに一度MATLABスクリプトパスを追加 -- 作業ディレクトリ問題を避けるために絶対パスを使用 -- 繰り返しYAML解析を避けるために問題レジストリをキャッシュ - -**サブプロセス最適化**: -- 解決ごとに単一MATLABコマンド（複数呼び出しなし） -- 可能な場合は複数問題をバッチ処理 -- データシリアライゼーションオーバーヘッドを最小化 - ---- - -## 統合ポイント要約 - -### Pythonベンチマークシステムとの統合 - -1. **SolverInterface準拠**: 他のソルバーのドロップイン置換 -2. **問題レジストリ**: シームレスなDIMACS/SDPLIB問題解決 -3. **データベース統合**: ストレージ用の標準SolverResult形式 -4. **レポート生成**: レポートパイプライン用の一貫したメタデータ - -### 外部ライブラリとの統合 - -1. **DIMACS問題**: `mat_loader.m`によるネイティブMATファイルサポート -2. **SDPLIB問題**: `dat_loader.m`によるDAT-Sファイルサポート -3. **問題メタデータ**: レジストリ駆動の問題分類とメタデータ - -### MATLABエコシステムとの統合 - -1. **ソルバー検出**: インストールされたソルバーの動的発見 -2. **バージョン追跡**: 包括的なバージョンメタデータ収集 -3. **エラー伝播**: MATLABからPythonへの意味のあるエラーメッセージ -4. **解ストレージ**: 詳細分析用のオプションMATファイル出力 - -この設計は、既存のPythonアーキテクチャ内で包括的なMATLABソルバーサポートを提供しながら、公正なベンチマーク哲学を維持する堅牢で本番対応の統合を提供します。 \ No newline at end of file diff --git a/docs/development/basic_design.md b/docs/development/basic_design.md index 69aa843..ad36d11 100644 --- a/docs/development/basic_design.md +++ b/docs/development/basic_design.md @@ -1,39 +1,39 @@ # Optimization Solver Benchmark System - Basic Design ## Project Vision -Create the leading open-source platform for benchmarking optimization solvers across multiple problem types, providing researchers and practitioners with comprehensive performance insights through automated execution, transparent reporting, and fair baseline comparisons. + +Research tool for benchmarking optimization solvers across LP, QP, SOCP, and SDP problems using external problem libraries with automated execution and fair comparison methodology. --- ## Core Mission + **"Regularly benchmark publicly available solvers and publish the results as data"** -This system prioritizes unbiased solver comparison through minimal configuration, establishing "out of the box" performance baselines rather than optimized configurations. +This system prioritizes unbiased solver comparison through minimal configuration, establishing "out of the box" performance baselines for research use. --- ## Design Philosophy -### Fair Baseline Benchmarking Principles -- **Solver Defaults**: Uses each solver's default parameters to avoid optimization bias -- **Baseline Performance**: Establishes genuine "out of the box" capabilities -- **Transparent Comparison**: Prevents inadvertent parameter tuning favoring specific solvers -- **Reproducible Results**: Standardized environments ensure consistent benchmarking -- **Open Data**: Results published as accessible JSON/CSV for research use +### Fair Benchmarking Principles +- **Solver Defaults**: Use default parameters to avoid optimization bias +- **Reproducible Results**: Fixed problem libraries and solver versions +- **Open Data**: Results published as accessible JSON/CSV for research +- **Version Tracking**: Complete metadata for result verification ### Technical Principles -1. **Minimal Configuration**: Fair comparison using solver defaults for unbiased benchmarking -2. **Modular Design**: Independent addition of solvers, problems, and environments -3. **Configuration-Driven**: Manage benchmark content via YAML configuration -4. **Result Standardization**: Common data format across different solver environments -5. **Error Resilience**: System continues despite individual solver failures -6. **Automated Operation**: GitHub Actions enables hands-off execution and deployment +1. **Minimal Configuration**: Fair comparison using solver defaults +2. **Modular Design**: Independent addition of solvers and problems +3. **Result Standardization**: Common data format across solver environments +4. **Error Resilience**: System continues despite individual solver failures +5. **Automated Operation**: GitHub Actions enables hands-off execution --- ## System Overview -### Core Architecture (LOCAL DEVELOPMENT FIRST) +### Core Architecture ``` LOCAL DEVELOPMENT: ┌─────────────┐ ┌──────────────┐ ┌─────────────┐ @@ -52,12 +52,6 @@ LOCAL DEVELOPMENT: │ Report │ │ Generation │ └─────────────┘ - │ - ▼ - ┌─────────────┐ - │ Commit to │ - │ docs/ │ - └─────────────┘ GITHUB ACTIONS (Publishing Only): ┌─────────────┐ ┌──────────────┐ @@ -66,189 +60,120 @@ GITHUB ACTIONS (Publishing Only): └─────────────┘ └──────────────┘ ``` -### Target Environments ✅ -- **Python**: CVXPY multi-backend (CLARABEL, SCS, ECOS, OSQP), SciPy optimization suite -- **Version Tracking**: Complete solver backend version detection and Git commit recording -- **External Libraries**: DIMACS and SDPLIB integration via git clone approach +### Supported Components +**Problem Libraries**: +- **DIMACS**: 47 problems in SeDuMi .mat format +- **SDPLIB**: 92+ problems in SDPA .dat-s format +- **Total Coverage**: 139+ optimization problems -### Supported Problem Types ✅ -- **LP**: Linear Programming (DIMACS) -- **QP**: Quadratic Programming (DIMACS) -- **SOCP**: Second-Order Cone Programming (DIMACS) -- **SDP**: Semidefinite Programming (DIMACS + SDPLIB) -- **Total Coverage**: 139+ problems from external libraries only +**Solver Support**: +- **Python (9)**: SciPy, CVXPY backends (CLARABEL, SCS, ECOS, OSQP, CVXOPT, SDPA, SCIP, HIGHS) +- **MATLAB/Octave (2)**: SeDuMi, SDPT3 (optional) ---- - -## Infrastructure Constraints & Solutions - -### GitHub Actions Platform -- **Advantages**: Free for public repos, integrated CI/CD, standardized environments -- **Constraints**: 6-hour execution limit, 20 parallel jobs max -- **Solutions**: Staged problem sets, efficient parallel execution - -### Storage Strategy ✅ IMPLEMENTED -```yaml -✅ External Problem Libraries - COMPLETED - - DIMACS library: 47 problems via git submodule - - SDPLIB library: 92+ problems via git submodule - - CVXPY conversion for solver compatibility - -✅ Production Ready: External-Only Focus - - 139+ total problems from external libraries - - Professional reporting with structure analysis - - Comprehensive metadata and version tracking - - Simplified architecture with MAT/DAT loaders only -``` +**Problem Types**: +- **LP**: Linear Programming +- **QP**: Quadratic Programming +- **SOCP**: Second-Order Cone Programming +- **SDP**: Semidefinite Programming --- -## Development Roadmap +## Implementation Strategy -### Phase 1: MVP Foundation ✅ COMPLETED -**Goal**: Verify basic functionality with minimal viable system +### Development Approach +**External Library Focus**: Use established problem libraries (DIMACS, SDPLIB) rather than creating internal problems to ensure research relevance and standardization. -**Achievements**: -- Python solvers (SciPy, CVXPY default backends) -- GitHub Actions CI/CD with manual triggers -- Interactive HTML reports via GitHub Pages -- SQLite storage with structured schema -- Comprehensive validation framework +**Git Submodule Integration**: Pin problem libraries to specific commits for reproducibility while enabling updates when needed. -### Phase 2: Data Publishing Platform ✅ COMPLETED -**Goal**: Robust data publishing with expanded solver ecosystem +**Multi-Language Support**: Unified interface supporting both Python and MATLAB/Octave solvers with standardized result format. -**Achievements**: -- Multi-backend CVXPY (CLARABEL, SCS, ECOS, OSQP, CVXOPT, SDPA, SCIP, HiGHS) -- Complete SOCP/SDP problem support via external libraries -- Clean JSON/CSV data exports for research use -- External library integration (DIMACS + SDPLIB) -- Production-ready benchmark system +### Data Management +**Database Storage**: SQLite with complete version tracking (solver versions, Git commits, environment details) for historical analysis. -**Current Solver Coverage**: -``` -Problem Type | Solver Count | Backends -LP | 9 | SciPy + CLARABEL + SCS + ECOS + OSQP + CVXOPT + SDPA + SCIP + HiGHS -QP | 8 | SciPy + CLARABEL + SCS + ECOS + OSQP + CVXOPT + SCIP + HiGHS -SOCP | 6 | CLARABEL + SCS + ECOS + CVXOPT + SDPA + SCIP -SDP | 5 | CLARABEL + SCS + CVXOPT + SDPA + SCIP -``` +**Result Format**: Standardized SolverResult across all solvers with timing, objective values, status, and metadata. -### Phase 3: Architecture Optimization ✅ COMPLETED -**Goal**: Simplified architecture and enhanced testing capabilities - -**Achievements**: -- Direct registry iteration (removed helper functions) -- Clean separation of --library_names vs --problems filtering -- Dry-run mode for testing without database pollution -- Memo column for result annotations -- Streamlined codebase focused on external libraries only -- Removed internal synthetic problems for production clarity - -### Phase 4: Advanced Ecosystem ⏳ PLANNED -**Goal**: Enhanced analysis capabilities and broader solver support - -**Planned Features**: -- MATLAB Optimization Toolbox integration -- Commercial solver support (Gurobi, CPLEX) via user licenses -- Mixed-Integer Programming (MIP) and Nonlinear Programming (NLP) -- Advanced statistical analysis and performance profiling -- Community problem sharing and solver recommendations - -### Phase 5: Production Platform ⏳ PLANNED -**Goal**: Scalable cloud deployment with enterprise features - -**Planned Features**: -- Cloud-native deployment (AWS/GCP/Azure) -- Auto-scaling based on benchmark workload -- API with authentication and quotas -- Real-time performance monitoring -- Academic institution partnerships +**Report Generation**: Automated HTML dashboards with JSON/CSV data export for research use. --- -## Success Metrics +## Development Status -### Functional Metrics -- **Solver Coverage**: Target 10+ open-source solvers across 4 problem types -- **Execution Reliability**: 99%+ successful benchmark completion rate -- **Data Quality**: Comprehensive validation with error detection -- **Report Generation**: Automated HTML/JSON/CSV output +### Completed Features ✅ +- **Core Benchmarking**: Python and MATLAB solver execution +- **External Libraries**: DIMACS and SDPLIB integration via git submodules +- **Data Publishing**: Interactive HTML reports with CSV/JSON export +- **Version Tracking**: Complete environment and solver version recording +- **CI/CD Pipeline**: GitHub Actions automation with GitHub Pages deployment +- **Validation Framework**: Problem and solver compatibility testing -### Performance Metrics -- **Execution Speed**: <5 minutes for light problem sets -- **System Throughput**: Efficient parallel solver execution -- **Resource Utilization**: Optimal GitHub Actions usage +### Current Solver Coverage +``` +Problem Type | Solvers Available | Success Rate +LP | 9 solvers | ~100% +QP | 8 solvers | ~100% +SOCP | 6 solvers | ~40-60% +SDP | 5 solvers | ~20-40% +``` -### Adoption Metrics -- **Community Usage**: Active viewers and data consumers -- **Research Impact**: Citations and academic references -- **Contribution Rate**: Community-submitted problems and solvers +### Architecture Benefits +- **Fair Comparison**: Minimal configuration prevents solver bias +- **Research Ready**: Complete metadata for paper publication +- **Extensible**: Easy addition of new solvers and problem formats +- **Automated**: Hands-off operation via GitHub Actions --- -## Quality Assurance +## Usage Scenarios -### Testing Strategy -- **Unit Tests**: Individual component validation -- **Integration Tests**: Complete workflow testing -- **End-to-End Tests**: GitHub Actions simulation -- **Performance Regression**: Continuous performance monitoring +### Research Applications +- **Solver Performance Analysis**: Compare solvers across problem types +- **Algorithm Development**: Benchmark new optimization algorithms +- **Problem Difficulty Assessment**: Analyze which problems are challenging +- **Historical Trends**: Track solver performance improvements over time -### Validation Framework -- **Input Validation**: Problem file format verification -- **Result Validation**: Solver output consistency checks -- **Environment Validation**: System specification recording -- **Configuration Validation**: YAML syntax and semantic validation +### Academic Use Cases +- **Course Materials**: Demonstrate optimization solver capabilities +- **Student Projects**: Provide baseline results for comparison +- **Research Publication**: Reference standardized benchmark results +- **Collaboration**: Share reproducible performance data --- -## Security & Compliance - -### Data Protection -- **Public Data Only**: No sensitive information in benchmark problems -- **Environment Anonymization**: Optional system information privacy -- **Result Transparency**: Open publishing with privacy controls +## System Requirements -### License Compliance -- **Open-Source First**: Prioritize freely available solvers -- **Clear Documentation**: License information for all components -- **Commercial Integration**: Support user-provided commercial licenses +### Runtime Dependencies +- **Python 3.12+**: Core execution environment +- **Git**: Repository and submodule management +- **Optional MATLAB/Octave**: For MATLAB solver support ---- - -## Future Vision +### Problem Library Setup +```bash +# Clone with submodules for problem libraries +git clone --recursive -### Long-term Goals (2-3 years) -- **Leading Platform**: Recognized standard for optimization solver benchmarking -- **Research Community**: 1000+ active users, 500+ contributed problems -- **Academic Impact**: 100+ citations, integration with research workflows -- **Industry Adoption**: Enterprise use cases and professional references +# Validate environment +python main.py --validate -### Technology Evolution -- **Multi-language Support**: Python, Julia, MATLAB, C++ solver integration -- **Advanced Analytics**: Machine learning for solver performance prediction -- **Real-time Monitoring**: Live performance tracking and alerting -- **Cloud Deployment**: Scalable infrastructure with global availability +# Run benchmarks +python main.py --all +``` --- -## Project Values +## Quality Assurance -### Open Source Commitment -- **Transparent Development**: Public development process -- **Community-Driven**: User feedback shapes feature development -- **Open Data**: All results publicly available for research -- **Free Access**: Core functionality available without cost +### Validation Strategy +- **Environment Validation**: System compatibility checking +- **Problem Validation**: File format and data integrity verification +- **Solver Validation**: Execution capability testing +- **Result Validation**: Output consistency verification -### Scientific Rigor -- **Reproducible Results**: Consistent, repeatable benchmarking methodology -- **Fair Comparison**: Unbiased evaluation using solver defaults -- **Statistical Validity**: Proper performance analysis and reporting -- **Documentation**: Comprehensive methodology and implementation details +### Reproducibility Measures +- **Fixed Dependencies**: Pinned solver versions in requirements.txt +- **Consistent Environments**: Standardized execution via GitHub Actions +- **Complete Metadata**: Environment info saved with all results +- **Version Control**: Git commit tracking for all benchmark runs --- -*This basic design document establishes the high-level vision, principles, and roadmap for the optimization solver benchmark system. For detailed implementation specifications, see [detail_design.md](detail_design.md).* - -*Last Updated: December 2025* \ No newline at end of file +*This basic design establishes the conceptual framework and development approach. For implementation details, see [detail_design.md](detail_design.md).* \ No newline at end of file diff --git a/docs/development/detail_design.md b/docs/development/detail_design.md index 0f671cb..7d05aef 100644 --- a/docs/development/detail_design.md +++ b/docs/development/detail_design.md @@ -1,115 +1,208 @@ -# Optimization Solver Benchmark System - Technical Design Specification +# Optimization Solver Benchmark System - Technical Design -This document provides comprehensive technical specifications for the optimization solver benchmark system, including the complete MATLAB/Octave integration architecture. The system is designed for simplicity, reliability, and maintainability while supporting both Python and MATLAB solvers. +Detailed technical specifications for the optimization solver benchmark system supporting Python and MATLAB/Octave solvers across LP, QP, SOCP, and SDP problems. --- ## System Overview -### Core Mission -"Regularly benchmark publicly available solvers and publish the results as data" +**Purpose**: Benchmark optimization solvers using external problem libraries (DIMACS, SDPLIB) with minimal configuration for unbiased performance evaluation. -### Key Features -- **Fair Baseline Benchmarking**: Minimal configuration using solver defaults -- **Multi-Language Support**: Python (CVXPY, SciPy) and MATLAB/Octave (SeDuMi, SDPT3) solvers -- **External Problem Libraries**: DIMACS (47 problems) and SDPLIB (92 problems) -- **Production Ready**: 139+ problems with comprehensive solver coverage -- **Automated Publishing**: GitHub Actions with GitHub Pages deployment +**Core Components**: +- **Problem Loaders**: Parse MAT/DAT formats → standardized ProblemData +- **Solver Interfaces**: Execute Python/MATLAB solvers → standardized SolverResult +- **Database Storage**: SQLite with complete version tracking +- **Report Generation**: HTML dashboards with CSV/JSON export -### Current System Statistics -``` -Problem Type | Total Results | Success Rate | Solver Coverage -LP | 12 results | 100% | SciPy + CLARABEL + SCS + ECOS + OSQP -QP | 6 results | 100% | SciPy + CLARABEL + SCS + ECOS + OSQP -SOCP | 31 results | ~43% | CLARABEL + SCS + ECOS + SeDuMi + SDPT3 -SDP | 38 results | ~29% | CLARABEL + SCS + SeDuMi + SDPT3 - -External Libraries: -DIMACS | 47 problems | SeDuMi .mat format -SDPLIB | 92 problems | SDPA .dat-s format +**Supported Systems**: +- **Python Solvers (9)**: SciPy, CVXPY backends (CLARABEL, SCS, ECOS, OSQP, CVXOPT, SDPA, SCIP, HIGHS) +- **MATLAB/Octave Solvers (2)**: SeDuMi, SDPT3 +- **Problem Types**: LP, QP, SOCP, SDP (139 problems total) + +--- + +## Project Structure + +### Directory Layout +``` +optimization-solver-benchmark/ +├── main.py # Entry point with argument parsing +├── config/ # Configuration files +│ ├── site_config.yaml # Site metadata and overview +│ └── problem_registry.yaml # Problem metadata and file paths +├── scripts/ +│ ├── benchmark/ # Benchmark execution engine +│ │ ├── __init__.py +│ │ └── runner.py # Main BenchmarkRunner class +│ ├── solvers/ # Solver interface implementations +│ │ ├── __init__.py +│ │ ├── solver_interface.py # Abstract base classes +│ │ ├── python/ # Python solver implementations +│ │ │ ├── __init__.py +│ │ │ ├── python_interface.py # Python solver coordinator +│ │ │ ├── cvxpy_runner.py # CVXPY backend handler +│ │ │ └── scipy_runner.py # SciPy linprog handler +│ │ └── matlab_octave/ # MATLAB/Octave integration +│ │ ├── __init__.py +│ │ ├── matlab_interface.py # Python-MATLAB bridge +│ │ ├── matlab_interface.m # MATLAB entry point +│ │ ├── sedumi_runner.m # SeDuMi solver wrapper +│ │ ├── sdpt3_runner.m # SDPT3 solver wrapper +│ │ ├── setup_matlab_solvers.m # MEX compilation script +│ │ ├── sedumi/ # SeDuMi solver (git submodule) +│ │ └── sdpt3/ # SDPT3 solver (git submodule) +│ ├── data_loaders/ # Problem format loaders +│ │ ├── __init__.py +│ │ ├── problem_loader.py # ProblemData class definition +│ │ ├── python/ # Python format loaders +│ │ │ ├── __init__.py +│ │ │ ├── problem_interface.py # Problem loading coordinator +│ │ │ ├── mat_loader.py # DIMACS .mat loader +│ │ │ └── dat_loader.py # SDPLIB .dat-s loader +│ │ └── matlab_octave/ # MATLAB format loaders +│ │ ├── mat_loader.m # MATLAB .mat loader +│ │ └── dat_loader.m # MATLAB .dat-s loader +│ ├── database/ # Database management +│ │ ├── __init__.py +│ │ ├── database_manager.py # Database operations +│ │ ├── models.py # Data models +│ │ └── schema.sql # Database schema +│ ├── reporting/ # Report generation +│ │ ├── __init__.py +│ │ ├── html_generator.py # HTML report creation +│ │ ├── result_processor.py # Result aggregation +│ │ └── data_exporter.py # JSON/CSV export +│ └── utils/ # Utility modules +│ ├── __init__.py +│ ├── environment_info.py # System information capture +│ ├── git_utils.py # Git operations +│ ├── logger.py # Logging configuration +│ └── temp_file_manager.py # Temporary file handling +├── problems/ # Problem library files +│ ├── DIMACS/ # External DIMACS library (git submodule) +│ └── SDPLIB/ # External SDPLIB library (git submodule) +├── database/ # SQLite database files +│ └── results.db # Benchmark results storage +├── docs/ # Generated reports and documentation +│ ├── pages/ # Generated HTML reports +│ │ ├── index.html # Main dashboard +│ │ ├── results_matrix.html # Problems × Solvers matrix +│ │ ├── raw_data.html # Detailed data view +│ │ └── data/ # Exported data files +│ │ ├── benchmark_results.json +│ │ ├── benchmark_results.csv +│ │ └── summary.json +│ └── development/ # Development documentation +└── requirements.txt # Python dependencies ``` --- -## System Architecture +## Component Architecture -### Unified Data Flow Architecture +### System Data Flow ``` -LOCAL DEVELOPMENT: -┌─────────────┐ ┌──────────────┐ ┌─────────────┐ -│ Problem │───▶│ Solver │───▶│ Result │ -│ Interface │ │ Interfaces │ │ Collection │ -│ (Unified) │ │ (Py + MATLAB)│ │ (Standard) │ -└─────────────┘ └──────────────┘ └─────────────┘ - │ │ │ - ▼ ▼ ▼ -┌─────────────┐ ┌──────────────┐ ┌─────────────┐ -│ External │ │ Environment │ │ Database │ -│ Libraries │ │ Capture │ │ Storage │ -│ (DIMACS/SDL)│ │ (Unified) │ │ (SQLite) │ -└─────────────┘ └──────────────┘ └─────────────┘ - │ - ▼ - ┌─────────────┐ - │ Report │ - │ Generation │ - │ (Bootstrap) │ - └─────────────┘ - -GITHUB ACTIONS (Publishing Only): -┌─────────────┐ ┌──────────────┐ -│ Pre-built │───▶│ GitHub │ -│ docs/ │ │ Pages │ -└─────────────┘ └──────────────┘ +Problem Files → Loaders → ProblemData → Solver Interfaces → SolverResult → Database → Reports + ↓ ↓ ↓ ↓ ↓ ↓ ↓ +DIMACS (.mat) MATLoader Unified Python/MATLAB Standardized SQLite HTML/CSV +SDPLIB (.dat-s) DATLoader Format Execution Results Storage Export ``` -### Symmetrical Interface Architecture +### Component Hierarchy ``` -BenchmarkRunner -├── PythonInterface -│ ├── cvxpy_clarabel, cvxpy_scs, cvxpy_ecos, cvxpy_osqp -│ └── scipy_linprog -├── MatlabInterface -│ ├── matlab_sedumi -│ └── matlab_sdpt3 -└── ProblemInterface - ├── MATLoader (DIMACS .mat files) - ├── DATLoader (SDPLIB .dat-s files) - ├── MPSLoader (LP .mps files) - ├── QPSLoader (QP .qps files) - └── PythonLoader (Python problem definitions) +BenchmarkRunner (scripts/benchmark/runner.py) +├── ProblemInterface (scripts/data_loaders/python/problem_interface.py) +│ ├── MATLoader (scripts/data_loaders/python/mat_loader.py) +│ └── DATLoader (scripts/data_loaders/python/dat_loader.py) +├── PythonInterface (scripts/solvers/python/python_interface.py) +│ ├── CvxpyRunner (scripts/solvers/python/cvxpy_runner.py) +│ └── ScipyRunner (scripts/solvers/python/scipy_runner.py) +├── MatlabInterface (scripts/solvers/matlab_octave/matlab_interface.py) +│ ├── matlab_interface.m → sedumi_runner.m +│ └── matlab_interface.m → sdpt3_runner.m +└── DatabaseManager (scripts/database/database_manager.py) + └── ResultProcessor (scripts/reporting/result_processor.py) ``` --- -## Component Architecture +## Core Data Models -### 1. Problem Interface Module +### Problem Data Structure +```python +# scripts/data_loaders/problem_loader.py +class ProblemData: + """Unified problem representation using SeDuMi format""" + def __init__(self, name: str, problem_class: str): + self.name = name # Problem identifier + self.problem_class = problem_class # 'LP', 'QP', 'SOCP', 'SDP' + + # SeDuMi standard format (internal representation) + self.A_eq = None # Constraint matrix (sparse) + self.b_eq = None # RHS vector + self.c = None # Objective coefficients + self.cone_structure = {} # Cone constraints specification + + # Optional QP data + self.P = None # Quadratic term matrix + + # Problem metadata + self._num_variables = 0 + self._num_constraints = 0 + self.metadata = {} # Additional problem information +``` -#### Component Role Definitions -The system follows a clear separation of concerns across three key components: +### Solver Result Structure +```python +# scripts/solvers/solver_interface.py +@dataclass +class SolverResult: + """Standardized result format for all solvers""" + solve_time: float # Execution time in seconds + status: str # 'optimal', 'infeasible', 'unbounded', 'error' + primal_objective_value: Optional[float] = None + dual_objective_value: Optional[float] = None + duality_gap: Optional[float] = None + primal_infeasibility: Optional[float] = None + dual_infeasibility: Optional[float] = None + iterations: Optional[int] = None + solver_name: Optional[str] = None + solver_version: Optional[str] = None + additional_info: Optional[Dict[str, Any]] = None + + @classmethod + def create_error_result(cls, error_msg: str, solve_time: float = 0.0) -> 'SolverResult': + """Create standardized error result""" + return cls(solve_time=solve_time, status='error', + additional_info={'error_message': error_msg}) +``` -- **Loaders**: Convert `problem_name` → `ProblemData` - - Role: Format-specific parsing and data loading - - Input: Problem name (string identifier) - - Output: Standardized ProblemData object - - Examples: `mat_loader.py`, `dat_loader.py`, `mat_loader.m`, `dat_loader.m` +--- -- **Runners**: Convert `ProblemData` → `SolverResult` - - Role: Solver-specific execution and result generation - - Input: ProblemData object with A, b, c, K matrices - - Output: Standardized SolverResult object - - Examples: `cvxpy_runner.py`, `sedumi_runner.m`, `sdpt3_runner.m` +## Implementation Details -- **Interfaces**: Orchestrate `problem_name` → `SolverResult` - - Role: Coordinate loaders and runners for complete workflow - - Input: Problem name (string identifier) - - Output: Standardized SolverResult object - - Examples: `python_interface.py`, `matlab_interface.py` +### 1. Problem Loading Implementation -This modular design ensures clean separation between data loading, solver execution, and workflow coordination, enabling independent development and testing of each component. +#### Configuration Registry +```yaml +# config/problem_registry.yaml +problem_libraries: + nb: # DIMACS problem example + display_name: "NB (DIMACS)" + file_path: "problems/DIMACS/data/ANTENNA/nb.mat.gz" + file_type: "mat" + library_name: "DIMACS" + for_test_flag: true + + arch0: # SDPLIB problem example + display_name: "ARCH0 (SDPLIB)" + file_path: "problems/SDPLIB/data/arch0.dat-s" + file_type: "dat-s" + library_name: "SDPLIB" + for_test_flag: true +``` -#### Unified Problem Management +#### Problem Interface Implementation ```python # scripts/data_loaders/python/problem_interface.py class ProblemInterface: @@ -117,82 +210,111 @@ class ProblemInterface: FORMAT_LOADERS = { "mat": MATLoader, # SeDuMi format (DIMACS) - "dat-s": DATLoader, # SDPA format (SDPLIB) - "mps": MPSLoader, # MPS format (LP) - "qps": QPSLoader, # QPS format (QP) - "python": PythonLoader, # Python definitions + "dat-s": DATLoader, # SDPA format (SDPLIB) + # Future extensions for MPS, QPS, Python formats } def load_problem(self, problem_name: str) -> ProblemData: """Load problem using appropriate format loader""" + config = self.get_problem_config(problem_name) + file_type = config['file_type'] + file_path = config['file_path'] + + loader_class = self.FORMAT_LOADERS[file_type] + loader = loader_class() + return loader.load(problem_name, file_path) def get_problem_config(self, problem_name: str) -> Dict[str, Any]: """Get problem configuration from registry""" + # Load and parse problem_registry.yaml + # Return configuration for specified problem def get_available_problems(self, library_filter: List[str] = None) -> List[str]: """Get filtered list of available problems""" + # Filter by library name if specified ``` -#### Problem Data Standardization -```python -# scripts/data_loaders/problem_loader.py -@dataclass -class ProblemData: - """Standardized problem data structure""" - name: str - problem_class: str # 'LP', 'QP', 'SOCP', 'SDP' - - # SeDuMi format (unified internal representation) - A_eq: np.ndarray # Constraint matrix - b_eq: np.ndarray # RHS vector - c: np.ndarray # Objective coefficients - cone_structure: Dict # Cone constraints specification - - # Optional QP data - P: Optional[np.ndarray] = None # Quadratic term - - # Problem metadata - _num_variables: int - _num_constraints: int - metadata: Dict[str, Any] = field(default_factory=dict) -``` - -### 2. Solver Interface Architecture +### 2. Python Solver Integration -#### Python Solver Interface +#### Solver Configuration +```python +# scripts/solvers/python/python_interface.py +PYTHON_SOLVER_CONFIGS = { + 'cvxpy_clarabel': { + 'solver_backend': 'CLARABEL', + 'runner_class': 'CvxpyRunner', + 'problem_types': ['LP', 'QP', 'SOCP', 'SDP'] + }, + 'cvxpy_scs': { + 'solver_backend': 'SCS', + 'runner_class': 'CvxpyRunner', + 'problem_types': ['LP', 'QP', 'SOCP', 'SDP'] + }, + 'cvxpy_ecos': { + 'solver_backend': 'ECOS', + 'runner_class': 'CvxpyRunner', + 'problem_types': ['LP', 'QP', 'SOCP'] + }, + 'cvxpy_osqp': { + 'solver_backend': 'OSQP', + 'runner_class': 'CvxpyRunner', + 'problem_types': ['QP'] + }, + 'cvxpy_cvxopt': { + 'solver_backend': 'CVXOPT', + 'runner_class': 'CvxpyRunner', + 'problem_types': ['LP', 'QP', 'SOCP', 'SDP'] + }, + 'cvxpy_sdpa': { + 'solver_backend': 'SDPA', + 'runner_class': 'CvxpyRunner', + 'problem_types': ['SDP'] + }, + 'cvxpy_scip': { + 'solver_backend': 'SCIP', + 'runner_class': 'CvxpyRunner', + 'problem_types': ['LP', 'QP'] + }, + 'cvxpy_highs': { + 'solver_backend': 'HIGHS', + 'runner_class': 'CvxpyRunner', + 'problem_types': ['LP'] + }, + 'scipy_linprog': { + 'method': 'highs', + 'runner_class': 'ScipyRunner', + 'problem_types': ['LP'] + }, +} +``` + +#### Python Execution Flow ```python # scripts/solvers/python/python_interface.py class PythonInterface: - """Unified interface for all Python-based solvers""" - - SOLVER_CONFIGURATIONS = { - 'cvxpy_clarabel': CvxpySolver, - 'cvxpy_scs': CvxpySolver, - 'cvxpy_ecos': CvxpySolver, - 'cvxpy_osqp': CvxpySolver, - 'scipy_linprog': ScipySolver, - } - def solve(self, problem_name: str, solver_name: str, problem_data: Optional[ProblemData] = None, timeout: Optional[float] = None) -> SolverResult: - """Unified solve method with automatic problem loading""" - - # 1. Create solver instance - solver = self.create_solver(solver_name) + """Execute Python solver with unified interface""" - # 2. Load problem data if not provided + # 1. Load problem data if not provided if problem_data is None: problem_data = self.problem_interface.load_problem(problem_name) - # 3. Validate compatibility - if not solver.validate_problem_compatibility(problem_data): - return SolverResult.create_error_result(...) + # 2. Get solver configuration + solver_config = self.PYTHON_SOLVER_CONFIGS[solver_name] + runner_class_name = solver_config['runner_class'] + + # 3. Create appropriate runner + if runner_class_name == 'CvxpyRunner': + runner = CvxpyRunner(solver_config['solver_backend']) + elif runner_class_name == 'ScipyRunner': + runner = ScipyRunner(solver_config['method']) # 4. Execute solver - result = solver.solve(problem_data, timeout=timeout) + result = runner.solve(problem_data, timeout=timeout) - # 5. Add problem class to additional_info for database storage + # 5. Add problem class information for database storage if not result.additional_info: result.additional_info = {} result.additional_info['problem_class'] = problem_data.problem_class @@ -200,69 +322,80 @@ class PythonInterface: return result ``` -#### MATLAB Solver Interface +### 3. MATLAB Solver Integration + +#### MATLAB Configuration ```python # scripts/solvers/matlab_octave/matlab_interface.py -class MatlabInterface: - """Unified interface for MATLAB/Octave solvers via subprocess execution""" - - MATLAB_SOLVER_CONFIGS = { - 'matlab_sedumi': { - 'matlab_solver': 'sedumi', - 'runner_function': 'sedumi_runner' - }, - 'matlab_sdpt3': { - 'matlab_solver': 'sdpt3', - 'runner_function': 'sdpt3_runner' - } +MATLAB_SOLVER_CONFIGS = { + 'matlab_sedumi': { + 'matlab_solver': 'sedumi', + 'runner_function': 'sedumi_runner' + }, + 'matlab_sdpt3': { + 'matlab_solver': 'sdpt3', + 'runner_function': 'sdpt3_runner' } - +} +``` + +#### MATLAB Execution Flow +```python +# scripts/solvers/matlab_octave/matlab_interface.py +class MatlabInterface: def solve(self, problem_name: str, solver_name: str, problem_data: Optional[ProblemData] = None, timeout: Optional[float] = None) -> SolverResult: - """Unified solve method calling matlab_interface.m directly""" + """Execute MATLAB solver via subprocess""" - # 1. Validate solver name + # 1. Validate solver configuration solver_config = self.MATLAB_SOLVER_CONFIGS[solver_name] - # 2. Call MATLAB interface via subprocess - result = self._call_matlab_interface( - problem_name=problem_name, - matlab_solver=solver_config["matlab_solver"], - runner_function=solver_config["runner_function"], - timeout=timeout or self.default_timeout - ) + # 2. Create temporary result file + result_file = self.temp_manager.create_temp_file('.json') + + # 3. Construct MATLAB command + matlab_command = f""" + cd('{os.getcwd()}'); + addpath(genpath('scripts/solvers/matlab_octave')); + matlab_interface('{problem_name}', '{solver_config["matlab_solver"]}', + '{result_file}', false, '{solver_config["runner_function"]}'); + """ - # 3. Add problem class information for database storage + # 4. Execute subprocess with timeout try: - problem_interface = ProblemInterface() - problem_data = problem_interface.load_problem(problem_name) - if not result.additional_info: - result.additional_info = {} - result.additional_info['problem_class'] = problem_data.problem_class - except Exception: - result.additional_info['problem_class'] = 'UNKNOWN' + process = subprocess.run([ + 'octave', '--eval', matlab_command + ], timeout=timeout, capture_output=True, text=True) + + if process.returncode != 0: + return SolverResult.create_error_result(f"MATLAB execution failed: {process.stderr}") + + # 5. Read JSON result + with open(result_file, 'r') as f: + matlab_result = json.load(f) + + # 6. Convert to SolverResult + result = self._convert_matlab_result(matlab_result) + + except subprocess.TimeoutExpired: + result = SolverResult.create_timeout_result(timeout) + except Exception as e: + result = SolverResult.create_error_result(str(e)) + finally: + self.temp_manager.cleanup_temp_file(result_file) return result ``` -### 3. MATLAB Integration Architecture - #### MATLAB Interface Entry Point ```matlab % scripts/solvers/matlab_octave/matlab_interface.m function matlab_interface(problem_name, solver_name, result_file, save_solutions, runner_function) % Main MATLAB interface for benchmark execution - % - % Input: - % problem_name: Name of problem from problem_registry.yaml - % solver_name: Name of solver ('sedumi' or 'sdpt3') - % result_file: Path to output JSON file for results - % save_solutions: Boolean flag to save solutions to .mat file - % runner_function: Name of the runner function to use try - % 1. Load problem registry and resolve problem file path + % 1. Load problem registry and resolve file path [problem_config, file_path] = read_problem_registry(problem_name); file_type = problem_config.file_type; @@ -276,76 +409,31 @@ function matlab_interface(problem_name, solver_name, result_file, save_solutions end % 3. Execute solver using dynamic function call - if exist(runner_function, 'file') - [x, y, result] = feval(runner_function, A, b, c, K); - else - error('Solver runner function not found: %s', runner_function); - end + [x, y, result] = feval(runner_function, A, b, c, K); % 4. Calculate standardized metrics if ~isempty(x) && ~isempty(y) result = calculate_solver_metrics(result, x, y, A, b, c, K); end - % 5. Save solution vectors if requested - if save_solutions && strcmp(result.status, 'optimal') - save_solutions_if_needed(problem_name, solver_name, x, y, save_solutions); - end - - % 6. Convert result to JSON-compatible format and save + % 5. Convert to JSON and save json_result = convert_to_json_result(result); save_json_safely(json_result, result_file); catch ME - % Save error result to JSON file error_result = create_error_result(ME, solver_name); save_json_safely(error_result, result_file); end end ``` -#### MATLAB Solver Runners -```matlab -% scripts/solvers/matlab_octave/sedumi_runner.m -function [x, y, result] = sedumi_runner(A, b, c, K) - % SeDuMi solver runner with standardized interface - - result = initialize_result_structure(); - result.solver_name = 'SeDuMi'; - - try - % Configure SeDuMi options - pars.fid = 0; % Suppress output for benchmarking - - % Execute SeDuMi solver - solve_start_time = tic; - [x, y, info] = sedumi(A, b, c, K, pars); - solve_time = toc(solve_start_time); - - % Map SeDuMi status to standardized format - result.status = map_sedumi_status(info.pinf, info.dinf, info.numerr); - result.solve_time = solve_time; - result.iterations = info.iter; - result.primal_objective_value = NaN; % Calculated by metrics function - result.dual_objective_value = NaN; % Calculated by metrics function - result.duality_gap = NaN; % Calculated by metrics function - result.primal_infeasibility = NaN; - result.dual_infeasibility = NaN; - result.solver_version = 'SeDuMi-1.3.7'; - - catch ME - result.status = 'error'; - result.error_message = ME.message; - x = []; - y = []; - end -end -``` +--- -### 4. Database Architecture +## Database Implementation -#### Unified Results Schema +### Schema Definition ```sql +-- scripts/database/schema.sql CREATE TABLE results ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -363,9 +451,9 @@ CREATE TABLE results ( commit_hash TEXT NOT NULL, -- Git commit hash timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, - -- Standardized solver results (unified across Python/MATLAB) + -- Standardized solver results solve_time REAL, -- Execution time in seconds - status TEXT, -- 'OPTIMAL', 'INFEASIBLE', 'UNBOUNDED', etc. + status TEXT, -- 'OPTIMAL', 'INFEASIBLE', 'UNBOUNDED', 'ERROR', 'TIMEOUT', 'UNSUPPORTED' primal_objective_value REAL, -- Primal objective value dual_objective_value REAL, -- Dual objective value duality_gap REAL, -- Primal-dual gap @@ -383,463 +471,131 @@ CREATE INDEX idx_solver_problem ON results(solver_name, problem_name); CREATE INDEX idx_problem_type ON results(problem_type); ``` -#### Standardized Result Format +### Database Manager Implementation ```python -# scripts/solvers/solver_interface.py -@dataclass -class SolverResult: - """Standardized solver result format for both Python and MATLAB solvers""" - - solve_time: float - status: str # Standardized status strings - primal_objective_value: Optional[float] = None - dual_objective_value: Optional[float] = None - duality_gap: Optional[float] = None - primal_infeasibility: Optional[float] = None - dual_infeasibility: Optional[float] = None - iterations: Optional[int] = None - solver_name: Optional[str] = None - solver_version: Optional[str] = None - additional_info: Optional[Dict[str, Any]] = None - - @classmethod - def create_error_result(cls, error_msg: str, solve_time: float = 0.0, - solver_name: str = None, solver_version: str = None) -> 'SolverResult': - """Create standardized error result""" - - @classmethod - def create_timeout_result(cls, timeout: float, solver_name: str = None, - solver_version: str = None) -> 'SolverResult': - """Create standardized timeout result""" -``` - -### 5. Benchmark Execution Engine - -#### Unified Benchmark Runner -```python -# scripts/benchmark/runner.py -class BenchmarkRunner: - """Main benchmark execution engine with symmetrical solver interfaces""" - - def __init__(self, database_manager: Optional[DatabaseManager] = None, - dry_run: bool = False, save_solutions: bool = False): - """Initialize with unified interfaces""" - - # Initialize all interfaces - self.python_interface = PythonInterface() - self.matlab_interface = MatlabInterface() - self.problem_interface = ProblemInterface() +# scripts/database/database_manager.py +class DatabaseManager: + def store_result(self, solver_name: str, problem_name: str, + result: SolverResult, problem_config: Dict[str, Any]) -> None: + """Store benchmark result with complete metadata""" + + # Extract metadata + problem_library = problem_config.get('library_name', 'internal') + problem_type = result.additional_info.get('problem_class', 'UNKNOWN') - # Create explicit solver-to-interface mapping - self._solver_interface_map = self._build_solver_interface_map() - - def run_single_benchmark(self, problem_name: str, solver_name: str) -> None: - """Execute single benchmark with explicit interface routing""" - - # Get the appropriate interface type from mapping - interface_type = self._solver_interface_map.get(solver_name) - - if interface_type == 'python': - result = self.python_interface.solve(problem_name, solver_name) - elif interface_type == 'matlab': - result = self.matlab_interface.solve(problem_name, solver_name) - else: - raise ValueError(f"Unknown solver '{solver_name}'") - - # Store result in database - problem_config = self.problem_interface.get_problem_config(problem_name) - self.store_result(solver_name, problem_name, result, problem_config) - - def _build_solver_interface_map(self) -> Dict[str, str]: - """Build explicit mapping of solver names to interface types""" - mapping = {} + # Get environment and version information + environment_info = self.environment_collector.get_environment_info() + commit_hash = self.git_utils.get_current_commit_hash() - # Add Python solvers - for solver_name in self.python_interface.get_available_solvers(): - mapping[solver_name] = 'python' + # Insert into database + query = """ + INSERT OR REPLACE INTO results + (solver_name, solver_version, problem_library, problem_name, problem_type, + environment_info, commit_hash, solve_time, status, primal_objective_value, + dual_objective_value, duality_gap, primal_infeasibility, dual_infeasibility, + iterations, memo) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ - # Add MATLAB solvers - for solver_name in self.matlab_interface.get_available_solvers(): - mapping[solver_name] = 'matlab' + self.cursor.execute(query, ( + solver_name, result.solver_version, problem_library, problem_name, problem_type, + json.dumps(environment_info), commit_hash, result.solve_time, result.status, + result.primal_objective_value, result.dual_objective_value, result.duality_gap, + result.primal_infeasibility, result.dual_infeasibility, result.iterations, + json.dumps(result.additional_info) if result.additional_info else None + )) - return mapping + self.connection.commit() ``` --- -## Data Flow Specifications - -### 1. Problem Loading Flow - -#### External Library Integration -``` -DIMACS Problems (47 problems): -├── Source: SeDuMi .mat format -├── Loader: scripts/data_loaders/python/mat_loader.py -├── MATLAB: scripts/data_loaders/matlab_octave/mat_loader.m -└── Format: {A, b, c, K} SeDuMi standard - -SDPLIB Problems (92 problems): -├── Source: SDPA .dat-s format -├── Loader: scripts/data_loaders/python/dat_loader.py -├── MATLAB: scripts/data_loaders/matlab_octave/dat_loader.m -└── Format: Converted to SeDuMi {A, b, c, K} -``` - -#### Unified Problem Registry -```yaml -# config/problem_registry.yaml -problem_libraries: - nb: # DIMACS problem - display_name: "NB (DIMACS)" - file_path: "problems/DIMACS/data/ANTENNA/nb.mat.gz" - file_type: "mat" - library_name: "DIMACS" - for_test_flag: true - - arch0: # SDPLIB problem - display_name: "ARCH0 (SDPLIB)" - file_path: "problems/SDPLIB/data/arch0.dat-s" - file_type: "dat-s" - library_name: "SDPLIB" - for_test_flag: true -``` - -### 2. Solver Execution Flow +## Report Generation -#### Python Solver Flow -``` -PythonInterface.solve() -├── 1. Create solver instance (cvxpy_clarabel, etc.) -├── 2. Load problem data via ProblemInterface -├── 3. Validate solver-problem compatibility -├── 4. Execute solver.solve(problem_data) -├── 5. Add problem_class to result.additional_info -└── 6. Return standardized SolverResult -``` - -#### MATLAB Solver Flow -``` -MatlabInterface.solve() -├── 1. Validate MATLAB solver configuration -├── 2. Create temporary result file -├── 3. Execute subprocess: octave --eval "matlab_interface(...)" -├── 4. matlab_interface.m execution: -│ ├── Load problem via mat_loader.m or dat_loader.m -│ ├── Execute sedumi_runner.m or sdpt3_runner.m -│ ├── Calculate standardized metrics -│ └── Save JSON result to temporary file -├── 5. Read JSON result and convert to SolverResult -├── 6. Load problem data to get problem_class -├── 7. Add problem_class to result.additional_info -└── 8. Return standardized SolverResult -``` - -### 3. Result Storage and Processing - -#### Database Storage Flow -```python -def store_result(self, solver_name: str, problem_name: str, - result: SolverResult, problem_config: Dict[str, Any]) -> None: - """Store result with unified metadata extraction""" - - # Determine problem library and type - problem_library = problem_config.get('library_name', 'internal') - - # Get problem type from result.additional_info (set by interfaces) - if hasattr(result, 'additional_info') and result.additional_info: - problem_type = result.additional_info.get('problem_class', 'UNKNOWN') - else: - problem_type = problem_config.get('problem_type', 'UNKNOWN') - - # Store in database with standardized schema - self.db.store_result( - solver_name=solver_name, - solver_version=result.solver_version, - problem_library=problem_library, - problem_name=problem_name, - problem_type=problem_type, - # ... all standardized fields - ) -``` - ---- - -## Implementation Guidelines - -### 1. Fair Benchmarking Principles - -#### Minimal Configuration Approach +### HTML Generation Implementation ```python -# Python solvers - use defaults with verbose=False only -cvxpy_options = {'verbose': False} -scipy_options = {'method': 'highs', 'verbose': False} - -# MATLAB solvers - use defaults with output suppression only -sedumi_options = {'fid': 0} # Suppress output -sdpt3_options = {'printlevel': 0} # Suppress output -``` - -#### Standardized Metrics Calculation -```matlab -% scripts/solvers/matlab_octave/matlab_interface.m -function result = calculate_solver_metrics(result, x, y, A, b, c, K) - % Calculate standardized metrics for fair comparison - - % Primal objective: c'*x - if ~isempty(x) && ~isempty(c) - result.primal_objective_value = c(:)' * x(:); - end - - % Dual objective: b'*y - if ~isempty(y) && ~isempty(b) - result.dual_objective_value = b(:)' * y(:); - end - - % Duality gap: |primal - dual| - if ~isnan(result.primal_objective_value) && ~isnan(result.dual_objective_value) - result.duality_gap = abs(result.primal_objective_value - result.dual_objective_value); - end - - % Primal infeasibility: ||Ax - b|| / (1 + ||b||) - if ~isempty(x) && ~isempty(A) && ~isempty(b) - primal_residual = A * x - b; - result.primal_infeasibility = norm(primal_residual) / (1 + norm(b)); - end - - % Dual infeasibility: cone projection distance - % (Implementation specific to cone structure K) -end -``` - -### 2. Error Handling and Resilience - -#### Graceful Degradation Strategy -```python -# Individual solver failures don't affect overall system -try: - result = solver_interface.solve(problem_name, solver_name) - self.store_result(solver_name, problem_name, result, problem_config) -except Exception as e: - error_msg = f"Solver {solver_name} failed on {problem_name}: {e}" - logger.error(error_msg) - self.store_error_result(solver_name, problem_name, error_msg, problem_config) - # Continue with next solver/problem combination -``` - -#### MATLAB-Specific Error Handling -```python -def _call_matlab_interface(self, problem_name: str, matlab_solver: str, - runner_function: str, timeout: float) -> SolverResult: - """Call MATLAB with comprehensive error handling""" - - try: - # Execute MATLAB process with timeout - process = subprocess.run([ - 'octave', '--eval', matlab_command - ], timeout=timeout, capture_output=True, text=True) - - # Check for MATLAB execution errors - if process.returncode != 0: - return SolverResult.create_error_result( - f"MATLAB execution failed: {process.stderr}" - ) - - # Read result from temporary JSON file - with open(result_file, 'r') as f: - matlab_result = json.load(f) - - # Convert to standardized SolverResult - return self._convert_matlab_result(matlab_result, solve_time) - - except subprocess.TimeoutExpired: - return SolverResult.create_timeout_result(timeout) - except Exception as e: - return SolverResult.create_error_result(str(e)) - finally: - # Always cleanup temporary files - cleanup_temporary_files() -``` - -### 3. Performance Optimization - -#### Lazy Initialization Pattern -```python -class PythonInterface: - def __init__(self): - """Initialize interface without creating solver instances""" - self._solver_cache = {} # Cache for expensive solver objects - - def create_solver(self, solver_name: str) -> SolverInterface: - """Create solver instance with caching""" - if solver_name not in self._solver_cache: - solver_class = self.SOLVER_CONFIGURATIONS[solver_name] - self._solver_cache[solver_name] = solver_class(...) - return self._solver_cache[solver_name] -``` - -#### Efficient Database Queries -```python -# Use indexed queries for latest results -def get_latest_results(self) -> List[BenchmarkResult]: - """Get latest results using optimized query""" - query = """ - SELECT * FROM results - WHERE (commit_hash, environment_info, timestamp) IN ( - SELECT commit_hash, environment_info, MAX(timestamp) - FROM results - GROUP BY solver_name, problem_name - ) - ORDER BY problem_library, problem_name, solver_name - """ - return self.execute_query(query) +# scripts/reporting/html_generator.py +class HTMLGenerator: + def generate_all_reports(self) -> None: + """Generate complete set of HTML reports""" + + # Load latest results from database + results = self.db.get_latest_results() + + # Generate main dashboard + self.generate_dashboard(results, 'docs/pages/index.html') + + # Generate results matrix + self.generate_results_matrix(results, 'docs/pages/results_matrix.html') + + # Generate raw data view + self.generate_raw_data_view(results, 'docs/pages/raw_data.html') + + # Export data files + self.data_exporter.export_json(results, 'docs/pages/data/benchmark_results.json') + self.data_exporter.export_csv(results, 'docs/pages/data/benchmark_results.csv') + self.data_exporter.export_summary(results, 'docs/pages/data/summary.json') ``` --- -## Testing and Validation - -### 1. System Validation -```bash -# Complete environment validation -python main.py --validate-verbose - -# Solver-specific validation -python main.py --validate --solvers cvxpy_clarabel,matlab_sedumi - -# Problem-specific validation -python main.py --validate --problems nb,arch0 -``` +## Fair Benchmarking Implementation -### 2. Integration Testing +### Minimal Configuration Approach ```python -# Test Python-MATLAB parity on same problems -def test_solver_parity(): - problem_name = "nb" # SOCP problem - - # Run Python solver - python_result = python_interface.solve(problem_name, "cvxpy_clarabel") - - # Run MATLAB solver - matlab_result = matlab_interface.solve(problem_name, "matlab_sedumi") - - # Compare results (allowing numerical tolerance) - assert abs(python_result.primal_objective_value - - matlab_result.primal_objective_value) < 1e-6 -``` - -### 3. Performance Benchmarking -```bash -# Measure execution time across all solvers -python tests/performance/benchmark_matlab_vs_python.py - -# Memory usage analysis -python tests/performance/memory_profiling.py -``` - ---- - -## Deployment and Maintenance - -### 1. GitHub Actions Integration -```yaml -# .github/workflows/benchmark.yml -name: Solver Benchmark -on: - schedule: - - cron: '0 2 * * 1' # Weekly Monday 2AM - workflow_dispatch: - -jobs: - benchmark: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.12' - - name: Setup Octave - run: sudo apt-get install octave - - name: Install dependencies - run: pip install -r requirements.txt - - name: Run benchmarks - run: python main.py --all - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./docs/pages +# Solver configurations use minimal parameters to ensure fair comparison +FAIR_BENCHMARKING_CONFIG = { + 'python_solvers': { + 'cvxpy_options': {'verbose': False}, # Only suppress output + 'scipy_options': {'method': 'highs'} # Use default method + }, + 'matlab_solvers': { + 'sedumi_options': {'fid': 0}, # Only suppress output + 'sdpt3_options': {'printlevel': 0} # Only suppress output + } +} ``` -### 2. Monitoring and Alerting +### Reproducibility Implementation ```python -# Monitor solver success rates -def monitor_solver_health(): - """Alert if solver success rate drops below threshold""" - results = get_latest_results() - - for solver_name in get_available_solvers(): - solver_results = [r for r in results if r.solver_name == solver_name] - success_rate = len([r for r in solver_results if r.status == 'OPTIMAL']) / len(solver_results) - - if success_rate < 0.8: # Alert threshold - send_alert(f"Solver {solver_name} success rate: {success_rate:.1%}") -``` - -### 3. Data Backup and Recovery -```bash -# Automated database backup -cp database/results.db database/results_$(date +%Y%m%d).backup - -# Recovery from backup -cp database/results_20241201.backup database/results.db +# scripts/utils/environment_info.py +class EnvironmentCollector: + def get_environment_info(self) -> Dict[str, Any]: + """Collect complete environment information for reproducibility""" + return { + 'python_version': sys.version, + 'platform': platform.platform(), + 'architecture': platform.architecture(), + 'processor': platform.processor(), + 'installed_packages': self._get_package_versions(), + 'git_commit': self.git_utils.get_current_commit_hash(), + 'timestamp': datetime.now().isoformat(), + 'hostname': socket.gethostname() + } ``` --- -## Future Extensions +## Development Guidelines -### 1. Additional Solver Support -```python -# Adding new solver types -class MOSEKInterface: - """Interface for MOSEK commercial solver""" - -class GurobiInterface: - """Interface for Gurobi commercial solver""" -``` +### Adding New Python Solvers +1. **Add configuration** to `PYTHON_SOLVER_CONFIGS` in `python_interface.py` +2. **Create runner class** if needed (or extend existing `CvxpyRunner`) +3. **Update problem type compatibility** in configuration +4. **Test with validation framework** -### 2. Advanced Analytics -```python -# Performance trend analysis -def analyze_solver_trends(): - """Analyze solver performance over time""" - -# Convergence analysis -def analyze_convergence_patterns(): - """Study solver convergence behavior""" -``` +### Adding New MATLAB Solvers +1. **Create solver runner** `{solver}_runner.m` following standard interface +2. **Add configuration** to `MATLAB_SOLVER_CONFIGS` in `matlab_interface.py` +3. **Implement MEX compilation** in `setup_matlab_solvers.m` +4. **Test with validation framework** -### 3. Cloud Deployment -```yaml -# Kubernetes deployment configuration -apiVersion: apps/v1 -kind: CronJob -metadata: - name: solver-benchmark -spec: - schedule: "0 2 * * 1" - jobTemplate: - spec: - template: - spec: - containers: - - name: benchmark - image: solver-benchmark:latest - command: ["python", "main.py", "--all"] -``` +### Adding New Problem Formats +1. **Create loader classes** in both `python/` and `matlab_octave/` directories +2. **Add format mapping** to `FORMAT_LOADERS` in `problem_interface.py` +3. **Update problem registry** to include new format problems +4. **Implement format conversion** to unified SeDuMi format --- -This technical design provides a comprehensive foundation for the optimization solver benchmark system, supporting both current functionality and future extensions while maintaining the core principles of fair benchmarking and production reliability. \ No newline at end of file +This detailed design provides comprehensive implementation guidance while maintaining focus on research applications and eliminating unnecessary production complexity. \ No newline at end of file From f1fbce2522d887f8f480a07bd2f20dadba2e4fbd Mon Sep 17 00:00:00 2001 From: napinoco Date: Sun, 20 Jul 2025 23:49:12 +0900 Subject: [PATCH 02/28] [Complete] Redesign HTML reports for reliability and usability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Overview Dashboard (index.html) Improvements: - Remove unreliable status/success rate information (self-reported values) - Focus on objective performance metrics: solver count, problem count, libraries - Add comprehensive solver list display with proper formatting - Replace performance analysis with TBD section (evaluation criteria in development) - Add detailed environment information (Git commit, platform, Python, MATLAB) ## Data Export System: - Create dedicated data/index.html with professional download interface - Remove summary.json generation and references (redundant with main reports) - Provide JSON and CSV formats with clear documentation - Remove Usage License section per requirements ## Technical Improvements: - Convert all timestamp displays to UTC with explicit timezone indication - Fix bullet point positioning issues in lists (add padding-left: 1.5rem) - Add new result processor methods for library/type problem counting - Maintain data independence: HTML generation uses database directly, not export files ## System Architecture: - Preserve data integrity: all reports generated from same database source - Enable independent data format changes without affecting HTML generation - Streamline export process: only essential files (JSON/CSV) generated Generated files: - docs/pages/index.html (redesigned overview) - docs/pages/results_matrix.html (UTC timestamps) - docs/pages/raw_data.html (UTC timestamps) - docs/pages/data/index.html (new data export page) - docs/pages/data/benchmark_results.json - docs/pages/data/benchmark_results.csv 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- docs/pages/data/index.html | 219 ++++++++++++ docs/pages/data/summary.json | 320 ----------------- docs/pages/index.html | 474 +++----------------------- docs/pages/raw_data.html | 71 ++-- docs/pages/results_matrix.html | 30 +- main.py | 4 +- scripts/reporting/html_generator.py | 468 +++++++++++++++++-------- scripts/reporting/result_processor.py | 86 +++++ 8 files changed, 751 insertions(+), 921 deletions(-) create mode 100644 docs/pages/data/index.html delete mode 100644 docs/pages/data/summary.json diff --git a/docs/pages/data/index.html b/docs/pages/data/index.html new file mode 100644 index 0000000..11ce952 --- /dev/null +++ b/docs/pages/data/index.html @@ -0,0 +1,219 @@ + + + + + + Optimization Solver Benchmark - Data Exports + + + +
+

🔬 Optimization Solver Benchmark

+

Data Exports - Download Benchmark Results

+

Generated: 2025-07-20 14:46:32 UTC

+
+ + + +
+
+

📁 Available Data Files

+
+

+ Download complete benchmark results in various formats for further analysis and research. +

+ +
+

📊 Complete Results (JSON)

+

Full benchmark results with all metadata in JSON format. Includes solver information, problem details, execution environment, and detailed performance metrics.

+ 📥 Download JSON + Structured data format +
+ +
+

📈 Complete Results (CSV)

+

Full benchmark results in CSV format for spreadsheet analysis. Contains the same data as JSON but in tabular format for easy import into analysis tools.

+ 📥 Download CSV + Spreadsheet compatible +
+ +
+
+ +
+

📖 Data Format Documentation

+
+

JSON Structure

+
    +
  • benchmark_results.json: Array of result objects with complete solver execution data
    • +
+ +

CSV Format

+
    +
  • benchmark_results.csv: Flattened tabular format with one row per solver-problem combination
    • +
  • Headers include: solver_name, problem_name, problem_type, solve_time, status, objective_value, etc.
    • +
+
+
+
+ + + + \ No newline at end of file diff --git a/docs/pages/data/summary.json b/docs/pages/data/summary.json deleted file mode 100644 index ea7a9f0..0000000 --- a/docs/pages/data/summary.json +++ /dev/null @@ -1,320 +0,0 @@ -{ - "metadata": { - "generated_at": "2025-07-19T23:39:48.346008", - "total_results": 1026, - "format": "summary_only" - }, - "overall_statistics": { - "total_results": 1026, - "total_solvers": 11, - "total_problems": 138, - "success_rate": 0.4, - "avg_solve_time": 12.413195807016402, - "successful_results": 232, - "unsupported_results": 446, - "applicable_results": 580, - "problem_type_distribution": { - "UNKNOWN": 69, - "SDP": 809, - "SOCP": 148 - }, - "library_distribution": { - "DIMACS": 330, - "SDPLIB": 696 - }, - "status_distribution": { - "ERROR": 145, - "NUM_ERROR": 40, - "OPTIMAL (INACCURATE)": 86, - "OPTIMAL": 232, - "UNSUPPORTED": 446, - "TIMEOUT": 3, - "MAX_ITER": 10, - "INFEASIBLE": 16, - "UNKNOWN": 33, - "UNBOUNDED (INACCURATE)": 1, - "UNBOUNDED": 12, - "INFEASIBLE (INACCURATE)": 2 - }, - "solver_names": [ - "cvxpy_clarabel", - "cvxpy_cvxopt", - "cvxpy_ecos", - "cvxpy_highs", - "cvxpy_osqp", - "cvxpy_scip", - "cvxpy_scs", - "cvxpy_sdpa", - "matlab_sdpt3", - "matlab_sedumi", - "scipy_linprog" - ], - "problem_names": [ - "arch0", - "arch2", - "arch4", - "arch8", - "biomedP", - "bm1", - "control1", - "control10", - "control11", - "control2", - "control3", - "control4", - "control5", - "control6", - "control7", - "control8", - "control9", - "copo14", - "copo23", - "equalG11", - "equalG51", - "fap09", - "fap36", - "filter48", - "filtinf1", - "gpp100", - "gpp124-1", - "gpp124-2", - "gpp124-3", - "gpp124-4", - "gpp250-1", - "gpp250-2", - "gpp250-3", - "gpp250-4", - "gpp500-1", - "gpp500-2", - "gpp500-3", - "gpp500-4", - "hamming_11_2", - "hamming_7_5_6", - "hamming_8_3_4", - "hamming_9_5_6", - "hamming_9_8", - "hinf1", - "hinf10", - "hinf11", - "hinf12", - "hinf12_sdp", - "hinf13", - "hinf13_sdp", - "hinf14", - "hinf15", - "hinf2", - "hinf3", - "hinf4", - "hinf5", - "hinf6", - "hinf7", - "hinf8", - "hinf9", - "industry2", - "infd1", - "infd2", - "infp1", - "infp2", - "maxG11", - "maxG32", - "maxG51", - "maxG55", - "maxG60", - "mcp100", - "mcp124-1", - "mcp124-2", - "mcp124-3", - "mcp124-4", - "mcp250-1", - "mcp250-2", - "mcp250-3", - "mcp250-4", - "mcp500-1", - "mcp500-2", - "mcp500-3", - "mcp500-4", - "minphase", - "nb", - "nb_L1", - "nb_L2", - "nb_L2_bessel", - "nql180", - "nql180old", - "nql30", - "nql30old", - "nql60", - "nql60old", - "qap10", - "qap5", - "qap6", - "qap7", - "qap8", - "qap9", - "qpG11", - "qpG51", - "qssp180old", - "qssp30", - "qssp30old", - "qssp60", - "qssp60old", - "sched_100_100_orig", - "sched_100_100_scaled", - "sched_100_50_orig", - "sched_100_50_scaled", - "sched_200_100_orig", - "sched_200_100_scaled", - "sched_50_50_orig", - "sched_50_50_scaled", - "ss30", - "theta1", - "theta2", - "theta3", - "theta4", - "theta5", - "theta6", - "thetaG11", - "thetaG51", - "torusg3-15", - "torusg3-8", - "toruspm3-15-50", - "toruspm3-8-50", - "truss1", - "truss2", - "truss3", - "truss4", - "truss5", - "truss5_sdp", - "truss6", - "truss7", - "truss8", - "truss8_sdp" - ] - }, - "solver_performance": [ - { - "solver_name": "cvxpy_scip", - "problems_attempted": 81, - "problems_applicable": 3, - "problems_solved": 2, - "problems_unsupported": 78, - "success_rate": 0.6666666666666666, - "avg_solve_time": 54.10598909854889, - "min_solve_time": 50.8220100402832, - "max_solve_time": 57.389968156814575 - }, - { - "solver_name": "cvxpy_clarabel", - "problems_attempted": 79, - "problems_applicable": 79, - "problems_solved": 47, - "problems_unsupported": 0, - "success_rate": 0.5949367088607594, - "avg_solve_time": 8.219515762831035, - "min_solve_time": 0.00506281852722168, - "max_solve_time": 116.92459893226624 - }, - { - "solver_name": "cvxpy_scs", - "problems_attempted": 62, - "problems_applicable": 62, - "problems_solved": 31, - "problems_unsupported": 0, - "success_rate": 0.5, - "avg_solve_time": 14.027361787733485, - "min_solve_time": 0.005390167236328125, - "max_solve_time": 92.80738425254822 - }, - { - "solver_name": "matlab_sdpt3", - "problems_attempted": 135, - "problems_applicable": 135, - "problems_solved": 50, - "problems_unsupported": 0, - "success_rate": 0.37037037037037035, - "avg_solve_time": 8.737274696712733, - "min_solve_time": 0.24263908333333334, - "max_solve_time": 138.05433633333334 - }, - { - "solver_name": "matlab_sedumi", - "problems_attempted": 138, - "problems_applicable": 138, - "problems_solved": 51, - "problems_unsupported": 0, - "success_rate": 0.3695652173913043, - "avg_solve_time": 19.24742645824121, - "min_solve_time": 0.18138175, - "max_solve_time": 300.0 - }, - { - "solver_name": "cvxpy_cvxopt", - "problems_attempted": 74, - "problems_applicable": 74, - "problems_solved": 25, - "problems_unsupported": 0, - "success_rate": 0.33783783783783783, - "avg_solve_time": 12.945587092882967, - "min_solve_time": 0.027730941772460938, - "max_solve_time": 228.55329608917236 - }, - { - "solver_name": "cvxpy_ecos", - "problems_attempted": 90, - "problems_applicable": 12, - "problems_solved": 4, - "problems_unsupported": 78, - "success_rate": 0.3333333333333333, - "avg_solve_time": 13.632525010542436, - "min_solve_time": 0.17055892944335938, - "max_solve_time": 52.14936709403992 - }, - { - "solver_name": "cvxpy_sdpa", - "problems_attempted": 74, - "problems_applicable": 74, - "problems_solved": 22, - "problems_unsupported": 0, - "success_rate": 0.2972972972972973, - "avg_solve_time": 9.483143959959893, - "min_solve_time": 0.007824182510375977, - "max_solve_time": 104.95400619506836 - }, - { - "solver_name": "cvxpy_highs", - "problems_attempted": 98, - "problems_applicable": 1, - "problems_solved": 0, - "problems_unsupported": 97, - "success_rate": 0.0, - "avg_solve_time": 0.0, - "min_solve_time": 0.0, - "max_solve_time": 0.0 - }, - { - "solver_name": "cvxpy_osqp", - "problems_attempted": 98, - "problems_applicable": 1, - "problems_solved": 0, - "problems_unsupported": 97, - "success_rate": 0.0, - "avg_solve_time": 0.0, - "min_solve_time": 0.0, - "max_solve_time": 0.0 - }, - { - "solver_name": "scipy_linprog", - "problems_attempted": 97, - "problems_applicable": 1, - "problems_solved": 0, - "problems_unsupported": 96, - "success_rate": 0.0, - "avg_solve_time": 0.0, - "min_solve_time": 0.0, - "max_solve_time": 0.0 - } - ], - "environment": { - "commit_hash": "96ca043787ef8c38515cfa8d108d16b61b823e04", - "platform": "Darwin" - } -} \ No newline at end of file diff --git a/docs/pages/index.html b/docs/pages/index.html index 2f7e4dd..a7020d1 100644 --- a/docs/pages/index.html +++ b/docs/pages/index.html @@ -255,7 +255,7 @@

🔬 Optimization Solver Benchmark

Overview Dashboard - Latest Results

-

Generated: 2025-07-19 23:39:48

+

Generated: 2025-07-20 14:46:32 UTC