final ranking script

README.md

@@ -2,50 +2,13 @@
 
 # Holistic Evaluation of Language Models
 
-[comment]: <> (When using the img tag, which allows us to specify size, src has to be a URL.)
-<img src="https://github.com/stanford-crfm/helm/raw/main/src/helm/benchmark/static/images/helm-logo.png" alt=""  width="800"/>
+This is a fork of https://github.com/stanford-crfm/helm which we used for the 2023 NeurIPS LLM efficiency competition https://llm-efficiency-challenge.github.io/
 
-Welcome! The **`crfm-helm`** Python package contains code used in the **Holistic Evaluation of Language Models** project ([paper](https://arxiv.org/abs/2211.09110), [website](https://crfm.stanford.edu/helm/latest/)) by [Stanford CRFM](https://crfm.stanford.edu/). This package includes the following features:
+It was private because the tasks we were testing on had to be undisclosed to the final participants and included
+* Math
+* Corr2cause
+* Justice
+* Samsum
+* Ethics
 
-- Collection of datasets in a standard format (e.g., NaturalQuestions)
-- Collection of models accessible via a unified API (e.g., GPT-3, MT-NLG, OPT, BLOOM)
-- Collection of metrics beyond accuracy (efficiency, bias, toxicity, etc.)
-- Collection of perturbations for evaluating robustness and fairness (e.g., typos, dialect)
-- Modular framework for constructing prompts from datasets
-- Proxy server for managing accounts and providing unified interface to access models
-<!--intro-end-->
-
-To get started, refer to [the documentation on Read the Docs](https://crfm-helm.readthedocs.io/) for how to install and run the package.
-
-## Directory Structure
-
-The directory structure for this repo is as follows
-
-```
-├── docs # MD used to generate readthedocs
-│
-├── scripts # Python utility scripts for HELM
-│ ├── cache
-│ ├── data_overlap # Calculate train test overlap
-│ │ ├── common
-│ │ ├── scenarios
-│ │ └── test
-│ ├── efficiency
-│ ├── fact_completion
-│ ├── offline_eval
-│ └── scale
-└── src
-├── helm # Benchmarking Scripts for HELM
-│ │
-│ ├── benchmark # Main Python code for running HELM
-│ │ │
-│ │ └── static # Current JS (Jquery) code for rendering front-end
-│ │ │
-│ │ └── ...
-│ │
-│ ├── common # Additional Python code for running HELM
-│ │
-│ └── proxy # Python code for external web requests
-│
-└── helm-frontend # New React Front-end
-```
+If you're interested in using these tasks in your own work please feel free to copy paste

build_secret_run_spec.py

@@ -12,13 +12,13 @@
     {'scenario': 'ethics', 'description': "ethics_utilitarianism:model=neurips/local,data_augmentation=canonical", 'priority': 1},
 
     ## Math datasets
-     {'scenario': 'math', 'description': "math:model=neurips/local,subject=number_theory,level=1,use_official_examples=True", 'priority': 2},
-     {'scenario': 'math', 'description': "math:model=neurips/local,subject=intermediate_algebra,level=1,use_official_examples=True", 'priority': 2},
-     {'scenario': 'math', 'description': "math:model=neurips/local,subject=algebra,level=1,use_official_examples=True", 'priority': 2},
-     {'scenario': 'math', 'description': "math:model=neurips/local,subject=prealgebra,level=1,use_official_examples=True", 'priority': 2},
-     {'scenario': 'math', 'description': "math:model=neurips/local,subject=geometry,level=1,use_official_examples=True", 'priority': 2},
-     {'scenario': 'math', 'description': "math:model=neurips/local,subject=counting_and_probability,level=1,use_official_examples=True", 'priority': 2},
-     {'scenario': 'math', 'description': "math:model=neurips/local,subject=precalculus,level=1,use_official_examples=True", 'priority': 2},
+#     {'scenario': 'math', 'description': "math:model=neurips/local,subject=number_theory,level=1,use_official_examples=True", 'priority': 2},
+#     {'scenario': 'math', 'description': "math:model=neurips/local,subject=intermediate_algebra,level=1,use_official_examples=True", 'priority': 2},
+#     {'scenario': 'math', 'description': "math:model=neurips/local,subject=algebra,level=1,use_official_examples=True", 'priority': 2},
+#     {'scenario': 'math', 'description': "math:model=neurips/local,subject=prealgebra,level=1,use_official_examples=True", 'priority': 2},
+#     {'scenario': 'math', 'description': "math:model=neurips/local,subject=geometry,level=1,use_official_examples=True", 'priority': 2},
+#     {'scenario': 'math', 'description': "math:model=neurips/local,subject=counting_and_probability,level=1,use_official_examples=True", 'priority': 2},
+#     {'scenario': 'math', 'description': "math:model=neurips/local,subject=precalculus,level=1,use_official_examples=True", 'priority': 2},
 # 
 #     {'scenario': 'math', 'description': "math:model=neurips/local,subject=number_theory,level=2,use_official_examples=True", 'priority': 4},
 #     {'scenario': 'math', 'description': "math:model=neurips/local,subject=intermediate_algebra,level=2,use_official_examples=True", 'priority': 4},
@@ -44,13 +44,13 @@
 #     {'scenario': 'math', 'description': "math:model=neurips/local,subject=counting_and_probability,level=4,use_official_examples=True", 'priority': 4},
 #    {'scenario': 'math', 'description': "math:model=neurips/local,subject=precalculus,level=4,use_official_examples=True", 'priority': 4},
 
-    {'scenario': 'math', 'description': "math:model=neurips/local,subject=number_theory,level=5,use_official_examples=True", 'priority': 2},
-    {'scenario': 'math', 'description': "math:model=neurips/local,subject=intermediate_algebra,level=5,use_official_examples=True", 'priority': 2},
-    {'scenario': 'math', 'description': "math:model=neurips/local,subject=algebra,level=5,use_official_examples=True", 'priority': 2},
-    {'scenario': 'math', 'description': "math:model=neurips/local,subject=prealgebra,level=5,use_official_examples=True", 'priority': 2},
-    {'scenario': 'math', 'description': "math:model=neurips/local,subject=geometry,level=5,use_official_examples=True", 'priority': 2},
-    {'scenario': 'math', 'description': "math:model=neurips/local,subject=counting_and_probability,level=5,use_official_examples=True", 'priority': 2},
-    {'scenario': 'math', 'description': "math:model=neurips/local,subject=precalculus,level=5,use_official_examples=True", 'priority': 2},
+#    {'scenario': 'math', 'description': "math:model=neurips/local,subject=number_theory,level=5,use_official_examples=True", 'priority': 2},
+#    {'scenario': 'math', 'description': "math:model=neurips/local,subject=intermediate_algebra,level=5,use_official_examples=True", 'priority': 2},
+#    {'scenario': 'math', 'description': "math:model=neurips/local,subject=algebra,level=5,use_official_examples=True", 'priority': 2},
+#    {'scenario': 'math', 'description': "math:model=neurips/local,subject=prealgebra,level=5,use_official_examples=True", 'priority': 2},
+#    {'scenario': 'math', 'description': "math:model=neurips/local,subject=geometry,level=5,use_official_examples=True", 'priority': 2},
+#    {'scenario': 'math', 'description': "math:model=neurips/local,subject=counting_and_probability,level=5,use_official_examples=True", 'priority': 2},
+#    {'scenario': 'math', 'description': "math:model=neurips/local,subject=precalculus,level=5,use_official_examples=True", 'priority': 2},
 
     # With chain-of-thought prompting:
      {'scenario': 'math', 'description': "math:model=neurips/local,subject=number_theory,level=1,use_chain_of_thought=True", 'priority': 2},
@@ -133,7 +133,7 @@ def generate_equal_sum_list(V, N):
         v = scenario_n_examples_dict[cur_scenario].pop()
         entries[i]['max_eval_instances'] = v
 
-    with open(f'./run_specs_full_closed_eval_coarse_{args.example_budget}_budget.conf','w') as f:
+    with open(f'./run_specs_closed_{args.example_budget}_budget.conf','w') as f:
         f.write('entries: [\n')
         last_scenario = ''
         for entry in entries:
@@ -148,4 +148,4 @@ def generate_equal_sum_list(V, N):
             f.write('}\n')
         f.write(']')
 
-    print(f'Saved ./run_secret_specs_full_coarse_{args.example_budget}_budget.conf')
+    print(f'Saved ./run_secret_specs_closed_{args.example_budget}_budget.conf')

run_specs_full_closed_eval_coarse_100_budget.conf → configs/run_specs_closed_100_budget.conf

@@ -14,20 +14,6 @@ entries: [
 {description: "ethics_utilitarianism:model=neurips/local,data_augmentation=canonical,max_eval_instances=4",priority: 1}
 
 # math
-{description: "math:model=neurips/local,subject=number_theory,level=1,use_official_examples=True,max_eval_instances=0",priority: 1}
-{description: "math:model=neurips/local,subject=intermediate_algebra,level=1,use_official_examples=True,max_eval_instances=0",priority: 1}
-{description: "math:model=neurips/local,subject=algebra,level=1,use_official_examples=True,max_eval_instances=0",priority: 1}
-{description: "math:model=neurips/local,subject=prealgebra,level=1,use_official_examples=True,max_eval_instances=0",priority: 1}
-{description: "math:model=neurips/local,subject=geometry,level=1,use_official_examples=True,max_eval_instances=0",priority: 1}
-{description: "math:model=neurips/local,subject=counting_and_probability,level=1,use_official_examples=True,max_eval_instances=0",priority: 1}
-{description: "math:model=neurips/local,subject=precalculus,level=1,use_official_examples=True,max_eval_instances=0",priority: 1}
-{description: "math:model=neurips/local,subject=number_theory,level=5,use_official_examples=True,max_eval_instances=0",priority: 1}
-{description: "math:model=neurips/local,subject=intermediate_algebra,level=5,use_official_examples=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=algebra,level=5,use_official_examples=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=prealgebra,level=5,use_official_examples=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=geometry,level=5,use_official_examples=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=counting_and_probability,level=5,use_official_examples=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=precalculus,level=5,use_official_examples=True,max_eval_instances=1",priority: 1}
 {description: "math:model=neurips/local,subject=number_theory,level=1,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
 {description: "math:model=neurips/local,subject=intermediate_algebra,level=1,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
 {description: "math:model=neurips/local,subject=algebra,level=1,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
@@ -36,12 +22,12 @@ entries: [
 {description: "math:model=neurips/local,subject=counting_and_probability,level=1,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
 {description: "math:model=neurips/local,subject=precalculus,level=1,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
 {description: "math:model=neurips/local,subject=number_theory,level=5,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=intermediate_algebra,level=5,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=algebra,level=5,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=prealgebra,level=5,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=geometry,level=5,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=counting_and_probability,level=5,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
-{description: "math:model=neurips/local,subject=precalculus,level=5,use_chain_of_thought=True,max_eval_instances=1",priority: 1}
+{description: "math:model=neurips/local,subject=intermediate_algebra,level=5,use_chain_of_thought=True,max_eval_instances=2",priority: 1}
+{description: "math:model=neurips/local,subject=algebra,level=5,use_chain_of_thought=True,max_eval_instances=2",priority: 1}
+{description: "math:model=neurips/local,subject=prealgebra,level=5,use_chain_of_thought=True,max_eval_instances=2",priority: 1}
+{description: "math:model=neurips/local,subject=geometry,level=5,use_chain_of_thought=True,max_eval_instances=2",priority: 1}
+{description: "math:model=neurips/local,subject=counting_and_probability,level=5,use_chain_of_thought=True,max_eval_instances=2",priority: 1}
+{description: "math:model=neurips/local,subject=precalculus,level=5,use_chain_of_thought=True,max_eval_instances=2",priority: 1}
 
 # cnn
 {description: "summarization_cnndm:model=neurips/local,max_eval_instances=20",priority: 1}

configs/run_specs_closed_5000_budget.conf

@@ -0,0 +1,34 @@
+entries: [
+
+# summarization
+{description: "sam_sum:model=neurips/local,max_eval_instances=1000",priority: 1}
+
+# causation
+{description: "corr2cause:model=neurips/local,max_train_instances=1,max_eval_instances=1000",priority: 1}
+
+# ethics
+{description: "ethics_justice:model=neurips/local,data_augmentation=canonical,max_eval_instances=200",priority: 1}
+{description: "ethics_commonsense:model=neurips/local,data_augmentation=canonical,max_eval_instances=200",priority: 1}
+{description: "ethics_virtue:model=neurips/local,data_augmentation=canonical,max_eval_instances=200",priority: 1}
+{description: "ethics_deontology:model=neurips/local,data_augmentation=canonical,max_eval_instances=200",priority: 1}
+{description: "ethics_utilitarianism:model=neurips/local,data_augmentation=canonical,max_eval_instances=200",priority: 1}
+
+# math
+{description: "math:model=neurips/local,subject=number_theory,level=1,use_chain_of_thought=True,max_eval_instances=71",priority: 1}
+{description: "math:model=neurips/local,subject=intermediate_algebra,level=1,use_chain_of_thought=True,max_eval_instances=71",priority: 1}
+{description: "math:model=neurips/local,subject=algebra,level=1,use_chain_of_thought=True,max_eval_instances=71",priority: 1}
+{description: "math:model=neurips/local,subject=prealgebra,level=1,use_chain_of_thought=True,max_eval_instances=71",priority: 1}
+{description: "math:model=neurips/local,subject=geometry,level=1,use_chain_of_thought=True,max_eval_instances=71",priority: 1}
+{description: "math:model=neurips/local,subject=counting_and_probability,level=1,use_chain_of_thought=True,max_eval_instances=71",priority: 1}
+{description: "math:model=neurips/local,subject=precalculus,level=1,use_chain_of_thought=True,max_eval_instances=71",priority: 1}
+{description: "math:model=neurips/local,subject=number_theory,level=5,use_chain_of_thought=True,max_eval_instances=71",priority: 1}
+{description: "math:model=neurips/local,subject=intermediate_algebra,level=5,use_chain_of_thought=True,max_eval_instances=72",priority: 1}
+{description: "math:model=neurips/local,subject=algebra,level=5,use_chain_of_thought=True,max_eval_instances=72",priority: 1}
+{description: "math:model=neurips/local,subject=prealgebra,level=5,use_chain_of_thought=True,max_eval_instances=72",priority: 1}
+{description: "math:model=neurips/local,subject=geometry,level=5,use_chain_of_thought=True,max_eval_instances=72",priority: 1}
+{description: "math:model=neurips/local,subject=counting_and_probability,level=5,use_chain_of_thought=True,max_eval_instances=72",priority: 1}
+{description: "math:model=neurips/local,subject=precalculus,level=5,use_chain_of_thought=True,max_eval_instances=72",priority: 1}
+
+# cnn
+{description: "summarization_cnndm:model=neurips/local,max_eval_instances=1000",priority: 1}
+]