110 nsam reviewers require experimented problems data

experiments/concurrent_execution/combine_numeric_precision_results.py

@@ -0,0 +1,41 @@
+import sys
+from pathlib import Path
+
+import pandas as pd
+import glob, re
+
+
+def combine_numeric_precision_results(input_directory: str, output_directory: Path) -> None:
+    INPUT_GLOB = f"{input_directory}/solving_combined_statistics_*_digit*.csv"
+    OUTPUT_CSV = output_directory / "numeric_precision_out.csv"
+    KEEP_ALGO = "numeric_sam"
+
+    paths = sorted(glob.glob(INPUT_GLOB))
+    dfs = []
+
+    for p in paths:
+        df = pd.read_csv(p)
+        m = re.search(r"_(\d+)_digits\.csv", p)
+        if "num_digits" not in df.columns and m:
+            df["num_digits"] = int(m.group(1))
+        if "learning_algorithm" in df.columns:
+            df = df[df["learning_algorithm"].astype(str) == KEEP_ALGO].copy()
+        dfs.append(df)
+
+    all_df = pd.concat(dfs, ignore_index=True)
+
+    out = (
+        all_df.groupby(["num_digits", "num_trajectories"], as_index=False)["percent_ok"]
+        .mean()
+        .sort_values(["num_digits", "num_trajectories"])
+    )
+
+    out = out.rename(columns={"percent_ok": "percent_solved"})
+    out = out[["num_trajectories", "percent_solved", "num_digits"]]
+    out.to_csv(OUTPUT_CSV, index=False)
+
+
+if __name__ == "__main__":
+    input_dir = sys.argv[1]
+    output_dir = Path(sys.argv[2])
+    combine_numeric_precision_results(input_dir, output_dir)

experiments/concurrent_execution/numeric_distributed_results_collector.py

@@ -1,13 +1,15 @@
 import argparse
+import csv
 import logging
 from pathlib import Path
 from typing import List
 
 from pddl_plus_parser.lisp_parsers import DomainParser
 
-from experiments.concurrent_execution.distributed_results_collector import DistributedResultsCollector
+from experiments.concurrent_execution.distributed_results_collector import DistributedResultsCollector, FOLD_FIELD
 from experiments.plotting.plot_nsam_results import plot_results
 from experiments.plotting.plot_nsam_solo_results import plot_solo_results
+from statistics.trajectories_statistics import TRAJECTORY_STATS_COLUMNS
 
 
 def parse_arguments() -> argparse.Namespace:
@@ -34,6 +36,24 @@ def __init__(
     ):
         super().__init__(working_directory_path, domain_file_name, learning_algorithms, num_folds, iterations)
 
+    def collect_trajectories_statistics(self) -> None:
+        """Collects the trajectories statistics from the results directory."""
+        results_directory = self.working_directory_path / "results_directory"
+        max_iteration = max(self.iterations) if self.iterations else 0
+        combined_statistics_data = []
+        for fold_index in range(self.num_folds):
+            trajectory_stats_path = results_directory / f"trajectories_statistics_{fold_index}_{max_iteration}.csv"
+            with open(trajectory_stats_path, "r") as trajectory_stats_file:
+                self.logger.info(f"Trajectory statistics for fold {fold_index} at iteration {max_iteration}")
+                reader = csv.DictReader(trajectory_stats_file)
+                combined_statistics_data.extend([{FOLD_FIELD: fold_index, **row} for row in reader])
+
+        combined_statistics_path = results_directory / f"combined_trajectories_statistics.csv"
+        with open(combined_statistics_path, "wt") as combined_statistics_file:
+            writer = csv.DictWriter(combined_statistics_file, fieldnames=[FOLD_FIELD, *TRAJECTORY_STATS_COLUMNS])
+            writer.writeheader()
+            writer.writerows(combined_statistics_data)
+
     def collect_numeric_statistics(self) -> None:
         """Collects the statistics from the results directory."""
         self._collect_solving_statistics(collecting_triplets=False)
@@ -43,6 +63,7 @@ def collect_numeric_statistics(self) -> None:
         domain = DomainParser(self.working_directory_path / self.domain_file_name).parse_domain()
         self._collect_syntactic_performance_statistics(domain)
         self._collect_numeric_semantic_performance_statistics(collecting_triplets=False)
+        self.collect_trajectories_statistics()
 
     def collect_numeric_statistics_with_triplets_statistics(self) -> None:
         self._collect_solving_statistics(collecting_triplets=True)

experiments/concurrent_execution/parallel_basic_experiment_runner.py

@@ -13,6 +13,7 @@
 
 from experiments.experiments_consts import MAX_SIZE_MB, DEFAULT_NUMERIC_TOLERANCE, NUMERIC_ALGORITHMS
 from statistics.learning_statistics_manager import LearningStatisticsManager
+from statistics.trajectories_statistics import compute_trajectory_statistics, export_trajectory_statistics
 from statistics.utils import init_semantic_performance_calculator
 from utilities import LearningAlgorithmType, SolverType, NegativePreconditionPolicy
 from validators import DomainValidator
@@ -197,25 +198,34 @@ def read_domain_file(self, train_set_dir_path: Path) -> Domain:
         return DomainParser(domain_path=partial_domain_path, partial_parsing=True).parse_domain()
 
     def collect_observations(
-        self, train_set_dir_path: Path, partial_domain: Domain
+        self, train_set_dir_path: Path, partial_domain: Domain, fold: int
     ) -> Union[List[Observation], List[MultiAgentObservation]]:
         """Collects all the observations from the trajectories in the train set directory.
 
         :param train_set_dir_path: the path to the directory containing the trajectories.
         :param partial_domain: the partial domain without the actions' preconditions and effects.
+        :param fold: the index of the current fold.
         :return: the allowed observations.
         """
         allowed_observations = []
+        used_problems = []
         sorted_trajectory_paths = sorted(train_set_dir_path.glob("*.trajectory"))  # for consistency
         for index, trajectory_file_path in enumerate(sorted_trajectory_paths):
             # assuming that the folders were created so that each folder contains only the correct number of trajectories, i.e., iteration_number
             problem_path = train_set_dir_path / f"{trajectory_file_path.stem}.pddl"
             problem = ProblemParser(problem_path, partial_domain).parse_problem()
-            complete_observation = TrajectoryParser(partial_domain, problem).parse_trajectory(
+            used_problems.append(problem)
+            complete_observation = TrajectoryParser(partial_domain).parse_trajectory(
                 trajectory_file_path, executing_agents=self.executing_agents
             )
             allowed_observations.append(complete_observation)
 
+        self.logger.info("Exporting trajectories statistics.")
+        trajectories_stats = compute_trajectory_statistics(allowed_observations, used_problems, domain=partial_domain)
+        export_trajectory_statistics(
+            trajectories_stats,
+            self.working_directory_path / "results_directory" / f"trajectories_statistics_{fold}_{len(allowed_observations)}.csv",
+        )
         return allowed_observations
 
     def learn_model_offline(
@@ -236,7 +246,7 @@ def learn_model_offline(
         self.logger.info(f"Starting the learning phase for the fold - {fold_num}!")
         partial_domain_path = train_set_dir_path / self.domain_file_name
         partial_domain = DomainParser(domain_path=partial_domain_path, partial_parsing=True).parse_domain()
-        allowed_observations = self.collect_observations(train_set_dir_path, partial_domain)
+        allowed_observations = self.collect_observations(train_set_dir_path, partial_domain, fold_num)
         self.logger.info(f"Learning the action model using {len(allowed_observations)} trajectories!")
         learned_model, learning_report = self._apply_learning_algorithm(partial_domain, allowed_observations, test_set_dir_path)
         self.learning_statistics_manager.add_to_action_stats(

experiments/concurrent_execution/parallel_numeric_experiment_runner.py

@@ -6,6 +6,7 @@
 import shutil
 import signal
 import subprocess
+import threading
 import time
 import uuid
 from pathlib import Path
@@ -127,6 +128,15 @@ def _create_learned_domain_for_evaluation(self, plan_miner_output_domain_path: P
             domain_file.write(plan_miner_domain.to_pddl())
             return plan_miner_domain
 
+    def _log_plan_miner_output(self, process: subprocess.Popen) -> None:
+        """Logs the output of the Plan-Miner process.
+
+        :param process: the Plan-Miner process.
+        """
+        assert process.stdout is not None
+        for line in process.stdout:
+            self.logger.info(f"Plan-Miner: {line.strip()}")
+
     def _run_plan_miner_process(self, plan_miner_trajectory_file_path: Path) -> Optional[Path]:
         """Runs the Plan-Miner process to learn the action model.
 
@@ -136,7 +146,17 @@ def _run_plan_miner_process(self, plan_miner_trajectory_file_path: Path) -> Opti
         plan_miner_domain_name = f"{self.domain_file_name.split('.')[0]}_{uuid.uuid4()}"
         os.chdir(PLAN_MINER_DIR_PATH)
         # cmd = ./PlanMiner {path to pts file} {domain name without extension}
-        process = subprocess.Popen(f"./bin/PlanMiner {plan_miner_trajectory_file_path} {plan_miner_domain_name}", shell=True)
+        process = subprocess.Popen(
+            f"./bin/PlanMiner {plan_miner_trajectory_file_path} {plan_miner_domain_name}",
+            shell=True,
+            stdout=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+        )
+        logging_lambda = lambda: self._log_plan_miner_output(process)
+        logging_thread = threading.Thread(target=logging_lambda, daemon=True)
+        logging_thread.start()
+
         try:
             process.wait(timeout=LEARNING_TIMEOUT)
 

experiments/experiments_consts.py

@@ -1,4 +1,5 @@
 from sam_learning.learners import NumericSAMLearner, IncrementalNumericSAMLearner
+from sam_learning.learners.baseline_learners.naive_numeric_sam import NaivePolynomialSAMLearning
 from utilities import LearningAlgorithmType
 
 DEFAULT_SPLIT = 5
@@ -19,4 +20,5 @@
 NUMERIC_SAM_ALGORITHM_VERSIONS = {
     LearningAlgorithmType.numeric_sam: NumericSAMLearner,
     LearningAlgorithmType.incremental_nsam: IncrementalNumericSAMLearner,
+    LearningAlgorithmType.naive_nsam: NaivePolynomialSAMLearning,
 }

experiments/plotting/plot_numeric_precision.py

@@ -0,0 +1,46 @@
+import sys
+from pathlib import Path
+
+import pandas as pd
+import matplotlib.pyplot as plt
+
+
+# Load data
+def plot_numeric_precision(input_path: Path, output_folder_path: Path) -> None:
+    df = pd.read_csv(input_path)
+
+    plt.figure(figsize=(12, 8))
+
+    digits = sorted(df["num_digits"].unique())
+    linestyles = ["solid", "dashed", "dotted", "dashdot"]
+    colors = ["#0072B2", "#D55E00", "#009E73", "#CC79A7", "#F0E442", "#56B4E9"]
+
+    for i, d in enumerate(digits):
+        subset = df[df["num_digits"] == d].sort_values("num_trajectories")
+        plt.plot(
+            subset["num_trajectories"],
+            subset["percent_solved"],
+            label=f"{int(d)} digits",
+            linewidth=8,
+            linestyle=linestyles[i % len(linestyles)],
+            color=colors[i % len(colors)],
+        )
+
+    plt.xlabel("# Trajectories", fontsize=44)
+    plt.ylabel("AVG % of solved", fontsize=44)
+    plt.ylim(0, 100)
+    plt.xticks(fontsize=44)
+    plt.yticks(fontsize=44)
+    plt.legend(fontsize=40)
+    plt.grid(True)
+    plt.tight_layout()
+
+    output_path = output_folder_path / "numeric_precision.pdf"
+    plt.savefig(output_path, bbox_inches="tight", dpi=300)
+    plt.close()
+
+
+if __name__ == "__main__":
+    input_path = Path(sys.argv[1])
+    output_folder_path = Path(sys.argv[2])
+    plot_numeric_precision(input_path, output_folder_path)

requirements.txt

@@ -8,7 +8,7 @@ scikit-learn==1.5.0
 networkx==3.1
 jdk4py==17.0.7.0
 anytree~=2.8.0
-pddl-plus-parser==3.16.3
+pddl-plus-parser==3.16.5
 
 pandas==2.1.1
 scikit-obliquetree==0.1.4