Last Dev Release into Main

.github/workflows/tests.yml

@@ -19,10 +19,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v3
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Install packages
         run: |

EventStream/baseline/FT_task_baseline.py

@@ -16,6 +16,7 @@
 import polars.selectors as cs
 import wandb
 from hydra.core.config_store import ConfigStore
+from loguru import logger
 from omegaconf import OmegaConf
 from sklearn.decomposition import NMF, PCA
 from sklearn.ensemble import RandomForestClassifier
@@ -35,7 +36,7 @@
 from ..tasks.profile import add_tasks_from
 from ..utils import task_wrapper
 
-pl.enable_string_cache(True)
+pl.enable_string_cache()
 
 
 def load_flat_rep(
@@ -187,6 +188,7 @@ def load_flat_rep(
                     if do_cache_filtered_task:
                         cached_fp.parent.mkdir(exist_ok=True, parents=True)
                         df.collect().write_parquet(cached_fp, use_pyarrow=True)
+                        df = pl.scan_parquet(cached_fp).select("subject_id", "timestamp", *window_features)
 
                 df = df.select("subject_id", "timestamp", *window_features)
                 if subjects_included.get(sp, None) is not None:
@@ -649,7 +651,7 @@ def eval_binary_classification(Y: np.ndarray, probs: np.ndarray) -> dict[str, fl
 
 
 def train_sklearn_pipeline(cfg: SklearnConfig):
-    print(f"Saving config to {cfg.save_dir / 'config.yaml'}")
+    logger.info(f"Saving config to {cfg.save_dir / 'config.yaml'}")
     cfg.save_dir.mkdir(exist_ok=True, parents=True)
     OmegaConf.save(cfg, cfg.save_dir / "config.yaml")
 
@@ -674,7 +676,7 @@ def train_sklearn_pipeline(cfg: SklearnConfig):
 
     # TODO(mmd): Window sizes may violate start_time constraints in task dfs!
 
-    print(f"Loading representations for {', '.join(cfg.feature_selector.window_sizes)}")
+    logger.info(f"Loading representations for {', '.join(cfg.feature_selector.window_sizes)}")
     subjects_included = {}
 
     if cfg.train_subset_size not in (None, "FULL"):
@@ -706,24 +708,26 @@ def train_sklearn_pipeline(cfg: SklearnConfig):
     Xs_and_Ys = {}
     for split in ("train", "tuning", "held_out"):
         st = datetime.now()
-        print(f"Loading dataset for {split}")
+        logger.info(f"Loading dataset for {split}")
         df = flat_reps[split].with_columns(normalized_label.alias(cfg.finetuning_task_label)).collect()
 
         X = df.drop(["subject_id", "timestamp", cfg.finetuning_task_label])
         Y = df[cfg.finetuning_task_label].to_numpy()
-        print(f"Done with {split} dataset with X of shape {X.shape} " f"(elapsed: {datetime.now() - st})")
+        logger.info(
+            f"Done with {split} dataset with X of shape {X.shape} " f"(elapsed: {datetime.now() - st})"
+        )
         Xs_and_Ys[split] = (X, Y)
 
-    print("Initializing model!")
+    logger.info("Initializing model!")
     model = cfg.get_model(dataset=ESD)
 
-    print("Fitting model!")
+    logger.info("Fitting model!")
     model.fit(*Xs_and_Ys["train"])
-    print(f"Saving model to {cfg.save_dir}")
+    logger.info(f"Saving model to {cfg.save_dir}")
     with open(cfg.save_dir / "model.pkl", mode="wb") as f:
         pickle.dump(model, f)
 
-    print("Evaluating model!")
+    logger.info("Evaluating model!")
     all_metrics = {}
     for split in ("tuning", "held_out"):
         X, Y = Xs_and_Ys[split]

EventStream/data/README.md

@@ -76,8 +76,8 @@ the following data:
    indices of the measures that correspond to the measurement observations in `dynamic_indices`.
 8. `dynamic_values`, which is of the same (ragged) shape as `dynamic_indices` and contains any unique
    numerical values associated with those measurements. Items may be missing (reflected with `None` or
-   `np.NaN`, depending on the data library format) or may have been filtered out as outliers (reflected with
-   `np.NaN`).
+   `float('nan')`, depending on the data library format) or may have been filtered out as outliers (reflected with
+   `float('nan')`).
 
 ### Measurements
 
@@ -390,7 +390,7 @@ Let us define the following variables:
 }
 ```
 
-`static_data_values` and `data_values` in the above dictionary may contain `np.NaN` entries where values were
+`static_data_values` and `data_values` in the above dictionary may contain `float('nan')` entries where values were
 not observed with a given data element. All other data elements are fully observed. The elements correspond to
 the following kinds of features: