Merge pull request #36 from KumarLabJax/select-feature-subsets

Adds the ability to control feature subsets for classifiers.
Changes ownership of project settings from UI components to project.
Classifiers make a copy of project settings.

Author: SkepticRaven

.gitignore

@@ -1,4 +1,5 @@
 .idea
+.coverage
 *.pyc
 test-reports
 .DS_Store

classify.py

@@ -41,23 +41,16 @@ def train_and_classify(
         override_classifier: typing.Optional[ClassifierType] = None,
         fps=DEFAULT_FPS,
         feature_dir: typing.Optional[str] = None):
-    try:
-        training_file, _ = load_training_data(training_file_path)
-    except OSError as e:
-        sys.exit(f"Unable to open training data\n{e}")
-
-    behavior = training_file['behavior']
-    window_size = training_file['window_size']
-    use_social = training_file['has_social_features']
+    if not training_file_path.exists():
+        sys.exit(f"Unable to open training data\n")
 
     classifier = train(training_file_path, override_classifier)
-    classify_pose(classifier, input_pose_file, out_dir, behavior, window_size,
-                  use_social, fps, feature_dir)
+    classify_pose(classifier, input_pose_file, out_dir, behavior, fps, feature_dir)
 
 
 def classify_pose(classifier: Classifier, input_pose_file: Path, out_dir: Path,
-                  behavior: str, window_size: int, use_social: bool,
-                  fps=DEFAULT_FPS, feature_dir: typing.Optional[str] = None):
+                  behavior: str, fps=DEFAULT_FPS,
+                  feature_dir: typing.Optional[str] = None):
     pose_est = open_pose_file(input_pose_file)
     pose_stem = get_pose_stem(input_pose_file)
 
@@ -67,36 +60,7 @@ def classify_pose(classifier: Classifier, input_pose_file: Path, out_dir: Path,
         dtype=np.int8)
     prediction_prob = np.zeros_like(prediction_labels, dtype=np.float32)
 
-    if use_social and pose_est.format_major_version < 3:
-        print(f"Skipping {input_pose_file}")
-        print("  classifier requires v3 or higher pose files")
-        return
-
-    # make sure the pose file supports all required extended features
-    supported_features = IdentityFeatures.get_available_extended_features(
-        pose_est.format_major_version, pose_est.static_objects)
-    required_features = classifier.extended_features
-    extended_feature_check_ok = True
-    for group, features in required_features.items():
-        if group not in supported_features:
-            extended_feature_check_ok = False
-        else:
-            for f in required_features[group]:
-                if f not in supported_features[group]:
-                    extended_feature_check_ok = False
-    if not extended_feature_check_ok:
-        print(f"Skipping {input_pose_file}")
-        print("  pose file does not support all required features")
-        return
-
-    distance_scale_factor = 1.0
-    if classifier.distance_unit == ProjectDistanceUnit.CM:
-        if pose_est.cm_per_pixel is None:
-            print(f"Skipping {input_pose_file}")
-            print("  classifier uses cm distance units but pose file does not have cm_per_pixel attribute")
-            return
-        else:
-            distance_scale_factor = pose_est.cm_per_pixel
+    classifier_settings = classifier.project_settings
 
     print(f"Classifying {input_pose_file}...")
 
@@ -106,12 +70,10 @@ def classify_pose(classifier: Classifier, input_pose_file: Path, out_dir: Path,
                          complete_as_percent=False, suffix='identities')
 
         features = IdentityFeatures(
-            input_pose_file, curr_id, feature_dir, pose_est, fps=fps,
-            distance_scale_factor=distance_scale_factor,
-            extended_features=classifier.extended_features
-        ).get_features(window_size, use_social)
-        per_frame_features = pd.DataFrame(IdentityFeatures.merge_per_frame_features(features['per_frame'], use_social))
-        window_features = pd.DataFrame(IdentityFeatures.merge_window_features(features['window'], use_social))
+            input_pose_file, curr_id, feature_dir, pose_est, fps=fps, op_settings=classifier_settings
+        ).get_features(classifier_settings['window_size'])
+        per_frame_features = pd.DataFrame(IdentityFeatures.merge_per_frame_features(features['per_frame']))
+        window_features = pd.DataFrame(IdentityFeatures.merge_window_features(features['window']))
 
         data = Classifier.combine_data(per_frame_features, window_features)
 
@@ -154,18 +116,21 @@ def train(
 ) -> Classifier:
 
     try:
-        training_file, _ = load_training_data(training_file)
+        loaded_training_data, _ = load_training_data(training_file)
     except OSError as e:
         sys.exit(f"Unable to open training data\n{e}")
 
-    behavior = training_file['behavior']
+    behavior = loaded_training_data['behavior']
 
     classifier = Classifier()
+    classifier.set_dict_settings(loaded_training_data['settings'])
+
+    # Override the classifier type
     if override_classifier is not None:
         classifier_type = override_classifier
     else:
         classifier_type = ClassifierType(
-            training_file['classifier_type'])
+            loaded_training_data['classifier_type'])
 
     if classifier_type in classifier.classifier_choices():
         classifier.set_classifier(classifier_type)
@@ -177,27 +142,21 @@ def train(
     print("Training classifier for:", behavior)
     print("  Classifier Type: "
           f"{__CLASSIFIER_CHOICES[classifier.classifier_type]}")
-    print(f"  Window Size: {training_file['window_size']}")
-    print(f"  Social: {training_file['has_social_features']}")
-    print(f"  Balanced Labels: {training_file['balance_labels']}")
-    print(f"  Symmetric Behavior: {training_file['symmetric']}")
-    print(f"  Distance Unit: {training_file['distance_unit'].name}")
-
-    training_features = classifier.combine_data(training_file['per_frame'],
-                                                training_file['window'])
+    print(f"  Window Size: {loaded_training_data['settings']['window_size']}")
+    print(f"  Social: {loaded_training_data['settings']['social']}")
+    print(f"  Balanced Labels: {loaded_training_data['settings']['balance_labels']}")
+    print(f"  Symmetric Behavior: {loaded_training_data['settings']['symmetric_behavior']}")
+    print(f"  CM Units: {loaded_training_data['settings']['cm_units']}")
+
+    training_features = classifier.combine_data(loaded_training_data['per_frame'],
+                                                loaded_training_data['window'])
     classifier.train(
         {
             'training_data': training_features,
-            'training_labels': training_file['labels']
+            'training_labels': loaded_training_data['labels']
         },
         behavior,
-        training_file['window_size'],
-        training_file['has_social_features'],
-        training_file['balance_labels'],
-        training_file['symmetric'],
-        training_file['extended_features'],
-        training_file['distance_unit'],
-        random_seed=training_file['training_seed']
+        random_seed=loaded_training_data['training_seed']
     )
 
     return classifier
@@ -296,9 +255,7 @@ def classify_main():
             sys.exit(e)
 
         behavior = classifier.behavior_name
-        window_size = classifier.window_size
-        use_social = classifier.uses_social
-        distance_unit = classifier.distance_unit
+        classifier_settings = classifier.project_settings
 
         print(f"Classifying using trained classifier: {args.classifier}")
         try:
@@ -307,12 +264,11 @@ def classify_main():
         except KeyError:
             sys.exit("Error: Classifier type not supported on this platform")
         print(f"  Behavior: {behavior}")
-        print(f"  Window Size: {window_size}")
-        print(f"  Social: {use_social}")
-        print(f"  Distance Unit: {distance_unit.name}")
+        print(f"  Window Size: {classifier_settings['window_size']}")
+        print(f"  Social: {classifier_settings['social']}")
+        print(f"  CM Units: {classifier_settings['cm_units']}")
 
-        classify_pose(classifier, in_pose_path, out_dir, behavior, window_size,
-                      use_social, fps=args.fps, feature_dir=args.feature_dir)
+        classify_pose(classifier, in_pose_path, out_dir, behavior, fps=args.fps, feature_dir=args.feature_dir)
 
 
 def train_main():

docs/user_guide/user_guide.md

@@ -4,32 +4,32 @@
 
 A JABS project is a directory of video files and their corresponding pose
 estimation files. The first time a project directory is opened in JABS, it will
-create a subdirectory called "rotta", which contains various files created by
+create a subdirectory called "jabs", which contains various files created by
 JABS to save project state, including labels and current predictions.
 
 ### Example JABS project directory listing:
 
 ```text
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_08-00-00_500.avi
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_08-00-00_500_pose_est_v3.h5
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_09-00-00_500.avi
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_09-00-00_500_pose_est_v3.h5
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_10-00-00_500.avi
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_10-00-00_500_pose_est_v3.h5
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_11-00-00_500.avi
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_11-00-00_500_pose_est_v3.h5
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_12-00-00_500.avi
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_12-00-00_500_pose_est_v3.h5
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_13-00-00_500.avi
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_13-00-00_500_pose_est_v3.h5
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_14-00-00_500.avi
-NV11-CBAX2+2019-07-26+MDX0009_2019-07-26_14-00-00_500_pose_est_v3.h5 rotta
+VIDEO_1.avi
+VIDEO_1_pose_est_v3.h5
+VIDEO_2.avi
+VIDEO_2_pose_est_v3.h5
+VIDEO_3.avi
+VIDEO_3_pose_est_v3.h5
+VIDEO_4.avi
+VIDEO_4_pose_est_v3.h5
+VIDEO_5.avi
+VIDEO_5_pose_est_v3.h5
+VIDEO_6.avi
+VIDEO_6_pose_est_v3.h5
+VIDEO_7.avi
+VIDEO_7_pose_est_v3.h5 jabs
 ```
 
-## Initializing  A JABS Project Directory
+## Initializing A JABS Project Directory
 
 The first time you open a project directory in with JABS it will create the "
-rotta" subdirectory. Features will be computed the first time the "Train" button
+jabs" subdirectory. Features will be computed the first time the "Train" button
 is clicked. This can be very time consuming depending on the number and length
 of videos in the project directory.
 
@@ -76,48 +76,48 @@ will use up to 4 processes.
 
 `./initialize_project.py -p8 -w2 -w5 -w10 <path/to/project/dir>`
 
-## The Rotta Directory
+## The JABS Directory
 
-JABS creates a subdirectory called "rotta" inside the project directory (this
-directory is called "rotta" for historical reasons and may change prior to the
-1.0.0 release of JABS). This directory contains app-specific data such as
-project settings, generated features, user labels, cache files, and the latest
-predictions.
+JABS creates a subdirectory called "jabs" inside the project directory. This
+directory contains app-specific data such as project settings, generated
+features, user labels, cache files, and the latest predictions.
 
 project.json This file contains project settings and metadata.
 
-### rotta/annotations
+### jabs/annotations
 
 This directory stores the user's labels, stored in one JSON file per labeled
 video.
 
-### rotta/archive
+### jabs/archive
 
 This directory contains archived labels. These are compressed files (gzip)
 containing labels for behaviors that the user has removed from the project.
 Rotta only archives labels. Trained classifiers and predictions are deleted if a
 user removes a behavior from a project.
 
-### rotta/cache
+### jabs/cache
 
 Files cached by JABS to speed up performance. Some of these files may not be
 portable, so this directory should be deleted if a JABS project is copied to a
 different platform.
 
-### rotta/classifiers
+### jabs/classifiers
 
 This directory contains trained classifiers. Currently, these are stored in
-Python Pickle files and should be considered non-portable.
+Python Pickle files and should be considered non-portable. While non-portable,
+these files can be used alongside `classify.py classify --classifier` for
+predicting on the same machine the gui running the training.
 
-### rotta/features
+### jabs/features
 
 This directory contains the computed features. There is one directory per
 project video, and within each video directory there will be one feature
-directory per identity. Feature files are usually portable, but JABS may need
+directory per identity. Feature files are portable, but JABS may need
 to recompute the features if they were created with a different version of
 JABS.
 
-### rotta/predictions
+### jabs/predictions
 
 This directory contains prediction files. There will be one subdirectory per
 behavior containing one prediction file per video. Prediction files are
@@ -173,9 +173,6 @@ tool (`classify.py`).
   means that 11 frames are included into the window feature calculations for
   each frame (5 previous frames, current frame, 5 following frames).
 - **New Window Size:** Add a new window size to the project.
-- **Social Feature Toggle:** Turn on/off social features (disabled if project
-  includes pose file version 2). Allows training a classifier backwards
-  compatible with V2 pose files using V3 or higher poses.
 - **Label Balancing Toggle:** Balances the training data by downsampling the class with more labels such that the distribution is equal.
 - **Symmetric Behavior Toggle:** Tells the classifier that the behavior is symmetric. A symmetric behavior is when left and right features are interchangeable.
 - **All k-fold Toggle:** Uses the maximum number of cross validation folds. Useful when you wish to compare classifier performance and may have an outlier that can be held-out.
@@ -222,6 +219,22 @@ tool (`classify.py`).
   mouse
 - **View→Overlay Landmarks:** toggle the overlay of arena landmarks over the
   video.
+- **Features:** Menu item for controlling per-behavior classifier settings.
+  Menu items are disabled when at least 1 pose file in the project does not
+  contain the data to calculate features.
+- **Features→CM Units:** toggle using CM or pixel units
+  (Warning! Changing this will require features to be re-calculated)
+- **Features→Enable Window Features:** toggle using statistical window features
+- **Features→Enable Signal Features:** toggle using fft-based window features
+- **Features→Enable Social Features:** toggle using social features (v3+ projects)
+- **Features→Enable Corners Features:** toggle using arena corner features
+  (v5+ projects with arena corner static object)
+- **Features→Enable Lixit Features:** toggle using lixit features
+  (v5+ projects with lixit static object)
+- **Features→Enable Food_hopper Features:** toggle using food hopper features
+  (v5+ projects with food hopper static object)
+- **Features→Enable Segmentation Features:** toggle using segmentation features
+  (v6+ projects)
 
 **Track Overlay Example:**  
 <img src="imgs/track_overlay.png" alt="Track Overlay" width=400 />
@@ -412,7 +425,7 @@ one video file.
 #### Location
 
 The prediction files are saved
-in `<JABS project dir>/rotta/predictions/<behavior_name>/<video_name>.h5` if
+in `<JABS project dir>/jabs/predictions/<behavior_name>/<video_name>.h5` if
 they were generated by the JABS GUI. The `classify.py` script saves inference
 files in `<out-dir>/<behavior_name>/<video_name>.h5`