Dev

.gitignore

@@ -23,3 +23,6 @@ Examples/reprocessDataServer.py
 *.stats
 
 newsletter.py
+
+.DS_Store
+.vscode/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/opencap-test-data"]
+	path = tests/opencap-test-data
+	url = https://github.com/stanfordnmbl/opencap-test-data.git

CHANGELOG.md

@@ -1,9 +1,16 @@
-This document lists the changes to `opencap-core`. When possible, we provide the date, GitHub issues, or pull requests that are related to the items below. If there is no issue or pull request related to the change, then we may provide the commit.
+This document lists the changes to `opencap-core` for each version. When possible, we provide the GitHub issues or pull requests that are related to the items below. If there is no issue or pull request related to the change, then we may provide the commit.
 
 This is not a comprehensive list of changes but rather a hand-curated collection of the more notable ones. For a comprehensive history, see the [OpenCap Core GitHub repo](https://github.com/stanfordnmbl/opencap-core).
 
-Changes
-=======
+v1.1
+=====
+- Improved synchronization with an arm raise (hand punch). (#182)
+- Added option for downloading pose pickle files. (#248)
+- Moved synchronization specific files to new module utilsSync.py. (PR #259)
+- Added main regression tests and sync unit tests. (PR #259)
+
+Previous Changes
+================
 - 07/03/2024: Speed up IK by removing patella constraints from model ([pull request](https://github.com/stanfordnmbl/opencap-core/pull/174))
 - 06/25/2024: Add support for "any pose" scaling ([pull request](https://github.com/stanfordnmbl/opencap-core/pull/168))
 - 05/10/2024: Add reprojection error minimization to improve camera synchronization  ([pull request](https://github.com/stanfordnmbl/opencap-core/pull/159))

Examples/batchDownloadData.py

@@ -54,5 +54,7 @@
 
 # Batch download. 
 for session_id in session_ids:
+    print(f'Downloading session id {session_id}')
     utils.downloadAndZipSession(session_id,justDownload=True,data_dir=downloadFolder,
-                          useSubjectNameFolder=useSubjectIdentifierAsFolderName)
+                          useSubjectNameFolder=useSubjectIdentifierAsFolderName,
+                          include_pose_pickles=False)

defaults.py

@@ -0,0 +1 @@
+DEFAULT_SYNC_VER = '1.0'

main.py

@@ -22,14 +22,15 @@
 from utilsChecker import calcExtrinsicsFromVideo
 from utilsChecker import isCheckerboardUpsideDown
 from utilsChecker import autoSelectExtrinsicSolution
-from utilsChecker import synchronizeVideos
 from utilsChecker import triangulateMultiviewVideo
 from utilsChecker import writeTRCfrom3DKeypoints
 from utilsChecker import popNeutralPoseImages
 from utilsChecker import rotateIntrinsics
+from utilsSync import synchronizeVideos
 from utilsDetector  import runPoseDetector
 from utilsAugmenter import augmentTRC
 from utilsOpenSim import runScaleTool, getScaleTimeRange, runIKTool, generateVisualizerJson
+from defaults import DEFAULT_SYNC_VER
 
 def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
          intrinsicsFinalFolder='Deployed', isDocker=False,
@@ -42,7 +43,7 @@ def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
          dataDir=None, overwriteAugmenterModel=False,
          filter_frequency='default', overwriteFilterFrequency=False,
          scaling_setup='upright_standing_pose', overwriteScalingSetup=False,
-         overwriteCamerasToUse=False):
+         overwriteCamerasToUse=False, syncVer=None,):
 
     # %% High-level settings.
     # Camera calibration.
@@ -131,6 +132,10 @@ def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
     else:
         camerasToUse = cameras_to_use
 
+    # We'll use syncVer if provided to this function. If not, try to use one 
+    # from sessionMetadata, otherwise use the default one.
+    syncVer = syncVer or sessionMetadata.get('sync_ver', DEFAULT_SYNC_VER)
+
     # %% Paths to pose detector folder for local testing.
     if poseDetector == 'OpenPose':
         poseDetectorDirectory = getOpenPoseDirectory(isDocker)
@@ -392,7 +397,8 @@ def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
                     filtFreqs=filtFreqs, confidenceThreshold=0.4,
                     imageBasedTracker=False, cams2Use=camerasToUse_c, 
                     poseDetector=poseDetector, trialName=trialName,
-                    resolutionPoseDetection=resolutionPoseDetection))
+                    resolutionPoseDetection=resolutionPoseDetection,
+                    syncVer=syncVer))
         except Exception as e:
             if len(e.args) == 2: # specific exception
                 raise Exception(e.args[0], e.args[1])

tests/test_api.py

@@ -1,78 +1,79 @@
 import logging
+import os
+import sys
 import pytest
-import requests
-from unittest.mock import patch, Mock, ANY
+from unittest.mock import patch, Mock
 from http.client import HTTPMessage
 
+thisDir = os.path.dirname(os.path.realpath(__file__))
+repoDir = os.path.abspath(os.path.join(thisDir,'../'))
+sys.path.append(repoDir)
 from utils import makeRequestWithRetry
 
-class TestMakeRequestWithRetry:
-    logging.getLogger('urllib3').setLevel(logging.DEBUG)
+logging.getLogger('urllib3').setLevel(logging.DEBUG)
 
-    @patch("requests.Session.request")
-    def test_get(self, mock_response):
-        status_code = 200
-        mock_response.return_value.status_code = status_code
+@patch("requests.Session.request")
+def test_get(mock_response):
+    status_code = 200
+    mock_response.return_value.status_code = status_code
 
-        response = makeRequestWithRetry('GET', 'https://test.com', retries=2)
-        assert response.status_code == status_code
-        mock_response.assert_called_once_with('GET', 'https://test.com',
-                                              headers=None,
-                                              data=None,
-                                              params=None,
-                                              files=None)
+    response = makeRequestWithRetry('GET', 'https://test.com', retries=2)
+    assert response.status_code == status_code
+    mock_response.assert_called_once_with('GET', 'https://test.com',
+                                          headers=None,
+                                          data=None,
+                                          params=None,
+                                          files=None)
 
-    @patch("requests.Session.request")
-    def test_put(self, mock_response):
-        status_code = 201
-        mock_response.return_value.status_code = status_code
-
-        data = {
-            "key1": "value1",
-            "key2": "value2"
-        }
+@patch("requests.Session.request")
+def test_put(mock_response):
+    status_code = 201
+    mock_response.return_value.status_code = status_code
 
-        params = {
-            "param1": "value1"
-        }
-
-        response = makeRequestWithRetry('POST', 
-                                        'https://test.com',
-                                        data=data,
-                                        headers={"Authorization": "my_token"},
-                                        params=params,
-                                        retries=2)
-
-        assert response.status_code == status_code
-        mock_response.assert_called_once_with('POST', 
-                                             'https://test.com',
-                                             data=data,
-                                             headers={"Authorization": "my_token"},
-                                             params=params,
-                                             files=None)
+    data = {
+        "key1": "value1",
+        "key2": "value2"
+    }
 
-    @patch("urllib3.connectionpool.HTTPConnectionPool._get_conn")
-    def test_success_after_retries(self, mock_response):
-        mock_response.return_value.getresponse.side_effect = [
-            Mock(status=500, msg=HTTPMessage()),
-            Mock(status=502, msg=HTTPMessage()),
-            Mock(status=200, msg=HTTPMessage()),
-            Mock(status=429, msg=HTTPMessage()),
-        ]
+    params = {
+        "param1": "value1"
+    }
 
-        response = makeRequestWithRetry('GET', 
-                                        'https://test.com', 
-                                        retries=5, 
-                                        backoff_factor=0.1)
+    response = makeRequestWithRetry('POST',
+                                    'https://test.com',
+                                    data=data,
+                                    headers={"Authorization": "my_token"},
+                                    params=params,
+                                    retries=2)
 
-        assert response.status_code == 200
-        assert mock_response.call_count == 3
+    assert response.status_code == status_code
+    mock_response.assert_called_once_with('POST',
+                                         'https://test.com',
+                                         data=data,
+                                         headers={"Authorization": "my_token"},
+                                         params=params,
+                                         files=None)
 
-    # comment out test since httpbin can be unstable and we don't want to rely
-    # on it for tests. uncomment and see debug log to see retry attempts
-    '''def test_httpbin(self):
-        response = makeRequestWithRetry('GET', 
-                                        'https://httpbin.org/status/500', 
-                                        retries=4, 
-                                        backoff_factor=0.1)
-    '''
+@patch("urllib3.connectionpool.HTTPConnectionPool._get_conn")
+def test_success_after_retries(mock_response):
+    mock_response.return_value.getresponse.side_effect = [
+        Mock(status=500, msg=HTTPMessage()),
+        Mock(status=502, msg=HTTPMessage()),
+        Mock(status=200, msg=HTTPMessage()),
+        Mock(status=429, msg=HTTPMessage()),
+    ]
+
+    response = makeRequestWithRetry('GET',
+                                    'https://test.com',
+                                    retries=5,
+                                    backoff_factor=0.1)
+
+    assert response.status_code == 200
+    assert mock_response.call_count == 3
+
+# The httpbin test remains commented out for stability reasons
+# def test_httpbin():
+#     response = makeRequestWithRetry('GET',
+#                                     'https://httpbin.org/status/500',
+#                                     retries=4,
+#                                     backoff_factor=0.1)

tests/test_main.py

@@ -0,0 +1,147 @@
+import logging
+import os
+import sys
+import numpy as np
+import pandas as pd
+import pytest
+
+thisDir = os.path.dirname(os.path.realpath(__file__))
+repoDir = os.path.abspath(os.path.join(thisDir, '../'))
+sys.path.append(repoDir)
+from main import main
+
+# Helper functions to load and compare TRC and MOT files
+def load_trc(file, num_metadata_lines=5):
+    with open(file, 'r') as f:
+        lines = f.readlines()
+        metadata = lines[:num_metadata_lines]
+    df = pd.read_csv(file, sep='\t', skiprows=num_metadata_lines + 1, header=None)
+    return df, metadata
+
+def load_mot(file, num_metadata_lines=10):
+    with open(file, 'r') as f:
+        lines = f.readlines()
+        metadata = lines[:num_metadata_lines]
+    df = pd.read_csv(file, sep='\t', skiprows=num_metadata_lines)
+    return df, metadata
+
+
+def calc_rmse(series1, series2):
+    return np.sqrt(((series1 - series2) ** 2).mean())
+
+def compare_mot(output_mot_df, ref_mot_df, t0, tf):
+    '''Function to compare MOT dataframes within a time range [t0, tf].
+    We use the specific time range to analyze the range with the motion
+    of interest. In particular, the arm raise can create larger differences
+    on single frames.
+
+    - Time column is checked for equality (IK is frame-by-frame).
+    - Translation error is checked within 2 mm max per frame, RMSE within 
+      1 mm.
+    - Rotation error for wrist pronation/supination (coordinates pro_sup_r
+      and pro_sup_l) are checked within 5.0 degrees max per frame, RMSE
+      within 1.0 degrees.
+    - Rotation error for all other coordinates are tighter and checked 
+      within 2.5 degrees max per frame, RMSE within 0.5 degrees.
+    '''
+    output_mot_df_slice = output_mot_df[(output_mot_df['time'] >= t0) & (output_mot_df['time'] <= tf)]
+    ref_mot_df_slice = ref_mot_df[(ref_mot_df['time'] >= t0) & (ref_mot_df['time'] <= tf)]
+    for col in ref_mot_df.columns:
+        # time column should be equal since IK is frame-by-frame
+        if col == 'time':
+            pd.testing.assert_series_equal(output_mot_df[col], ref_mot_df[col])
+
+        # check translational within 2 mm max error, rmse within 1 mm
+        elif any(substr in col for substr in ['tx', 'ty', 'tz']):
+            pd.testing.assert_series_equal(
+                output_mot_df_slice[col], ref_mot_df_slice[col], atol=0.002
+            )
+            rmse = calc_rmse(output_mot_df_slice[col], ref_mot_df_slice[col])
+            assert rmse <= 0.001
+
+        elif 'pro_sup' in col:
+            pd.testing.assert_series_equal(
+                output_mot_df_slice[col], ref_mot_df_slice[col], atol=5.0
+            )
+            rmse = calc_rmse(output_mot_df_slice[col], ref_mot_df_slice[col])
+            assert rmse <= 1.0
+
+        # check rotational within 2.5 degrees max error, rmse within 0.5 degrees
+        else:
+            pd.testing.assert_series_equal(
+                output_mot_df_slice[col], ref_mot_df_slice[col], atol=2.5
+            )
+            rmse = calc_rmse(output_mot_df_slice[col], ref_mot_df_slice[col])
+            assert rmse <= 0.5
+
+# End to end tests with different sync methods (hand, gait, general).
+# Also check that syncVer updates with main changes.
+# Note: no pose detection, uses pre-scaled opensim model
+@pytest.mark.parametrize("syncVer", ['1.0', '1.1'])
+@pytest.mark.parametrize("trialName, t0, tf", [
+    ('squats-with-arm-raise', 5.0, 10.0),
+    ('squats', 3.0, 8.0),
+    ('walk', 1.0, 5.0),
+])
+def test_main(trialName, t0, tf, syncVer, caplog):
+    caplog.set_level(logging.INFO)
+
+    sessionName = 'sync_2-cameras'
+    trialID = trialName
+    dataDir = os.path.join(thisDir, 'opencap-test-data')
+    main(
+        sessionName,
+        trialName,
+        trialID,
+        dataDir=dataDir,
+        genericFolderNames=True,
+        poseDetector='hrnet',
+        syncVer=syncVer,
+    )
+    assert f"Synchronizing Keypoints using version {syncVer}" in caplog.text
+
+    # Compare marker data
+    output_trc = os.path.join(dataDir,
+        'Data',
+        sessionName,
+        'MarkerData',
+        'PostAugmentation',
+        f'{trialName}.trc',
+    )
+    ref_trc = os.path.join(
+        dataDir,
+        'Data',
+        sessionName,
+        'OutputReference',
+        f'{trialName}.trc',
+    )
+    output_trc_df, _ = load_trc(output_trc)
+    ref_trc_df, _ = load_trc(ref_trc)
+    pd.testing.assert_frame_equal(
+        output_trc_df, ref_trc_df, check_exact=False, atol=1e-3
+    )
+
+    # Compare IK data
+    output_mot = os.path.join(
+        dataDir,
+        'Data',
+        sessionName,
+        'OpenSimData',
+        'Kinematics',
+        f'{trialName}.mot',
+    )
+    ref_mot = os.path.join(
+        dataDir,
+        'Data',
+        sessionName,
+        'OutputReference',
+        f'{trialName}.mot',
+    )
+    output_mot_df, _ = load_mot(output_mot)
+    ref_mot_df, _ = load_mot(ref_mot)
+    pd.testing.assert_index_equal(output_mot_df.columns, ref_mot_df.columns)
+    compare_mot(output_mot_df, ref_mot_df, t0, tf)
+
+# TODO: calibration and neutral
+# TODO: > 2 cameras
+# TODO: augmenter versions