From 5df7bbd10afaf2abeff25ecbda943c71226e53c8 Mon Sep 17 00:00:00 2001 From: Moira Andrews Date: Wed, 18 Feb 2026 13:51:48 -0800 Subject: [PATCH 1/7] add a check and update for observation status --- tom_observations/cadences/retry_failed_observations.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index fc15217bb..157069494 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -2,7 +2,7 @@ from dateutil.parser import parse from tom_observations.cadence import BaseCadenceForm, CadenceStrategy -from tom_observations.models import ObservationRecord +from tom_observations.models import ObservationRecord, DynamicCadence from tom_observations.facility import get_service_class @@ -23,6 +23,11 @@ class RetryFailedObservationsStrategy(CadenceStrategy): form = RetryFailedObservationsForm def run(self): + last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() + facility = get_service_class(last_obs.facility)() + facility.update_observation_status(last_obs.observation_id) # Updates the DB record + last_obs.refresh_from_db() + failed_observations = [obsr for obsr in self.dynamic_cadence.observation_group.observation_records.all() if obsr.failed] From 0c2c80dd1753ee242a20b27b5727d4411f926c60 Mon Sep 17 00:00:00 2001 From: Moira Andrews Date: Wed, 18 Feb 2026 13:52:12 -0800 Subject: [PATCH 2/7] update dynamic cadence on obs complete --- tom_observations/cadences/retry_failed_observations.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 157069494..413ff1c33 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -28,6 +28,12 @@ def run(self): facility.update_observation_status(last_obs.observation_id) # Updates the DB record last_obs.refresh_from_db() + if last_obs.status == 'COMPLETED': + obs_group = last_obs.observationgroup_set.first() + dynamic_cadence = DynamicCadence.objects.get(observation_group=obs_group) + dynamic_cadence.active = False + dynamic_cadence.save() + failed_observations = [obsr for obsr in self.dynamic_cadence.observation_group.observation_records.all() if obsr.failed] From 25298c3f7dc08e595b9db26f3e4b6a5571890027 Mon Sep 17 00:00:00 2001 From: Moira Andrews Date: Wed, 18 Feb 2026 13:52:54 -0800 Subject: [PATCH 3/7] remove excess whitespace --- tom_observations/cadences/retry_failed_observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 413ff1c33..e10be522e 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -27,7 +27,7 @@ def run(self): facility = get_service_class(last_obs.facility)() facility.update_observation_status(last_obs.observation_id) # Updates the DB record last_obs.refresh_from_db() - + if last_obs.status == 'COMPLETED': obs_group = last_obs.observationgroup_set.first() dynamic_cadence = DynamicCadence.objects.get(observation_group=obs_group) From 982fe86b19f3a68e3fc5180a54e62ad906df1205 Mon Sep 17 00:00:00 2001 From: Joey Chatelain Date: Wed, 4 Mar 2026 15:29:53 -0700 Subject: [PATCH 4/7] mock status check --- tom_observations/tests/test_cadence.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 66cf49511..a106168ae 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -59,7 +59,9 @@ def setUp(self): cadence_strategy='Test Strategy', cadence_parameters={'cadence_frequency': 72}, active=True, observation_group=self.group) - def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', + 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4, mock_get_obs_status): num_records = self.group.observation_records.count() observing_record = self.group.observation_records.first() observing_record.status = 'CANCELED' From aed35bf7a9dd9cad0edd9aea4184aa320ca0a328 Mon Sep 17 00:00:00 2001 From: moira-andrews <85570657+moira-andrews@users.noreply.github.com> Date: Wed, 4 Mar 2026 15:07:02 -0800 Subject: [PATCH 5/7] Cleaning up logic and pulling changes from snex2 --- .../cadences/retry_failed_observations.py | 79 +++++++++++-------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index e10be522e..0f6f25ec1 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -23,44 +23,57 @@ class RetryFailedObservationsStrategy(CadenceStrategy): form = RetryFailedObservationsForm def run(self): - last_obs = self.dynamic_cadence.observation_group.observation_records.order_by('-created').first() - facility = get_service_class(last_obs.facility)() - facility.update_observation_status(last_obs.observation_id) # Updates the DB record + records = self.dynamic_cadence.observation_group.observation_records.all().order_by('-created') + last_obs = records.first() + + if not last_obs: + return + + facility_class = get_service_class(last_obs.facility) + facility = facility_class() + facility.update_observation_status(last_obs.observation_id) last_obs.refresh_from_db() - if last_obs.status == 'COMPLETED': - obs_group = last_obs.observationgroup_set.first() - dynamic_cadence = DynamicCadence.objects.get(observation_group=obs_group) - dynamic_cadence.active = False - dynamic_cadence.save() + if not last_obs.terminal: + return + elif last_obs.status == 'COMPLETED': + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + return + + if not last_obs.failed: + return + + observation_payload = last_obs.parameters.copy() - failed_observations = [obsr for obsr - in self.dynamic_cadence.observation_group.observation_records.all() - if obsr.failed] + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + ) + + obs_type = observation_payload.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + + if not form.is_valid(): + return + + observation_ids = facility.submit_observation(form.observation_payload()) new_observations = [] - for obs in failed_observations: - observation_payload = obs.parameters - facility = get_service_class(obs.facility)() - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + + for observation_id in observation_ids: + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=observation_payload, + observation_id=observation_id ) - obs_type = obs.parameters.get('observation_type', None) - form = facility.get_form(obs_type)(data=observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) - - for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=obs.target, - facility=facility.name, - parameters=observation_payload, - observation_id=observation_id - ) - self.dynamic_cadence.observation_group.observation_records.add(record) - self.dynamic_cadence.observation_group.save() - new_observations.append(record) + self.dynamic_cadence.observation_group.observation_records.add(record) + new_observations.append(record) + + self.dynamic_cadence.observation_group.save() + + for obsr in new_observations: + facility.update_observation_status(obsr.observation_id) return new_observations From 834339d60dd62710d628df3c85b7881fa11632f2 Mon Sep 17 00:00:00 2001 From: Joey Chatelain Date: Wed, 4 Mar 2026 16:20:45 -0700 Subject: [PATCH 6/7] fix linting --- tom_observations/cadences/retry_failed_observations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index 0f6f25ec1..ecbf6ef82 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -2,7 +2,7 @@ from dateutil.parser import parse from tom_observations.cadence import BaseCadenceForm, CadenceStrategy -from tom_observations.models import ObservationRecord, DynamicCadence +from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class @@ -50,7 +50,7 @@ def run(self): observation_payload = self.advance_window( observation_payload, start_keyword=start_keyword, end_keyword=end_keyword ) - + obs_type = observation_payload.get('observation_type') form = facility.get_form(obs_type)(observation_payload) @@ -59,7 +59,7 @@ def run(self): observation_ids = facility.submit_observation(form.observation_payload()) new_observations = [] - + for observation_id in observation_ids: record = ObservationRecord.objects.create( target=last_obs.target, From c55dece2a16e9e106351aeeb7fd33cf164b1443c Mon Sep 17 00:00:00 2001 From: Joey Chatelain Date: Wed, 4 Mar 2026 16:22:44 -0700 Subject: [PATCH 7/7] missed one --- tom_observations/cadences/retry_failed_observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index ecbf6ef82..264ad736b 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -53,7 +53,7 @@ def run(self): obs_type = observation_payload.get('observation_type') form = facility.get_form(obs_type)(observation_payload) - + if not form.is_valid(): return