diff --git a/tom_observations/cadences/retry_failed_observations.py b/tom_observations/cadences/retry_failed_observations.py index fc15217bb..264ad736b 100644 --- a/tom_observations/cadences/retry_failed_observations.py +++ b/tom_observations/cadences/retry_failed_observations.py @@ -23,33 +23,57 @@ class RetryFailedObservationsStrategy(CadenceStrategy): form = RetryFailedObservationsForm def run(self): - failed_observations = [obsr for obsr - in self.dynamic_cadence.observation_group.observation_records.all() - if obsr.failed] + records = self.dynamic_cadence.observation_group.observation_records.all().order_by('-created') + last_obs = records.first() + + if not last_obs: + return + + facility_class = get_service_class(last_obs.facility) + facility = facility_class() + facility.update_observation_status(last_obs.observation_id) + last_obs.refresh_from_db() + + if not last_obs.terminal: + return + elif last_obs.status == 'COMPLETED': + self.dynamic_cadence.active = False + self.dynamic_cadence.save() + return + + if not last_obs.failed: + return + + observation_payload = last_obs.parameters.copy() + + start_keyword, end_keyword = facility.get_start_end_keywords() + observation_payload = self.advance_window( + observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + ) + + obs_type = observation_payload.get('observation_type') + form = facility.get_form(obs_type)(observation_payload) + + if not form.is_valid(): + return + + observation_ids = facility.submit_observation(form.observation_payload()) new_observations = [] - for obs in failed_observations: - observation_payload = obs.parameters - facility = get_service_class(obs.facility)() - start_keyword, end_keyword = facility.get_start_end_keywords() - observation_payload = self.advance_window( - observation_payload, start_keyword=start_keyword, end_keyword=end_keyword + + for observation_id in observation_ids: + record = ObservationRecord.objects.create( + target=last_obs.target, + facility=facility.name, + parameters=observation_payload, + observation_id=observation_id ) - obs_type = obs.parameters.get('observation_type', None) - form = facility.get_form(obs_type)(data=observation_payload) - form.is_valid() - observation_ids = facility.submit_observation(form.observation_payload()) - - for observation_id in observation_ids: - # Create Observation record - record = ObservationRecord.objects.create( - target=obs.target, - facility=facility.name, - parameters=observation_payload, - observation_id=observation_id - ) - self.dynamic_cadence.observation_group.observation_records.add(record) - self.dynamic_cadence.observation_group.save() - new_observations.append(record) + self.dynamic_cadence.observation_group.observation_records.add(record) + new_observations.append(record) + + self.dynamic_cadence.observation_group.save() + + for obsr in new_observations: + facility.update_observation_status(obsr.observation_id) return new_observations diff --git a/tom_observations/tests/test_cadence.py b/tom_observations/tests/test_cadence.py index 66cf49511..a106168ae 100644 --- a/tom_observations/tests/test_cadence.py +++ b/tom_observations/tests/test_cadence.py @@ -59,7 +59,9 @@ def setUp(self): cadence_strategy='Test Strategy', cadence_parameters={'cadence_frequency': 72}, active=True, observation_group=self.group) - def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4): + @patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED', + 'scheduled_start': None, 'scheduled_end': None}) + def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4, mock_get_obs_status): num_records = self.group.observation_records.count() observing_record = self.group.observation_records.first() observing_record.status = 'CANCELED'