Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 49 additions & 25 deletions tom_observations/cadences/retry_failed_observations.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,33 +23,57 @@ class RetryFailedObservationsStrategy(CadenceStrategy):
form = RetryFailedObservationsForm

def run(self):
failed_observations = [obsr for obsr
in self.dynamic_cadence.observation_group.observation_records.all()
if obsr.failed]
records = self.dynamic_cadence.observation_group.observation_records.all().order_by('-created')
last_obs = records.first()

if not last_obs:
return

facility_class = get_service_class(last_obs.facility)
facility = facility_class()
facility.update_observation_status(last_obs.observation_id)
last_obs.refresh_from_db()

if not last_obs.terminal:
return
elif last_obs.status == 'COMPLETED':
self.dynamic_cadence.active = False
self.dynamic_cadence.save()
return

if not last_obs.failed:
return

observation_payload = last_obs.parameters.copy()

start_keyword, end_keyword = facility.get_start_end_keywords()
observation_payload = self.advance_window(
observation_payload, start_keyword=start_keyword, end_keyword=end_keyword
)

obs_type = observation_payload.get('observation_type')
form = facility.get_form(obs_type)(observation_payload)

if not form.is_valid():
return

observation_ids = facility.submit_observation(form.observation_payload())
new_observations = []
for obs in failed_observations:
observation_payload = obs.parameters
facility = get_service_class(obs.facility)()
start_keyword, end_keyword = facility.get_start_end_keywords()
observation_payload = self.advance_window(
observation_payload, start_keyword=start_keyword, end_keyword=end_keyword

for observation_id in observation_ids:
record = ObservationRecord.objects.create(
target=last_obs.target,
facility=facility.name,
parameters=observation_payload,
observation_id=observation_id
)
obs_type = obs.parameters.get('observation_type', None)
form = facility.get_form(obs_type)(data=observation_payload)
form.is_valid()
observation_ids = facility.submit_observation(form.observation_payload())

for observation_id in observation_ids:
# Create Observation record
record = ObservationRecord.objects.create(
target=obs.target,
facility=facility.name,
parameters=observation_payload,
observation_id=observation_id
)
self.dynamic_cadence.observation_group.observation_records.add(record)
self.dynamic_cadence.observation_group.save()
new_observations.append(record)
self.dynamic_cadence.observation_group.observation_records.add(record)
new_observations.append(record)

self.dynamic_cadence.observation_group.save()

for obsr in new_observations:
facility.update_observation_status(obsr.observation_id)

return new_observations

Expand Down
4 changes: 3 additions & 1 deletion tom_observations/tests/test_cadence.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ def setUp(self):
cadence_strategy='Test Strategy', cadence_parameters={'cadence_frequency': 72}, active=True,
observation_group=self.group)

def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4):
@patch('tom_observations.facilities.lco.LCOFacility.get_observation_status', return_value={'state': 'CANCELED',
'scheduled_start': None, 'scheduled_end': None})
def test_retry_when_failed_cadence(self, patch1, patch2, patch3, patch4, mock_get_obs_status):
num_records = self.group.observation_records.count()
observing_record = self.group.observation_records.first()
observing_record.status = 'CANCELED'
Expand Down
Loading