From 7a7f32fe6252394740bdcf808ee800de338433a0 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Tue, 3 Feb 2026 08:52:52 -0800 Subject: [PATCH 01/29] First non-functional stub-out of new JPL Scout dataservice --- tom_dataservices/data_services/scout.py | 75 +++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tom_dataservices/data_services/scout.py diff --git a/tom_dataservices/data_services/scout.py b/tom_dataservices/data_services/scout.py new file mode 100644 index 000000000..bcf152b42 --- /dev/null +++ b/tom_dataservices/data_services/scout.py @@ -0,0 +1,75 @@ +from math import sqrt, degrees + +from astropy.constants import GM_sun, au +import requests + + +from tom_dataservices.dataservices import BaseDataService +from tom_dataservices.forms import BaseQueryForm +from tom_targets.models import Target + +class ScoutDataService(BaseDataService): + """ + Docstring for ScoutDataService + """ + name = 'Scout' + info_url = 'https://cneos.jpl.nasa.gov/scout/intro.html' + # Gaussian gravitational constant + _k = degrees(sqrt(GM_sun.value) * au.value**-1.5 * 86400.0) + + def build_query_parameters(self, parameters, **kwargs): + """ + Args: + parameters: dictionary containing either: + + - optional cutoff parameters + + - Scout name e.g. 'P10vY9r' + + Returns: + json containing parameters for querying the Scout API. + """ + return super().build_query_parameters(parameters, **kwargs) + + def create_target_from_query(self, target_results, **kwargs): + """ + Returns a Target instance for an object defined by a query result, + + :returns: target object + :rtype: `Target` + """ + + # Construct dictionary from ['orbits']['fields'] and ['orbits']['data'][0] + elements = dict(zip(target_results['orbits']['fields'], target_results['orbits']['data'][0])) + + target = Target( + name=target_results['objectName'], + type='NON-SIDEREAL', + scheme='MPC_MINOR_PLANET', + arg_of_perihelion=elements['w'], + lng_asc_node=elements['om'], + inclination=elements['inc'], + eccentricity=elements['ec'], + epoch_of_elements=float(elements['epoch'][2:]) - 0.5, + epoch_of_perihelion=float(elements['tp'][2:]) - 0.5, + perihdist=elements['qr'], + abs_mag=elements['H'], + slope=elements.get('G', 0.15) # Never actually present ? + ) + try: + target.semimajor_axis = target.perihdist / (1.0 - target.eccentricity) + if target.semimajor_axis < 0 or target.semimajor_axis > 1000.0: + target.semimajor_axis = None + except ZeroDivisionError: + target.semimajor_axis = None + if target.semimajor_axis: + target.mean_daily_motion = self._k / (target.semimajor_axis * sqrt(target.semimajor_axis)) + if target.mean_daily_motion: + td = target.epoch_of_elements - target.epoch_of_perihelion + mean_anomaly = td * target.mean_daily_motion + # Normalize into 0...360 range + mean_anomaly = mean_anomaly % 360.0 + if mean_anomaly < 0.0: + mean_anomaly += 360.0 + target.mean_anomaly = mean_anomaly + return target From c88b31f4deab40d914687fe81d35850b2ffbde0d Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Tue, 3 Feb 2026 08:59:43 -0800 Subject: [PATCH 02/29] Begin building query params dict --- tom_dataservices/data_services/scout.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tom_dataservices/data_services/scout.py b/tom_dataservices/data_services/scout.py index bcf152b42..74898d52b 100644 --- a/tom_dataservices/data_services/scout.py +++ b/tom_dataservices/data_services/scout.py @@ -29,7 +29,15 @@ def build_query_parameters(self, parameters, **kwargs): Returns: json containing parameters for querying the Scout API. """ - return super().build_query_parameters(parameters, **kwargs) + data = {} + if parameters.get('tdes') is not None: + data['tdes'] = parameters['tdes'] + + # Return at least one orbit + data['orbits'] = 1 + data['n-orbits'] = 1 + self.query_parameters = data + return data def create_target_from_query(self, target_results, **kwargs): """ From 17363e4f17df63a7204811c172050d78fd3d943a Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Tue, 3 Feb 2026 21:37:53 -0800 Subject: [PATCH 03/29] First basic functional version: Multi & single-object query and Target creation working --- tom_base/settings.py | 5 +++ tom_dataservices/data_services/scout.py | 48 +++++++++++++++++++++---- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 487fdda77..fee8f703c 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -287,6 +287,11 @@ 'tom_observations.cadences.resume_cadence_after_failure.ResumeCadenceAfterFailureStrategy' ] +DATA_SERVICES = { + 'Scout' : { + 'base_url': 'https://ssd-api.jpl.nasa.gov/scout.api', + }, +} # Define extra target fields here. Types can be any of "number", "string", "boolean" or "datetime" # See https://tomtoolkit.github.io/docs/target_fields for documentation on this feature # For example: diff --git a/tom_dataservices/data_services/scout.py b/tom_dataservices/data_services/scout.py index 74898d52b..f894e15e5 100644 --- a/tom_dataservices/data_services/scout.py +++ b/tom_dataservices/data_services/scout.py @@ -1,6 +1,7 @@ from math import sqrt, degrees from astropy.constants import GM_sun, au +from django import forms import requests @@ -8,6 +9,12 @@ from tom_dataservices.forms import BaseQueryForm from tom_targets.models import Target +class ScoutForm(BaseQueryForm): + tdes = forms.CharField(required=False, + label='NEOCP temporary designation') + neo_score_min = forms.IntegerField(required=False, label='Minimum NEO digest score (0..100)') + + class ScoutDataService(BaseDataService): """ Docstring for ScoutDataService @@ -17,6 +24,15 @@ class ScoutDataService(BaseDataService): # Gaussian gravitational constant _k = degrees(sqrt(GM_sun.value) * au.value**-1.5 * 86400.0) + @classmethod + def urls(cls, **kwargs) -> dict: + """Dictionary of urls for the JPL Scout API (all identical in this case)""" + urls = super().urls() + urls['base_url'] = cls.get_configuration('base_url', 'https://ssd-api.jpl.nasa.gov/scout.api') + urls['object_url'] = urls['base_url'] + urls['search_url'] = urls['base_url'] + return urls + def build_query_parameters(self, parameters, **kwargs): """ Args: @@ -39,6 +55,26 @@ def build_query_parameters(self, parameters, **kwargs): self.query_parameters = data return data + def query_service(self, data, **kwargs): + response = requests.get(self.get_urls('search_url'), data) + response.raise_for_status() + json_response = response.json() + if 'data' in json_response: + self.query_results = json_response['data'] + else: + # Per-object data has different structure + self.query_results = json_response + return self.query_results + + def query_targets(self, query_parameters): + target_parameters = self.build_query_parameters(query_parameters) + target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) + return target_data + + @classmethod + def get_form_class(cls): + return ScoutForm + def create_target_from_query(self, target_results, **kwargs): """ Returns a Target instance for an object defined by a query result, @@ -54,14 +90,14 @@ def create_target_from_query(self, target_results, **kwargs): name=target_results['objectName'], type='NON-SIDEREAL', scheme='MPC_MINOR_PLANET', - arg_of_perihelion=elements['w'], - lng_asc_node=elements['om'], - inclination=elements['inc'], - eccentricity=elements['ec'], + arg_of_perihelion=float(elements['w']), + lng_asc_node=float(elements['om']), + inclination=float(elements['inc']), + eccentricity=float(elements['ec']), epoch_of_elements=float(elements['epoch'][2:]) - 0.5, epoch_of_perihelion=float(elements['tp'][2:]) - 0.5, - perihdist=elements['qr'], - abs_mag=elements['H'], + perihdist=float(elements['qr']), + abs_mag=float(elements['H']), slope=elements.get('G', 0.15) # Never actually present ? ) try: From 676d4087413b2c03d14ba8a1ccf5eda6216e2592 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Wed, 4 Feb 2026 16:21:19 -0800 Subject: [PATCH 04/29] Updates for form-driven workflows. Blank/unfilled tdes field is coming back as null string not None. Don't ask for orbits when no querying for specific object as this isn't allowed in the Scout API Add custom table template for Scout results. --- tom_dataservices/data_services/scout.py | 12 +++-- .../partials/scout_query_results_table.html | 47 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html diff --git a/tom_dataservices/data_services/scout.py b/tom_dataservices/data_services/scout.py index f894e15e5..35edbd590 100644 --- a/tom_dataservices/data_services/scout.py +++ b/tom_dataservices/data_services/scout.py @@ -21,6 +21,8 @@ class ScoutDataService(BaseDataService): """ name = 'Scout' info_url = 'https://cneos.jpl.nasa.gov/scout/intro.html' + query_results_table = 'tom_dataservices/scout/partials/scout_query_results_table.html' + # Gaussian gravitational constant _k = degrees(sqrt(GM_sun.value) * au.value**-1.5 * 86400.0) @@ -46,12 +48,14 @@ def build_query_parameters(self, parameters, **kwargs): json containing parameters for querying the Scout API. """ data = {} - if parameters.get('tdes') is not None: + # import pprint + # pprint.pprint(parameters) + if parameters.get('tdes') is not None and parameters['tdes'] != '': data['tdes'] = parameters['tdes'] - # Return at least one orbit - data['orbits'] = 1 - data['n-orbits'] = 1 + # Return at least one orbit + data['orbits'] = 1 + data['n-orbits'] = 1 self.query_parameters = data return data diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html new file mode 100644 index 000000000..f44974f28 --- /dev/null +++ b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + {% for result in results %} + + + + + + + + + + + + + + + + + + {% endfor %} + +
Object Class Score
Name#obsRADecV (mag)RatePOS Unc(')HImpact RatingNEOPHAIEOGeocentricRMS
{{result.objectName}}{{ result.nObs }}{{ result.ra }}{{ result.dec }}{{ result.Vmag }}{{ result.rate }}{{ result.unc }}{{ result.H }}{{ result.rating }}{{ result.neoScore }}{{ result.phaScore }}{{ result.ieoScore }}{{ result.geocentricScore }}{{ result.rmsN }}
From f4760e1a6348f2a13a646ef4174626cb949cc2ed Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Wed, 4 Feb 2026 16:25:10 -0800 Subject: [PATCH 05/29] Add ScoutDataService to data_services() --- tom_dataservices/apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tom_dataservices/apps.py b/tom_dataservices/apps.py index 91a0c1d23..8532823ed 100644 --- a/tom_dataservices/apps.py +++ b/tom_dataservices/apps.py @@ -34,5 +34,6 @@ def data_services(self): data_services = [ {'class': f'{self.name}.data_services.simbad.SimbadDataService'}, {'class': f'{self.name}.data_services.tns.TNSDataService'}, + {'class': f'{self.name}.data_services.scout.ScoutDataService'}, ] return data_services From eb14bcde811f8d45d49f3a095699bfaa88327c9c Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Wed, 4 Feb 2026 17:10:56 -0800 Subject: [PATCH 06/29] Rewrite query_targets() to iterate over results table fetching orbital elements --- tom_dataservices/data_services/scout.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tom_dataservices/data_services/scout.py b/tom_dataservices/data_services/scout.py index 35edbd590..1e98f33fe 100644 --- a/tom_dataservices/data_services/scout.py +++ b/tom_dataservices/data_services/scout.py @@ -60,7 +60,7 @@ def build_query_parameters(self, parameters, **kwargs): return data def query_service(self, data, **kwargs): - response = requests.get(self.get_urls('search_url'), data) + response = requests.get(kwargs['url'], data) response.raise_for_status() json_response = response.json() if 'data' in json_response: @@ -71,9 +71,16 @@ def query_service(self, data, **kwargs): return self.query_results def query_targets(self, query_parameters): - target_parameters = self.build_query_parameters(query_parameters) - target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) - return target_data + """Set up and run a specialized query for retrieving targets from a DataService.""" + results = super().query_targets(query_parameters, url=self.get_urls('search_url')) + targets = [] + for result in results: + query_parameters['tdes'] = result['objectName'] + target_parameters = self.build_query_parameters(query_parameters) + print(target_parameters) + target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) + targets.append(target_data) + return targets @classmethod def get_form_class(cls): From 51371c03a8b139076124099b3d7d8970618d59d2 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 5 Feb 2026 16:36:11 -0800 Subject: [PATCH 07/29] Add a basic check for errors so it doesn't fall completely on its face --- tom_dataservices/data_services/scout.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tom_dataservices/data_services/scout.py b/tom_dataservices/data_services/scout.py index 1e98f33fe..7cac76aaa 100644 --- a/tom_dataservices/data_services/scout.py +++ b/tom_dataservices/data_services/scout.py @@ -74,12 +74,13 @@ def query_targets(self, query_parameters): """Set up and run a specialized query for retrieving targets from a DataService.""" results = super().query_targets(query_parameters, url=self.get_urls('search_url')) targets = [] - for result in results: - query_parameters['tdes'] = result['objectName'] - target_parameters = self.build_query_parameters(query_parameters) - print(target_parameters) - target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) - targets.append(target_data) + if results is not None and 'error' not in results: + for result in results: + query_parameters['tdes'] = result['objectName'] + target_parameters = self.build_query_parameters(query_parameters) + print(target_parameters) + target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) + targets.append(target_data) return targets @classmethod From 4008d865bc7e03bcd6dd719aa6e76bca4b2d5d05 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Fri, 6 Feb 2026 14:41:11 -0800 Subject: [PATCH 08/29] Giving up on and commiting broken state when attempting to filter on form paramters --- tom_dataservices/data_services/scout.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tom_dataservices/data_services/scout.py b/tom_dataservices/data_services/scout.py index 7cac76aaa..5f8086b0c 100644 --- a/tom_dataservices/data_services/scout.py +++ b/tom_dataservices/data_services/scout.py @@ -2,8 +2,9 @@ from astropy.constants import GM_sun, au from django import forms +from django.contrib import messages import requests - +import pprint from tom_dataservices.dataservices import BaseDataService from tom_dataservices.forms import BaseQueryForm @@ -70,17 +71,25 @@ def query_service(self, data, **kwargs): self.query_results = json_response return self.query_results - def query_targets(self, query_parameters): + def query_targets(self, query_parameters, **kwargs): """Set up and run a specialized query for retrieving targets from a DataService.""" - results = super().query_targets(query_parameters, url=self.get_urls('search_url')) + pprint.pprint(query_parameters) + results = super().query_targets(self.build_query_parameters(query_parameters), url=self.get_urls('search_url')) + targets = [] if results is not None and 'error' not in results: for result in results: - query_parameters['tdes'] = result['objectName'] - target_parameters = self.build_query_parameters(query_parameters) - print(target_parameters) - target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) - targets.append(target_data) + if result['neoScore'] >= self.input_parameters.get('neo_score_min', 0): + query_parameters['tdes'] = result['objectName'] + target_parameters = self.build_query_parameters(query_parameters) + target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) + targets.append(target_data) + else: + msg = "Error retrieving data from Scout." + if query_parameters.get('tdes', '') != '': + msg += f" Object {query_parameters['tdes']} is no longer on Scout." + # if request is not None: + # messages.error(request, msg) return targets @classmethod From a668789996c6006a000a801bba79df598edaca24 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Mon, 9 Feb 2026 14:21:07 -0800 Subject: [PATCH 09/29] Rename scout.py to jpl.py (Since we likely will have other JPL DataServices). Change BaseDataService->DataService from upstream changes --- tom_dataservices/apps.py | 2 +- tom_dataservices/data_services/{scout.py => jpl.py} | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) rename tom_dataservices/data_services/{scout.py => jpl.py} (97%) diff --git a/tom_dataservices/apps.py b/tom_dataservices/apps.py index 8532823ed..1dc97e2d2 100644 --- a/tom_dataservices/apps.py +++ b/tom_dataservices/apps.py @@ -34,6 +34,6 @@ def data_services(self): data_services = [ {'class': f'{self.name}.data_services.simbad.SimbadDataService'}, {'class': f'{self.name}.data_services.tns.TNSDataService'}, - {'class': f'{self.name}.data_services.scout.ScoutDataService'}, + {'class': f'{self.name}.data_services.jpl.ScoutDataService'}, ] return data_services diff --git a/tom_dataservices/data_services/scout.py b/tom_dataservices/data_services/jpl.py similarity index 97% rename from tom_dataservices/data_services/scout.py rename to tom_dataservices/data_services/jpl.py index 5f8086b0c..4af90148f 100644 --- a/tom_dataservices/data_services/scout.py +++ b/tom_dataservices/data_services/jpl.py @@ -6,7 +6,7 @@ import requests import pprint -from tom_dataservices.dataservices import BaseDataService +from tom_dataservices.dataservices import DataService from tom_dataservices.forms import BaseQueryForm from tom_targets.models import Target @@ -16,11 +16,12 @@ class ScoutForm(BaseQueryForm): neo_score_min = forms.IntegerField(required=False, label='Minimum NEO digest score (0..100)') -class ScoutDataService(BaseDataService): +class ScoutDataService(DataService): """ Docstring for ScoutDataService """ name = 'Scout' + app_version = '0.0.2' info_url = 'https://cneos.jpl.nasa.gov/scout/intro.html' query_results_table = 'tom_dataservices/scout/partials/scout_query_results_table.html' From 5613a8f9fb9784a2c22c672f6a96ae5e0acf9a92 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Mon, 9 Feb 2026 16:36:50 -0800 Subject: [PATCH 10/29] Additional updates for Scout service: Save a copy of input form parameters so we can do the NEO score filtering (and other filters in future). Check for expected signature. Add get_additional_context_data() for a few extra things for the template. Update template to show total number of results and no. filtered from (if applicable) --- tom_dataservices/data_services/jpl.py | 50 +++++++++++++++---- .../partials/scout_query_results_table.html | 1 + 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 4af90148f..0304b3e55 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -3,6 +3,7 @@ from astropy.constants import GM_sun, au from django import forms from django.contrib import messages +import logging import requests import pprint @@ -10,6 +11,7 @@ from tom_dataservices.forms import BaseQueryForm from tom_targets.models import Target +logger = logging.getLogger(__name__) class ScoutForm(BaseQueryForm): tdes = forms.CharField(required=False, label='NEOCP temporary designation') @@ -24,6 +26,7 @@ class ScoutDataService(DataService): app_version = '0.0.2' info_url = 'https://cneos.jpl.nasa.gov/scout/intro.html' query_results_table = 'tom_dataservices/scout/partials/scout_query_results_table.html' + expected_signature = {'source': 'NASA/JPL Scout API', 'version': '1.3'} # Gaussian gravitational constant _k = degrees(sqrt(GM_sun.value) * au.value**-1.5 * 86400.0) @@ -50,8 +53,13 @@ def build_query_parameters(self, parameters, **kwargs): json containing parameters for querying the Scout API. """ data = {} - # import pprint - # pprint.pprint(parameters) + + # Save a copy of the input form parameters for later use as there are some parameters that are used in the + # query_targets method that are not part of the query to the Scout API. + # But don't save and overwrite later versions which don't have the form parameters (eg. when running + # query_targets with the tdes parameter set to a Scout name). + if 'neo_score_min' in parameters: + self.input_parameters = parameters if parameters.get('tdes') is not None and parameters['tdes'] != '': data['tdes'] = parameters['tdes'] @@ -62,25 +70,39 @@ def build_query_parameters(self, parameters, **kwargs): return data def query_service(self, data, **kwargs): - response = requests.get(kwargs['url'], data) + """Make call to the JPL Scout service + + :param data: Dictionary containing query parameters for the Scout API. + :type data: dict + :return: json containing response from Scout API. + :rtype: dict + """ + response = requests.get(self.get_urls(url_type='search_url'), data) + response.raise_for_status() json_response = response.json() - if 'data' in json_response: - self.query_results = json_response['data'] + if json_response['signature'] == self.expected_signature: + if 'data' in json_response: + self.query_results = json_response['data'] + self.total_results = int(json_response.get('count', 0)) + else: + # Per-object data has different structure + self.query_results = json_response else: - # Per-object data has different structure - self.query_results = json_response + logger.warning(f"Signature of response from Scout API does not match expected signature. Expected {self.expected_signature}, got {json_response['signature']}.") return self.query_results def query_targets(self, query_parameters, **kwargs): """Set up and run a specialized query for retrieving targets from a DataService.""" - pprint.pprint(query_parameters) - results = super().query_targets(self.build_query_parameters(query_parameters), url=self.get_urls('search_url')) + results = self.query_service(self.build_query_parameters(query_parameters)) targets = [] if results is not None and 'error' not in results: + neo_score_min = 0 + if self.input_parameters.get('neo_score_min', 0) is not None: + neo_score_min = self.input_parameters['neo_score_min'] for result in results: - if result['neoScore'] >= self.input_parameters.get('neo_score_min', 0): + if result['neoScore'] >= neo_score_min: query_parameters['tdes'] = result['objectName'] target_parameters = self.build_query_parameters(query_parameters) target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) @@ -97,6 +119,14 @@ def query_targets(self, query_parameters, **kwargs): def get_form_class(cls): return ScoutForm + def get_additional_context_data(self, **kwargs): + """Add additional context data for rendering the query results template.""" + context = {} + pprint.pprint(self.query_results, indent=2) + context['total_results'] = self.total_results if self.total_results is not None else 0 + context['neo_score_min'] = self.input_parameters.get('neo_score_min', 0) + return context + def create_target_from_query(self, target_results, **kwargs): """ Returns a Target instance for an object defined by a query result, diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html index f44974f28..ee18f8847 100644 --- a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html +++ b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html @@ -1,3 +1,4 @@ +Showing {{results|length}} results {% if results|length != total_results %}(filtered from {{total_results}} total results){% endif %} From 31f3ef158d40484f74ba68e9b041a54068c4446f Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Mon, 9 Feb 2026 20:19:36 -0800 Subject: [PATCH 11/29] All modes of form-filling should be functional now. Add auto-pluralizing of result numbers in template --- tom_dataservices/data_services/jpl.py | 43 ++++++++++++++----- .../partials/scout_query_results_table.html | 2 +- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 0304b3e55..acb9b7d58 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -27,6 +27,7 @@ class ScoutDataService(DataService): info_url = 'https://cneos.jpl.nasa.gov/scout/intro.html' query_results_table = 'tom_dataservices/scout/partials/scout_query_results_table.html' expected_signature = {'source': 'NASA/JPL Scout API', 'version': '1.3'} + total_results = None # Gaussian gravitational constant _k = degrees(sqrt(GM_sun.value) * au.value**-1.5 * 86400.0) @@ -60,6 +61,7 @@ def build_query_parameters(self, parameters, **kwargs): # query_targets with the tdes parameter set to a Scout name). if 'neo_score_min' in parameters: self.input_parameters = parameters + if parameters.get('tdes') is not None and parameters['tdes'] != '': data['tdes'] = parameters['tdes'] @@ -81,15 +83,26 @@ def query_service(self, data, **kwargs): response.raise_for_status() json_response = response.json() - if json_response['signature'] == self.expected_signature: - if 'data' in json_response: - self.query_results = json_response['data'] - self.total_results = int(json_response.get('count', 0)) + if json_response is not None and 'error' not in json_response: + if json_response['signature'] == self.expected_signature: + if 'data' in json_response: + self.query_results = json_response['data'] + self.total_results = int(json_response.get('count', 0)) + else: + # Per-object data has different structure, make it into a list of 1 target + # so the the `for result in results` in `query_targets()` iterates (once..) + # over the target(s) and not over the keys of a single target + self.query_results = [json_response, ] + if self.total_results is None: + self.total_results = 1 else: - # Per-object data has different structure - self.query_results = json_response + logger.warning(f"Signature of response from Scout API does not match expected signature. Expected {self.expected_signature}, got {json_response['signature']}.") else: - logger.warning(f"Signature of response from Scout API does not match expected signature. Expected {self.expected_signature}, got {json_response['signature']}.") + self.query_results = None + msg = "Error retrieving data from Scout." + if data.get('tdes', '') != '': + msg += f" Object {data['tdes']} is no longer on Scout." + logger.warning(msg) return self.query_results def query_targets(self, query_parameters, **kwargs): @@ -102,10 +115,18 @@ def query_targets(self, query_parameters, **kwargs): if self.input_parameters.get('neo_score_min', 0) is not None: neo_score_min = self.input_parameters['neo_score_min'] for result in results: + # print(result['objectName'], result['neoScore'], result['neoScore'] >= neo_score_min) if result['neoScore'] >= neo_score_min: - query_parameters['tdes'] = result['objectName'] - target_parameters = self.build_query_parameters(query_parameters) - target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) + if 'orbits' in result: + # This was a query for a specific target so we already have the needed info + target_data = result + else: + # Fetch per-target data + query_parameters['tdes'] = result['objectName'] + target_parameters = self.build_query_parameters(query_parameters) + target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) + if target_data is not None: + target_data = target_data[0] targets.append(target_data) else: msg = "Error retrieving data from Scout." @@ -122,7 +143,7 @@ def get_form_class(cls): def get_additional_context_data(self, **kwargs): """Add additional context data for rendering the query results template.""" context = {} - pprint.pprint(self.query_results, indent=2) + context['total_results'] = self.total_results if self.total_results is not None else 0 context['neo_score_min'] = self.input_parameters.get('neo_score_min', 0) return context diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html index ee18f8847..36a553a83 100644 --- a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html +++ b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html @@ -1,4 +1,4 @@ -Showing {{results|length}} results {% if results|length != total_results %}(filtered from {{total_results}} total results){% endif %} +Showing {{results|length}} result{{results|length|pluralize}} {% if results|length != total_results %}(filtered from {{total_results}} total result{{total_results|pluralize}}){% endif %}
From b6dd1a18321d79819ee66705a122800e882bca77 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Mon, 9 Feb 2026 21:14:43 -0800 Subject: [PATCH 12/29] First attempts at splitting form into simple and advanced sections. flake8 fixes --- tom_dataservices/data_services/jpl.py | 29 ++++++++++++++----- .../scout/partials/scout_advanced_form.html | 3 ++ .../scout/partials/scout_simple_form.html | 3 ++ 3 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html create mode 100644 tom_dataservices/templates/tom_dataservices/scout/partials/scout_simple_form.html diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index acb9b7d58..d9b7c6d59 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -12,10 +12,18 @@ from tom_targets.models import Target logger = logging.getLogger(__name__) + + class ScoutForm(BaseQueryForm): tdes = forms.CharField(required=False, label='NEOCP temporary designation') - neo_score_min = forms.IntegerField(required=False, label='Minimum NEO digest score (0..100)') + neo_score_min = forms.IntegerField(required=False, label='Minimum NEO digest score (0..100)', + help_text='Minimum NEO digest score (0..100) permissible') + pha_score_min = forms.IntegerField(required=False, label='Minimum PHA digest score (0..100)', + help_text='Minimum PHA digest score (0..100) permissible') + geo_score_max = forms.IntegerField(required=False, initial=5, + label='Maximum GEO digest score (0..100)', + help_text='Maximum Geocentric digest score (0..100) permissible') class ScoutDataService(DataService): @@ -41,12 +49,18 @@ def urls(cls, **kwargs) -> dict: urls['search_url'] = urls['base_url'] return urls + def get_simple_form_partial(self): + return 'tom_dataservices/scout/partials/scout_simple_form.html' + + def get_advanced_form_partial(self): + return 'tom_dataservices/scout/partials/scout_advanced_form.html' + def build_query_parameters(self, parameters, **kwargs): """ Args: parameters: dictionary containing either: - - optional cutoff parameters + - optional cutoff parameters - Scout name e.g. 'P10vY9r' @@ -61,6 +75,7 @@ def build_query_parameters(self, parameters, **kwargs): # query_targets with the tdes parameter set to a Scout name). if 'neo_score_min' in parameters: self.input_parameters = parameters + pprint.pprint(parameters, indent=2) if parameters.get('tdes') is not None and parameters['tdes'] != '': data['tdes'] = parameters['tdes'] @@ -79,7 +94,7 @@ def query_service(self, data, **kwargs): :return: json containing response from Scout API. :rtype: dict """ - response = requests.get(self.get_urls(url_type='search_url'), data) + response = requests.get(self.get_urls(url_type='search_url'), data) response.raise_for_status() json_response = response.json() @@ -157,7 +172,7 @@ def create_target_from_query(self, target_results, **kwargs): """ # Construct dictionary from ['orbits']['fields'] and ['orbits']['data'][0] - elements = dict(zip(target_results['orbits']['fields'], target_results['orbits']['data'][0])) + elements = dict(zip(target_results['orbits']['fields'], target_results['orbits']['data'][0])) target = Target( name=target_results['objectName'], @@ -171,17 +186,17 @@ def create_target_from_query(self, target_results, **kwargs): epoch_of_perihelion=float(elements['tp'][2:]) - 0.5, perihdist=float(elements['qr']), abs_mag=float(elements['H']), - slope=elements.get('G', 0.15) # Never actually present ? + slope=elements.get('G', 0.15) # Never actually present ? ) try: target.semimajor_axis = target.perihdist / (1.0 - target.eccentricity) if target.semimajor_axis < 0 or target.semimajor_axis > 1000.0: target.semimajor_axis = None - except ZeroDivisionError: + except (ZeroDivisionError, ValueError): target.semimajor_axis = None if target.semimajor_axis: target.mean_daily_motion = self._k / (target.semimajor_axis * sqrt(target.semimajor_axis)) - if target.mean_daily_motion: + if target.mean_daily_motion and target.epoch_of_elements and target.epoch_of_perihelion: td = target.epoch_of_elements - target.epoch_of_perihelion mean_anomaly = td * target.mean_daily_motion # Normalize into 0...360 range diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html new file mode 100644 index 000000000..983bf9e0f --- /dev/null +++ b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html @@ -0,0 +1,3 @@ +{% load bootstrap4 %} +{% bootstrap_field form.pha_score_min %} +{% bootstrap_field form.geo_score_max %} diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_simple_form.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_simple_form.html new file mode 100644 index 000000000..1f0ff9635 --- /dev/null +++ b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_simple_form.html @@ -0,0 +1,3 @@ +{% load bootstrap4 %} +{% bootstrap_field form.tdes %} +{% bootstrap_field form.neo_score_min %} From e62f5223d5981a77aaab0b56787693b558d97e54 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Tue, 10 Feb 2026 15:45:48 -0800 Subject: [PATCH 13/29] Implement filtering on PHA and GEO scores --- tom_dataservices/data_services/jpl.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index d9b7c6d59..7ba5e22de 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -5,7 +5,7 @@ from django.contrib import messages import logging import requests -import pprint +# import pprint from tom_dataservices.dataservices import DataService from tom_dataservices.forms import BaseQueryForm @@ -31,7 +31,7 @@ class ScoutDataService(DataService): Docstring for ScoutDataService """ name = 'Scout' - app_version = '0.0.2' + app_version = '0.0.3' info_url = 'https://cneos.jpl.nasa.gov/scout/intro.html' query_results_table = 'tom_dataservices/scout/partials/scout_query_results_table.html' expected_signature = {'source': 'NASA/JPL Scout API', 'version': '1.3'} @@ -75,7 +75,7 @@ def build_query_parameters(self, parameters, **kwargs): # query_targets with the tdes parameter set to a Scout name). if 'neo_score_min' in parameters: self.input_parameters = parameters - pprint.pprint(parameters, indent=2) + # pprint.pprint(parameters, indent=2) if parameters.get('tdes') is not None and parameters['tdes'] != '': data['tdes'] = parameters['tdes'] @@ -111,7 +111,9 @@ def query_service(self, data, **kwargs): if self.total_results is None: self.total_results = 1 else: - logger.warning(f"Signature of response from Scout API does not match expected signature. Expected {self.expected_signature}, got {json_response['signature']}.") + msg = "Signature of response from Scout API does not match expected signature. " + msg += f"Expected {self.expected_signature}, got {json_response['signature']}." + logger.warning(msg) else: self.query_results = None msg = "Error retrieving data from Scout." @@ -129,9 +131,16 @@ def query_targets(self, query_parameters, **kwargs): neo_score_min = 0 if self.input_parameters.get('neo_score_min', 0) is not None: neo_score_min = self.input_parameters['neo_score_min'] + pha_score_min = 0 + if self.input_parameters.get('pha_score_min', 0) is not None: + pha_score_min = self.input_parameters['pha_score_min'] + geo_score_max = 0 + if self.input_parameters.get('geo_score_max', 0) is not None: + geo_score_max = self.input_parameters['geo_score_max'] for result in results: # print(result['objectName'], result['neoScore'], result['neoScore'] >= neo_score_min) - if result['neoScore'] >= neo_score_min: + if result['neoScore'] >= neo_score_min and result['phaScore'] >= pha_score_min and \ + result['geocentricScore'] < geo_score_max: if 'orbits' in result: # This was a query for a specific target so we already have the needed info target_data = result @@ -161,6 +170,8 @@ def get_additional_context_data(self, **kwargs): context['total_results'] = self.total_results if self.total_results is not None else 0 context['neo_score_min'] = self.input_parameters.get('neo_score_min', 0) + context['pha_score_min'] = self.input_parameters.get('pha_score_min', 0) + context['geo_score_max'] = self.input_parameters.get('geo_score_max', 5) return context def create_target_from_query(self, target_results, **kwargs): From 59fb4ec74f4b035c6b4d4e047d231f7e1dd47816 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Tue, 10 Feb 2026 17:56:22 -0800 Subject: [PATCH 14/29] Add ability to filter on impact rating, CA distance and positional uncertainty. Add CA distance into results table. --- tom_dataservices/data_services/jpl.py | 56 ++++++++++++++++--- .../scout/partials/scout_advanced_form.html | 4 ++ .../partials/scout_query_results_table.html | 2 + 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 7ba5e22de..3a3aad02c 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -17,13 +17,27 @@ class ScoutForm(BaseQueryForm): tdes = forms.CharField(required=False, label='NEOCP temporary designation') - neo_score_min = forms.IntegerField(required=False, label='Minimum NEO digest score (0..100)', + neo_score_min = forms.IntegerField(required=False, min_value=0, max_value=100, + label='Minimum NEO digest score (0..100)', help_text='Minimum NEO digest score (0..100) permissible') - pha_score_min = forms.IntegerField(required=False, label='Minimum PHA digest score (0..100)', + pha_score_min = forms.IntegerField(required=False, min_value=0, max_value=100, + label='Minimum PHA digest score (0..100)', help_text='Minimum PHA digest score (0..100) permissible') - geo_score_max = forms.IntegerField(required=False, initial=5, + geo_score_max = forms.IntegerField(required=False, initial=5, min_value=0, max_value=100, label='Maximum GEO digest score (0..100)', help_text='Maximum Geocentric digest score (0..100) permissible') + help_text = 'Rating to character the chances of an Earth impact ' + help_text += '(0=negligible, 1=small, 2=modest, 3=moderate, 4=elevated)' + impact_rating_min = forms.IntegerField(required=False, min_value=0, max_value=4, + label='Minimum impact rating (0..4)', + help_text=help_text) + ca_dist_min = forms.FloatField(required=False, + label='Minimum CA distance (LD)', + help_text='Minimum close approach distance (lunar distances)') + pos_unc_min = forms.FloatField(required=False, + label='Minimum positional uncertainty (arcmin)') + pos_unc_max = forms.FloatField(required=False, + label='Maximum positional uncertainty (arcmin)') class ScoutDataService(DataService): @@ -134,13 +148,41 @@ def query_targets(self, query_parameters, **kwargs): pha_score_min = 0 if self.input_parameters.get('pha_score_min', 0) is not None: pha_score_min = self.input_parameters['pha_score_min'] - geo_score_max = 0 - if self.input_parameters.get('geo_score_max', 0) is not None: + geo_score_max = 101 + if self.input_parameters.get('geo_score_max', 101) is not None: geo_score_max = self.input_parameters['geo_score_max'] + # Might be None if we want all objects irrespective of impact chance + impact_rating_min = self.input_parameters['impact_rating_min'] + ca_dist_min = self.input_parameters['ca_dist_min'] + pos_unc_min = 0 + if self.input_parameters.get('pos_unc_min', 0) is not None: + pos_unc_min = self.input_parameters['pos_unc_min'] + pos_unc_max = 360 * 60 # 360 degrees (whole sky) as arcmin + if self.input_parameters.get('pos_unc_max', pos_unc_max) is not None: + pos_unc_max = self.input_parameters['pos_unc_max'] + for result in results: - # print(result['objectName'], result['neoScore'], result['neoScore'] >= neo_score_min) + # Filter on the many form + try: + pos_unc = float(result['unc']) + except ValueError: + pos_unc = 0.0 + try: + ca_dist = float(result['ca_dist']) + except TypeError: + ca_dist = None + # print("neoScore phaScore, geoScore, rating, caDist, posuncmin, posuncmax") + # print(result['neoScore'] >= neo_score_min, result['phaScore'] >= pha_score_min, \ + # result['geocentricScore'] < geo_score_max, ((result['rating'] is not None and \ + # impact_rating_min is not None and result['rating'] >= impact_rating_min) or \ + # impact_rating_min is None), (ca_dist_min is None or (ca_dist_min is not None and result['caDist'] is not None and\ + # float(result['caDist']) <= ca_dist_min)), pos_unc >= pos_unc_min , pos_unc <= pos_unc_max) if result['neoScore'] >= neo_score_min and result['phaScore'] >= pha_score_min and \ - result['geocentricScore'] < geo_score_max: + result['geocentricScore'] < geo_score_max and ((result['rating'] is not None and + impact_rating_min is not None and result['rating'] >= impact_rating_min) or + impact_rating_min is None) and (ca_dist_min is None or + (ca_dist_min is not None and ca_dist is not None and ca_dist <= ca_dist_min)) and \ + pos_unc >= pos_unc_min and pos_unc <= pos_unc_max: if 'orbits' in result: # This was a query for a specific target so we already have the needed info target_data = result diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html index 983bf9e0f..ba17b533c 100644 --- a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html +++ b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html @@ -1,3 +1,7 @@ {% load bootstrap4 %} {% bootstrap_field form.pha_score_min %} {% bootstrap_field form.geo_score_max %} +{% bootstrap_field form.impact_rating_min %} +{% bootstrap_field form.ca_dist_min %} +{% bootstrap_field form.pos_unc_min %} +{% bootstrap_field form.pos_unc_max %} diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html index 36a553a83..7066a2286 100644 --- a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html +++ b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_query_results_table.html @@ -17,6 +17,7 @@ + @@ -37,6 +38,7 @@ + From 560314a9c8f532d3866c92de56a75bbc8848dbcb Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Wed, 11 Feb 2026 14:04:47 -0800 Subject: [PATCH 15/29] Fix lint/flake errors --- tom_base/settings.py | 2 +- tom_dataservices/data_services/jpl.py | 29 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index fee8f703c..0748c23bd 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -288,7 +288,7 @@ ] DATA_SERVICES = { - 'Scout' : { + 'Scout': { 'base_url': 'https://ssd-api.jpl.nasa.gov/scout.api', }, } diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 3a3aad02c..5b40a34ff 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -2,7 +2,6 @@ from astropy.constants import GM_sun, au from django import forms -from django.contrib import messages import logging import requests # import pprint @@ -168,20 +167,28 @@ def query_targets(self, query_parameters, **kwargs): except ValueError: pos_unc = 0.0 try: - ca_dist = float(result['ca_dist']) + ca_dist = float(result['caDist']) except TypeError: ca_dist = None # print("neoScore phaScore, geoScore, rating, caDist, posuncmin, posuncmax") - # print(result['neoScore'] >= neo_score_min, result['phaScore'] >= pha_score_min, \ - # result['geocentricScore'] < geo_score_max, ((result['rating'] is not None and \ - # impact_rating_min is not None and result['rating'] >= impact_rating_min) or \ - # impact_rating_min is None), (ca_dist_min is None or (ca_dist_min is not None and result['caDist'] is not None and\ - # float(result['caDist']) <= ca_dist_min)), pos_unc >= pos_unc_min , pos_unc <= pos_unc_max) + # print(result['neoScore'] >= neo_score_min, result['phaScore'] >= pha_score_min, + # result['geocentricScore'] < geo_score_max, + # ((result['rating'] is not None and + # impact_rating_min is not None and + # result['rating'] >= impact_rating_min) or + # impact_rating_min is None), + # (ca_dist_min is None or + # (ca_dist_min is not None and ca_dist is not None and ca_dist <= ca_dist_min)), + # pos_unc >= pos_unc_min, pos_unc <= pos_unc_max) if result['neoScore'] >= neo_score_min and result['phaScore'] >= pha_score_min and \ - result['geocentricScore'] < geo_score_max and ((result['rating'] is not None and - impact_rating_min is not None and result['rating'] >= impact_rating_min) or - impact_rating_min is None) and (ca_dist_min is None or - (ca_dist_min is not None and ca_dist is not None and ca_dist <= ca_dist_min)) and \ + result['geocentricScore'] < geo_score_max and \ + ((result['rating'] is not None and + impact_rating_min is not None and + result['rating'] >= impact_rating_min + ) or + impact_rating_min is None) and \ + (ca_dist_min is None or + (ca_dist_min is not None and ca_dist is not None and ca_dist <= ca_dist_min)) and \ pos_unc >= pos_unc_min and pos_unc <= pos_unc_max: if 'orbits' in result: # This was a query for a specific target so we already have the needed info From 0561977e8a40adeba14c52bd984d23e35390f096 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Wed, 11 Feb 2026 16:05:03 -0800 Subject: [PATCH 16/29] Add start of tests for ScoutDataService --- tom_dataservices/data_services/jpl.py | 2 +- .../tests/data_services/__init__.py | 0 .../tests/data_services/test_jpl.py | 155 ++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 tom_dataservices/tests/data_services/__init__.py create mode 100644 tom_dataservices/tests/data_services/test_jpl.py diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 5b40a34ff..272282966 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -161,7 +161,7 @@ def query_targets(self, query_parameters, **kwargs): pos_unc_max = self.input_parameters['pos_unc_max'] for result in results: - # Filter on the many form + # Filter on the many, many form parameters. try: pos_unc = float(result['unc']) except ValueError: diff --git a/tom_dataservices/tests/data_services/__init__.py b/tom_dataservices/tests/data_services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_dataservices/tests/data_services/test_jpl.py b/tom_dataservices/tests/data_services/test_jpl.py new file mode 100644 index 000000000..966a2c4b8 --- /dev/null +++ b/tom_dataservices/tests/data_services/test_jpl.py @@ -0,0 +1,155 @@ +from django.test import TestCase +from unittest import mock + +from tom_dataservices.data_services.jpl import ScoutDataService +# from tom_dataservices.tests.factories import scout_resultsFactory +from tom_targets.models import Target + +class TestScoutDataService(TestCase): + """ + Test the functionality of the JPL ScoutDataService + """ + def setUp(self): + self.jpl_ds = ScoutDataService() + self.input_parameters = { 'ca_dist_min': None, + 'data_service': 'Scout', + 'geo_score_max': 5, + 'impact_rating_min': None, + 'neo_score_min': None, + 'pha_score_min': None, + 'pos_unc_max': None, + 'pos_unc_min': None, + 'query_name': '', + 'query_save': False, + 'tdes': ''} + self.scout_results = [{ 'uncP1': '1500', + 'tEphem': '2026-02-11 22:45', + 'caDist': '0.98', + 'phaScore': 0, + 'vInf': '20.1', + 'moid': '0.001', + 'ra': '08:54', + 'objectName': 'ZTF10BL', + 'neo1kmScore': 0, + 'geocentricScore': 1, + 'rate': '1.9', + 'Vmag': '20.8', + 'dec': '+28', + 'tisserandScore': 39, + 'rating': 2, + 'arc': '0.35', + 'H': '26.3', + 'elong': '156', + 'unc': '1400', + 'ieoScore': 0, + 'orbits': {'data': [[0, + '2461079.712931752', + '8.855752093403347E-01', + '4.998861947435592E-01', + '2461038.529885748', + '1.3898404962804094E+02', + '2.6665272638172337E+02', + '1.4692644033445657E+01', + '25.402927', + '4.34144690247134E-03', + '2.4610794091831E+06', + '1.971147172E-03', + '2.790457665E+01', + '1.278753791E+03', + 0]], + 'count': '1', + 'fields': ['idx', + 'epoch', + 'ec', + 'qr', + 'tp', + 'om', + 'w', + 'inc', + 'H', + 'dca', + 'tca', + 'moid', + 'vinf', + 'geoEcc', + 'impFlag']}, + 'nObs': 4, + 'rmsN': '0.12', + 'signature': {'source': 'NASA/JPL Scout API', 'version': '1.3'}, + 'lastRun': '2026-02-08 13:52', + 'neoScore': 100},] + + target_params = { 'name': 'ZTF10BL', + 'type': 'NON-SIDEREAL', + 'permissions': 'OPEN', + 'scheme': 'MPC_MINOR_PLANET', + 'epoch_of_elements': 61079.212931752, + 'mean_anomaly': 4.4452481923884894, + 'arg_of_perihelion': 266.65272638172337, + 'eccentricity': 0.8855752093403347, + 'lng_asc_node': 138.98404962804094, + 'inclination': 14.692644033445657, + 'mean_daily_motion': 0.10793879092762414, + 'semimajor_axis': 4.368687867914702, + 'epoch_of_perihelion': 61038.029885748, + 'perihdist': 0.4998861947435592, + 'abs_mag': 25.402927, + 'slope': 0.15} + self.test_target = Target.objects.create(**target_params) + + def test_build_query_parameters_no_target(self): + """ + Test that the build_query_parameters method correctly builds the query parameters for the JPL ScoutDataService + """ + expected_parameters = {} + + parameters = self.jpl_ds.build_query_parameters(self.input_parameters) + + self.assertEqual(parameters, expected_parameters) + self.assertEqual(self.jpl_ds.input_parameters, self.input_parameters) + + def test_build_query_parameters_with_target(self): + """ + Test that the build_query_parameters method correctly builds the query parameters for the JPL ScoutDataService + """ + self.input_parameters['tdes'] = 'ZTF10BL' + expected_parameters = {'tdes': 'ZTF10BL', 'orbits': 1, 'n-orbits': 1} + parameters = self.jpl_ds.build_query_parameters(self.input_parameters) + + self.assertEqual(parameters, expected_parameters) + self.assertEqual(self.jpl_ds.input_parameters, self.input_parameters) + + @mock.patch('tom_dataservices.data_services.jpl.ScoutDataService.query_service') + def test_query_targets_single(self, mock_client): + mock_client.side_effect = [self.scout_results, ] + self.input_parameters['tdes'] = 'ZTF10BL' + + targets = self.jpl_ds.query_targets(self.input_parameters) + expected_target_results = {'objectName': self.scout_results[0]['objectName'], + 'neoScore': self.scout_results[0]['neoScore'], + 'phaScore': self.scout_results[0]['phaScore'], + 'geocentricScore': self.scout_results[0]['geocentricScore'], + 'rating': self.scout_results[0]['rating'], + 'unc': self.scout_results[0]['unc'], + 'orbits': self.scout_results[0]['orbits'], + } + for target in targets: + for key in expected_target_results.keys(): + if key == 'orbits': + self.assertEqual(type(target[key]), type(expected_target_results[key])) + self.assertEqual(type(target[key]['data']), type(expected_target_results[key]['data'])) + else: + self.assertEqual(target[key], expected_target_results[key]) + + def test_create_target_from_query(self): + expected_target = self.test_target + + target = self.jpl_ds.create_target_from_query(self.scout_results[0]) + + self.assertEqual(target.name, expected_target.name) + self.assertEqual(target.type, expected_target.type) + self.assertEqual(target.ra, expected_target.ra) + self.assertEqual(target.dec, expected_target.dec) + self.assertEqual(target.scheme, expected_target.scheme) + self.assertEqual(target.epoch_of_elements, expected_target.epoch_of_elements) + self.assertAlmostEqual(target.mean_anomaly, expected_target.mean_anomaly, places=6) From 1fdef286069e32dcd000045bad6e0c92fba08f4f Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Wed, 11 Feb 2026 18:05:53 -0800 Subject: [PATCH 17/29] Update indentation to placate flake8 --- .../tests/data_services/test_jpl.py | 117 +++++++++--------- 1 file changed, 59 insertions(+), 58 deletions(-) diff --git a/tom_dataservices/tests/data_services/test_jpl.py b/tom_dataservices/tests/data_services/test_jpl.py index 966a2c4b8..fbdbc3e78 100644 --- a/tom_dataservices/tests/data_services/test_jpl.py +++ b/tom_dataservices/tests/data_services/test_jpl.py @@ -5,24 +5,25 @@ # from tom_dataservices.tests.factories import scout_resultsFactory from tom_targets.models import Target + class TestScoutDataService(TestCase): """ Test the functionality of the JPL ScoutDataService """ def setUp(self): self.jpl_ds = ScoutDataService() - self.input_parameters = { 'ca_dist_min': None, - 'data_service': 'Scout', - 'geo_score_max': 5, - 'impact_rating_min': None, - 'neo_score_min': None, - 'pha_score_min': None, - 'pos_unc_max': None, - 'pos_unc_min': None, - 'query_name': '', - 'query_save': False, - 'tdes': ''} - self.scout_results = [{ 'uncP1': '1500', + self.input_parameters = {'ca_dist_min': None, + 'data_service': 'Scout', + 'geo_score_max': 5, + 'impact_rating_min': None, + 'neo_score_min': None, + 'pha_score_min': None, + 'pos_unc_max': None, + 'pos_unc_min': None, + 'query_name': '', + 'query_save': False, + 'tdes': ''} + self.scout_results = [{'uncP1': '1500', 'tEphem': '2026-02-11 22:45', 'caDist': '0.98', 'phaScore': 0, @@ -43,58 +44,58 @@ def setUp(self): 'unc': '1400', 'ieoScore': 0, 'orbits': {'data': [[0, - '2461079.712931752', - '8.855752093403347E-01', - '4.998861947435592E-01', - '2461038.529885748', - '1.3898404962804094E+02', - '2.6665272638172337E+02', - '1.4692644033445657E+01', - '25.402927', - '4.34144690247134E-03', - '2.4610794091831E+06', - '1.971147172E-03', - '2.790457665E+01', - '1.278753791E+03', - 0]], - 'count': '1', - 'fields': ['idx', - 'epoch', - 'ec', - 'qr', - 'tp', - 'om', - 'w', - 'inc', - 'H', - 'dca', - 'tca', - 'moid', - 'vinf', - 'geoEcc', - 'impFlag']}, + '2461079.712931752', + '8.855752093403347E-01', + '4.998861947435592E-01', + '2461038.529885748', + '1.3898404962804094E+02', + '2.6665272638172337E+02', + '1.4692644033445657E+01', + '25.402927', + '4.34144690247134E-03', + '2.4610794091831E+06', + '1.971147172E-03', + '2.790457665E+01', + '1.278753791E+03', + 0]], + 'count': '1', + 'fields': ['idx', + 'epoch', + 'ec', + 'qr', + 'tp', + 'om', + 'w', + 'inc', + 'H', + 'dca', + 'tca', + 'moid', + 'vinf', + 'geoEcc', + 'impFlag']}, 'nObs': 4, 'rmsN': '0.12', 'signature': {'source': 'NASA/JPL Scout API', 'version': '1.3'}, 'lastRun': '2026-02-08 13:52', 'neoScore': 100},] - target_params = { 'name': 'ZTF10BL', - 'type': 'NON-SIDEREAL', - 'permissions': 'OPEN', - 'scheme': 'MPC_MINOR_PLANET', - 'epoch_of_elements': 61079.212931752, - 'mean_anomaly': 4.4452481923884894, - 'arg_of_perihelion': 266.65272638172337, - 'eccentricity': 0.8855752093403347, - 'lng_asc_node': 138.98404962804094, - 'inclination': 14.692644033445657, - 'mean_daily_motion': 0.10793879092762414, - 'semimajor_axis': 4.368687867914702, - 'epoch_of_perihelion': 61038.029885748, - 'perihdist': 0.4998861947435592, - 'abs_mag': 25.402927, - 'slope': 0.15} + target_params = {'name': 'ZTF10BL', + 'type': 'NON-SIDEREAL', + 'permissions': 'OPEN', + 'scheme': 'MPC_MINOR_PLANET', + 'epoch_of_elements': 61079.212931752, + 'mean_anomaly': 4.4452481923884894, + 'arg_of_perihelion': 266.65272638172337, + 'eccentricity': 0.8855752093403347, + 'lng_asc_node': 138.98404962804094, + 'inclination': 14.692644033445657, + 'mean_daily_motion': 0.10793879092762414, + 'semimajor_axis': 4.368687867914702, + 'epoch_of_perihelion': 61038.029885748, + 'perihdist': 0.4998861947435592, + 'abs_mag': 25.402927, + 'slope': 0.15} self.test_target = Target.objects.create(**target_params) def test_build_query_parameters_no_target(self): From 933407adc75df6d834f89396a0fdd97a3f0467e1 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 12 Feb 2026 13:23:59 -0800 Subject: [PATCH 18/29] First cut at canary tests for JPL Scout API --- .../tests/data_services/test_jpl.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tom_dataservices/tests/data_services/test_jpl.py b/tom_dataservices/tests/data_services/test_jpl.py index fbdbc3e78..3fa22a693 100644 --- a/tom_dataservices/tests/data_services/test_jpl.py +++ b/tom_dataservices/tests/data_services/test_jpl.py @@ -1,4 +1,4 @@ -from django.test import TestCase +from django.test import tag, TestCase from unittest import mock from tom_dataservices.data_services.jpl import ScoutDataService @@ -154,3 +154,46 @@ def test_create_target_from_query(self): self.assertEqual(target.scheme, expected_target.scheme) self.assertEqual(target.epoch_of_elements, expected_target.epoch_of_elements) self.assertAlmostEqual(target.mean_anomaly, expected_target.mean_anomaly, places=6) + + +@tag('canary') +class TestScoutDataServiceCanary(TestCase): + """Tests that actually hit the JPL Scout API.""" + + def setUp(self): + self.jpl_ds = ScoutDataService() + self.input_parameters = {'ca_dist_min': None, + 'data_service': 'Scout', + 'geo_score_max': 5, + 'impact_rating_min': None, + 'neo_score_min': None, + 'pha_score_min': None, + 'pos_unc_max': None, + 'pos_unc_min': None, + 'query_name': '', + 'query_save': False, + 'tdes': ''} + self.expected_result_keys = ['lastRun', 'neo1kmScore', 'phaScore', 'geocentricScore', 'arc', 'rate', + 'neoScore', 'rating', 'elong', 'uncP1', 'vInf', 'objectName', 'dec', 'H', + 'caDist', 'moid', 'ra', 'unc', 'Vmag', 'nObs', 'rmsN', 'tEphem', + 'tisserandScore', 'ieoScore'] + + def test_boilerplate(self): + self.assertTrue(True) + + def test_query_service(self): + """Test query_service.""" + results = self.jpl_ds.query_service(self.jpl_ds.build_query_parameters(self.input_parameters)) + + self.assertIsNotNone(results) + self.assertIsInstance(results, list) + for key in results[0].keys(): + self.assertIn(key, self.expected_result_keys) + + def test_query_targets_single(self): + """Test query_targets with a single result.""" + pass + + def test_create_target_from_query(self): + """Test create_target_from_query.""" + pass From 20ed3d0715debc40f770ab917716c715cd92baee Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Wed, 4 Mar 2026 10:41:29 -0800 Subject: [PATCH 19/29] Refactor overly complex query_targets() with the aid of Claude --- tom_dataservices/data_services/jpl.py | 154 +++++---- .../tests/data_services/test_jpl.py | 323 +++++++++++++++++- 2 files changed, 411 insertions(+), 66 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 272282966..b905cb0b0 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -135,78 +135,102 @@ def query_service(self, data, **kwargs): logger.warning(msg) return self.query_results + def _get_filter_thresholds(self): + """Extract and normalize filter thresholds from input parameters.""" + p = self.input_parameters + + neo_score_min = p.get('neo_score_min', 0) or 0 + pha_score_min = p.get('pha_score_min', 0) or 0 + geo_score_max = p.get('geo_score_max', 101) or 101 + + default_pos_unc_max = 360 * 60 # 360 degrees (whole sky) as arcmin + pos_unc_min = p.get('pos_unc_min', 0) or 0 + pos_unc_max = p.get('pos_unc_max', default_pos_unc_max) or default_pos_unc_max + + thresholds = { + 'neo_score_min': neo_score_min, + 'pha_score_min': pha_score_min, + 'geo_score_max': geo_score_max, + 'impact_rating_min': p['impact_rating_min'], # May be None intentionally + 'ca_dist_min': p['ca_dist_min'], + 'pos_unc_min': pos_unc_min, + 'pos_unc_max': pos_unc_max, + } + return thresholds + + def _parse_result_values(self, result): + """Parse and coerce numeric fields from a raw Scout result.""" + try: + pos_unc = float(result['unc']) + except (ValueError, TypeError): + pos_unc = 0.0 + + try: + ca_dist = float(result['caDist']) + except (ValueError, TypeError): + ca_dist = None + + return pos_unc, ca_dist + + def _passes_filters(self, result, pos_unc, ca_dist, thresholds): + """Return True if the result passes all filter thresholds.""" + impact_rating_min = thresholds['impact_rating_min'] + + impact_ok = ( + impact_rating_min is None or + (result['rating'] is not None and result['rating'] >= impact_rating_min) + ) + + ca_dist_min = thresholds['ca_dist_min'] + ca_dist_ok = ( + ca_dist_min is None or + (ca_dist is not None and ca_dist <= ca_dist_min) + ) + + return ( + result['neoScore'] >= thresholds['neo_score_min'] and + result['phaScore'] >= thresholds['pha_score_min'] and + result['geocentricScore'] < thresholds['geo_score_max'] and + thresholds['pos_unc_min'] <= pos_unc <= thresholds['pos_unc_max'] and + impact_ok and + ca_dist_ok + ) + + def _fetch_target_data(self, result, query_parameters): + """Return full target data for a result, fetching per-object data if needed.""" + if 'orbits' in result: + # Already a per-object query response + return result + + query_parameters['tdes'] = result['objectName'] + target_parameters = self.build_query_parameters(query_parameters) + target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) + + return target_data[0] if target_data is not None else None + def query_targets(self, query_parameters, **kwargs): """Set up and run a specialized query for retrieving targets from a DataService.""" results = self.query_service(self.build_query_parameters(query_parameters)) - targets = [] - if results is not None and 'error' not in results: - neo_score_min = 0 - if self.input_parameters.get('neo_score_min', 0) is not None: - neo_score_min = self.input_parameters['neo_score_min'] - pha_score_min = 0 - if self.input_parameters.get('pha_score_min', 0) is not None: - pha_score_min = self.input_parameters['pha_score_min'] - geo_score_max = 101 - if self.input_parameters.get('geo_score_max', 101) is not None: - geo_score_max = self.input_parameters['geo_score_max'] - # Might be None if we want all objects irrespective of impact chance - impact_rating_min = self.input_parameters['impact_rating_min'] - ca_dist_min = self.input_parameters['ca_dist_min'] - pos_unc_min = 0 - if self.input_parameters.get('pos_unc_min', 0) is not None: - pos_unc_min = self.input_parameters['pos_unc_min'] - pos_unc_max = 360 * 60 # 360 degrees (whole sky) as arcmin - if self.input_parameters.get('pos_unc_max', pos_unc_max) is not None: - pos_unc_max = self.input_parameters['pos_unc_max'] - - for result in results: - # Filter on the many, many form parameters. - try: - pos_unc = float(result['unc']) - except ValueError: - pos_unc = 0.0 - try: - ca_dist = float(result['caDist']) - except TypeError: - ca_dist = None - # print("neoScore phaScore, geoScore, rating, caDist, posuncmin, posuncmax") - # print(result['neoScore'] >= neo_score_min, result['phaScore'] >= pha_score_min, - # result['geocentricScore'] < geo_score_max, - # ((result['rating'] is not None and - # impact_rating_min is not None and - # result['rating'] >= impact_rating_min) or - # impact_rating_min is None), - # (ca_dist_min is None or - # (ca_dist_min is not None and ca_dist is not None and ca_dist <= ca_dist_min)), - # pos_unc >= pos_unc_min, pos_unc <= pos_unc_max) - if result['neoScore'] >= neo_score_min and result['phaScore'] >= pha_score_min and \ - result['geocentricScore'] < geo_score_max and \ - ((result['rating'] is not None and - impact_rating_min is not None and - result['rating'] >= impact_rating_min - ) or - impact_rating_min is None) and \ - (ca_dist_min is None or - (ca_dist_min is not None and ca_dist is not None and ca_dist <= ca_dist_min)) and \ - pos_unc >= pos_unc_min and pos_unc <= pos_unc_max: - if 'orbits' in result: - # This was a query for a specific target so we already have the needed info - target_data = result - else: - # Fetch per-target data - query_parameters['tdes'] = result['objectName'] - target_parameters = self.build_query_parameters(query_parameters) - target_data = self.query_service(target_parameters, url=self.get_urls('object_url')) - if target_data is not None: - target_data = target_data[0] - targets.append(target_data) - else: + if results is None or 'error' in results: msg = "Error retrieving data from Scout." if query_parameters.get('tdes', '') != '': msg += f" Object {query_parameters['tdes']} is no longer on Scout." - # if request is not None: - # messages.error(request, msg) + return [] + + thresholds = self._get_filter_thresholds() + targets = [] + + for result in results: + pos_unc, ca_dist = self._parse_result_values(result) + + if not self._passes_filters(result, pos_unc, ca_dist, thresholds): + continue + + target_data = self._fetch_target_data(result, query_parameters) + if target_data is not None: + targets.append(target_data) + return targets @classmethod diff --git a/tom_dataservices/tests/data_services/test_jpl.py b/tom_dataservices/tests/data_services/test_jpl.py index 3fa22a693..04448e9f2 100644 --- a/tom_dataservices/tests/data_services/test_jpl.py +++ b/tom_dataservices/tests/data_services/test_jpl.py @@ -1,4 +1,4 @@ -from django.test import tag, TestCase +from django.test import tag, SimpleTestCase, TestCase from unittest import mock from tom_dataservices.data_services.jpl import ScoutDataService @@ -6,6 +6,327 @@ from tom_targets.models import Target +def make_result(overrides=None): + """Return a minimal valid Scout result dict, with optional field overrides.""" + base = { + 'objectName': 'ZTF10BL', + 'neoScore': 100, + 'phaScore': 0, + 'geocentricScore': 1, + 'rating': 2, + 'unc': '1400', + 'caDist': '0.98', + } + if overrides: + base.update(overrides) + return base + + +def make_result_with_orbits(overrides=None): + """Return a Scout result that already contains orbit data (per-object query response).""" + result = make_result(overrides) + result['orbits'] = { + 'fields': ['idx', 'epoch', 'ec', 'qr', 'tp', 'om', 'w', 'inc', 'H', + 'dca', 'tca', 'moid', 'vinf', 'geoEcc', 'impFlag'], + 'data': [[0, '2461079.712931752', '8.855752093403347E-01', '4.998861947435592E-01', + '2461038.529885748', '1.3898404962804094E+02', '2.6665272638172337E+02', + '1.4692644033445657E+01', '25.402927', '4.34144690247134E-03', + '2.4610794091831E+06', '1.971147172E-03', '2.790457665E+01', + '1.278753791E+03', 0]], + 'count': '1', + } + return result + + +class TestGetFilterThresholds(SimpleTestCase): + """Tests for ScoutDataService._get_filter_thresholds()""" + + def setUp(self): + self.ds = ScoutDataService() + + def _set_input_params(self, overrides=None): + params = { + 'neo_score_min': None, + 'pha_score_min': None, + 'geo_score_max': None, + 'impact_rating_min': None, + 'ca_dist_min': None, + 'pos_unc_min': None, + 'pos_unc_max': None, + } + if overrides: + params.update(overrides) + self.ds.input_parameters = params + + def test_all_none_returns_permissive_defaults(self): + """When all optional params are None, defaults should allow everything through.""" + self._set_input_params() + thresholds = self.ds._get_filter_thresholds() + + self.assertEqual(thresholds['neo_score_min'], 0) + self.assertEqual(thresholds['pha_score_min'], 0) + self.assertEqual(thresholds['geo_score_max'], 101) + self.assertEqual(thresholds['pos_unc_min'], 0) + self.assertEqual(thresholds['pos_unc_max'], 360 * 60) + self.assertIsNone(thresholds['impact_rating_min']) + self.assertIsNone(thresholds['ca_dist_min']) + + def test_explicit_values_are_used(self): + """Explicitly set values should be returned as-is.""" + self._set_input_params({ + 'neo_score_min': 50, + 'pha_score_min': 10, + 'geo_score_max': 3, + 'pos_unc_min': 5, + 'pos_unc_max': 120, + 'impact_rating_min': 1, + 'ca_dist_min': 0.5, + }) + thresholds = self.ds._get_filter_thresholds() + + self.assertEqual(thresholds['neo_score_min'], 50) + self.assertEqual(thresholds['pha_score_min'], 10) + self.assertEqual(thresholds['geo_score_max'], 3) + self.assertEqual(thresholds['pos_unc_min'], 5) + self.assertEqual(thresholds['pos_unc_max'], 120) + self.assertEqual(thresholds['impact_rating_min'], 1) + self.assertEqual(thresholds['ca_dist_min'], 0.5) + + def test_zero_values_are_preserved_not_replaced_by_default(self): + """Explicit zero should be kept, not treated as falsy and replaced by a default.""" + self._set_input_params({'neo_score_min': 0, 'pha_score_min': 0, 'pos_unc_min': 0}) + thresholds = self.ds._get_filter_thresholds() + + self.assertEqual(thresholds['neo_score_min'], 0) + self.assertEqual(thresholds['pha_score_min'], 0) + self.assertEqual(thresholds['pos_unc_min'], 0) + + +class TestParseResultValues(SimpleTestCase): + """Tests for ScoutDataService._parse_result_values()""" + + def setUp(self): + self.ds = ScoutDataService() + + def test_valid_numeric_strings(self): + result = make_result({'unc': '1400', 'caDist': '0.98'}) + pos_unc, ca_dist = self.ds._parse_result_values(result) + + self.assertEqual(pos_unc, 1400.0) + self.assertAlmostEqual(ca_dist, 0.98) + + def test_non_numeric_unc_defaults_to_zero(self): + """If 'unc' cannot be cast to float, pos_unc should default to 0.0.""" + result = make_result({'unc': 'N/A'}) + pos_unc, _ = self.ds._parse_result_values(result) + + self.assertEqual(pos_unc, 0.0) + + def test_none_ca_dist_returns_none(self): + """A None 'caDist' (as returned by the API when unknown) should yield None.""" + result = make_result({'caDist': None}) + _, ca_dist = self.ds._parse_result_values(result) + + self.assertIsNone(ca_dist) + + def test_non_numeric_ca_dist_returns_none(self): + """A non-numeric 'caDist' string that raises TypeError should yield None.""" + result = make_result({'caDist': 'unknown'}) + _, ca_dist = self.ds._parse_result_values(result) + + self.assertIsNone(ca_dist) + + +class TestPassesFilters(SimpleTestCase): + """Tests for ScoutDataService._passes_filters()""" + + def setUp(self): + self.ds = ScoutDataService() + # Permissive thresholds that let everything through by default + self.permissive = { + 'neo_score_min': 0, + 'pha_score_min': 0, + 'geo_score_max': 101, + 'impact_rating_min': None, + 'ca_dist_min': None, + 'pos_unc_min': 0, + 'pos_unc_max': 360 * 60, + } + + def _thresholds(self, overrides=None): + t = dict(self.permissive) + if overrides: + t.update(overrides) + return t + + def test_passes_with_permissive_thresholds(self): + result = make_result() + self.assertTrue(self.ds._passes_filters(result, 1400.0, 0.98, self._thresholds())) + + def test_fails_neo_score_below_minimum(self): + result = make_result({'neoScore': 30}) + self.assertFalse(self.ds._passes_filters(result, 0.0, None, self._thresholds({'neo_score_min': 50}))) + + def test_fails_pha_score_below_minimum(self): + result = make_result({'phaScore': 0}) + self.assertFalse(self.ds._passes_filters(result, 0.0, None, self._thresholds({'pha_score_min': 5}))) + + def test_fails_geocentric_score_at_or_above_maximum(self): + result = make_result({'geocentricScore': 5}) + self.assertFalse(self.ds._passes_filters(result, 0.0, None, self._thresholds({'geo_score_max': 5}))) + + def test_passes_geocentric_score_strictly_below_maximum(self): + result = make_result({'geocentricScore': 4}) + self.assertTrue(self.ds._passes_filters(result, 0.0, None, self._thresholds({'geo_score_max': 5}))) + + def test_fails_pos_unc_below_minimum(self): + result = make_result() + self.assertFalse(self.ds._passes_filters(result, 50.0, None, self._thresholds({'pos_unc_min': 100}))) + + def test_fails_pos_unc_above_maximum(self): + result = make_result() + self.assertFalse(self.ds._passes_filters(result, 500.0, None, self._thresholds({'pos_unc_max': 100}))) + + def test_passes_when_impact_rating_min_is_none(self): + """impact_rating_min=None means no impact filter — any rating (or None) should pass.""" + result = make_result({'rating': None}) + self.assertTrue(self.ds._passes_filters(result, 0.0, None, self._thresholds({'impact_rating_min': None}))) + + def test_fails_when_rating_is_none_and_impact_rating_min_is_set(self): + """If a minimum rating is required but the result has no rating, it should be filtered out.""" + result = make_result({'rating': None}) + self.assertFalse(self.ds._passes_filters(result, 0.0, None, self._thresholds({'impact_rating_min': 1}))) + + def test_fails_when_rating_below_minimum(self): + result = make_result({'rating': 1}) + self.assertFalse(self.ds._passes_filters(result, 0.0, None, self._thresholds({'impact_rating_min': 2}))) + + def test_passes_when_ca_dist_min_is_none(self): + """ca_dist_min=None means no close-approach filter.""" + result = make_result() + self.assertTrue(self.ds._passes_filters(result, 0.0, None, self._thresholds({'ca_dist_min': None}))) + + def test_fails_when_ca_dist_exceeds_minimum(self): + """Result should be filtered out when its caDist is greater than the threshold.""" + result = make_result() + self.assertFalse(self.ds._passes_filters(result, 0.0, 1.5, self._thresholds({'ca_dist_min': 1.0}))) + + def test_fails_when_ca_dist_required_but_is_none(self): + """If a ca_dist_min threshold is set but the result has no caDist, filter it out.""" + result = make_result() + self.assertFalse(self.ds._passes_filters(result, 0.0, None, self._thresholds({'ca_dist_min': 1.0}))) + + def test_passes_when_ca_dist_within_minimum(self): + result = make_result() + self.assertTrue(self.ds._passes_filters(result, 0.0, 0.5, self._thresholds({'ca_dist_min': 1.0}))) + + +class TestFetchTargetData(SimpleTestCase): + """Tests for ScoutDataService._fetch_target_data()""" + + def setUp(self): + self.ds = ScoutDataService() + self.query_parameters = {'tdes': ''} + + def test_returns_result_directly_when_orbits_present(self): + """If the result already has orbit data, no additional query_service call should be made.""" + result = make_result_with_orbits() + with mock.patch.object(self.ds, 'query_service') as mock_qs: + target_data = self.ds._fetch_target_data(result, self.query_parameters) + + mock_qs.assert_not_called() + self.assertEqual(target_data, result) + + def test_fetches_per_object_data_when_orbits_absent(self): + """If orbits are not present, query_service should be called to fetch per-object data.""" + result = make_result() # no 'orbits' key + per_object_result = make_result_with_orbits() + + with mock.patch.object(self.ds, 'query_service', return_value=[per_object_result]) as mock_qs: + with mock.patch.object(self.ds, 'get_urls', return_value='http://mock-url'): + target_data = self.ds._fetch_target_data(result, self.query_parameters) + + mock_qs.assert_called_once() + self.assertEqual(target_data, per_object_result) + + def test_sets_tdes_on_query_parameters_when_fetching(self): + """query_parameters['tdes'] should be updated to the result's objectName before fetching.""" + result = make_result({'objectName': 'ZTF10BL'}) + per_object_result = make_result_with_orbits() + + with mock.patch.object(self.ds, 'query_service', return_value=[per_object_result]): + with mock.patch.object(self.ds, 'get_urls', return_value='http://mock-url'): + self.ds._fetch_target_data(result, self.query_parameters) + + self.assertEqual(self.query_parameters['tdes'], 'ZTF10BL') + + def test_returns_none_when_query_service_returns_none(self): + """If the per-object query_service call returns None, _fetch_target_data should return None.""" + result = make_result() # no 'orbits' key + + with mock.patch.object(self.ds, 'query_service', return_value=None): + with mock.patch.object(self.ds, 'get_urls', return_value='http://mock-url'): + target_data = self.ds._fetch_target_data(result, self.query_parameters) + + self.assertIsNone(target_data) + + +class TestQueryTargetsFiltering(TestCase): + """Integration-level tests for query_targets filtering behaviour using mocked query_service.""" + + def setUp(self): + self.ds = ScoutDataService() + self.base_input_parameters = { + 'ca_dist_min': None, + 'data_service': 'Scout', + 'geo_score_max': 5, + 'impact_rating_min': None, + 'neo_score_min': None, + 'pha_score_min': None, + 'pos_unc_max': None, + 'pos_unc_min': None, + 'query_name': '', + 'query_save': False, + 'tdes': 'ZTF10BL', + } + + @mock.patch('tom_dataservices.data_services.jpl.ScoutDataService.query_service') + def test_returns_empty_list_when_query_service_returns_none(self, mock_qs): + mock_qs.return_value = None + targets = self.ds.query_targets(self.base_input_parameters) + self.assertEqual(targets, []) + + @mock.patch('tom_dataservices.data_services.jpl.ScoutDataService.query_service') + def test_result_excluded_by_geocentric_score_filter(self, mock_qs): + """A result with geocentricScore >= geo_score_max should be excluded.""" + result = make_result_with_orbits({'geocentricScore': 5}) # fails geo_score_max=5 + mock_qs.return_value = [result] + + targets = self.ds.query_targets(self.base_input_parameters) + self.assertEqual(targets, []) + + @mock.patch('tom_dataservices.data_services.jpl.ScoutDataService.query_service') + def test_result_included_when_all_filters_pass(self, mock_qs): + """A result that satisfies all default filters should be included.""" + result = make_result_with_orbits({'geocentricScore': 1}) + mock_qs.return_value = [result] + + targets = self.ds.query_targets(self.base_input_parameters) + self.assertEqual(len(targets), 1) + self.assertEqual(targets[0]['objectName'], 'ZTF10BL') + + @mock.patch('tom_dataservices.data_services.jpl.ScoutDataService.query_service') + def test_multiple_results_partial_filter(self, mock_qs): + """Only results passing all filters should be returned from a multi-result response.""" + passing = make_result_with_orbits({'objectName': 'ZTF10BL', 'geocentricScore': 1}) + failing = make_result_with_orbits({'objectName': 'ZTF99XX', 'geocentricScore': 5}) + mock_qs.return_value = [passing, failing] + + targets = self.ds.query_targets(self.base_input_parameters) + self.assertEqual(len(targets), 1) + self.assertEqual(targets[0]['objectName'], 'ZTF10BL') + + class TestScoutDataService(TestCase): """ Test the functionality of the JPL ScoutDataService From 7d2af445d19aa991c679b04c81322ce6d5f3beda Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 5 Mar 2026 14:42:00 -0800 Subject: [PATCH 20/29] Create ScoutDetail model and migration to store metadata about Scout/NEOCP objects --- tom_dataservices/data_services/jpl.py | 62 ++++++++++++++++++- .../migrations/0002_scoutdetail.py | 35 +++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 tom_dataservices/migrations/0002_scoutdetail.py diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index b905cb0b0..1f06f8007 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -2,17 +2,45 @@ from astropy.constants import GM_sun, au from django import forms +from django.db import models import logging import requests # import pprint from tom_dataservices.dataservices import DataService from tom_dataservices.forms import BaseQueryForm -from tom_targets.models import Target +from tom_targets.models import BaseTarget, Target logger = logging.getLogger(__name__) +class ScoutDetail(models.Model): + class ScoutImpactRating(models.IntegerChoices): + NEGLIGIBLE = 0, 'Negligible' + SMALL = 1, 'Small' + MODEST = 2, 'Modest' + MODERATE = 3, 'Moderate' + ELEVATED = 4, 'Elevated' + target = models.OneToOneField(BaseTarget, on_delete=models.CASCADE, related_name='scout_detail') + num_obs = models.IntegerField(null=True, blank=True, help_text='Number of observations') + neo_score = models.IntegerField(null=True, blank=True, help_text='NEO digest score (0..100)') + neo1km_score = models.IntegerField(null=True, blank=True, help_text='NEO >1km digest score (0..100)') + pha_score = models.IntegerField(null=True, blank=True, help_text='PHA digest score (0..100)') + ieo_score = models.IntegerField(null=True, blank=True, help_text='IEO digest score (0..100)') + geocentric_score = models.IntegerField(null=True, blank=True, help_text='Geocentric digest score (0..100)') + impact_rating = models.IntegerField(null=True, blank=True, choices=ScoutImpactRating.choices, + help_text='Impact rating (0=negligible, 1=small, 2=modest, 3=moderate, ' + '4=elevated)') + ca_dist = models.FloatField(null=True, blank=True, help_text='Close approach distance (lunar distances)') + arc = models.FloatField(null=True, blank=True, help_text='Arc length (days)') + rms = models.FloatField(null=True, blank=True, help_text='RMS of the residuals to the orbit fit (arcsec)') + uncertainty = models.FloatField(null=True, blank=True, + help_text='1-sigma plane-of-sky positional uncertainty (arcmin)') + uncertainty_p1 = models.FloatField(null=True, blank=True, + help_text='1-sigma plane-of-sky positional uncertainty at +1 day (arcmin)') + last_run = models.DateTimeField(null=True, blank=True, help_text='Last time the data was updated from Scout') + + class ScoutForm(BaseQueryForm): tdes = forms.CharField(required=False, label='NEOCP temporary designation') @@ -229,6 +257,8 @@ def query_targets(self, query_parameters, **kwargs): target_data = self._fetch_target_data(result, query_parameters) if target_data is not None: + reduced_datums = self._parse_detail_data(target_data) + target_data['reduced_datums']['scout_detail'] = reduced_datums targets.append(target_data) return targets @@ -289,3 +319,33 @@ def create_target_from_query(self, target_results, **kwargs): mean_anomaly += 360.0 target.mean_anomaly = mean_anomaly return target + + def _parse_detail_data(self, query_results, **kwargs): + """Parse and coerce relevant fields from a per-object query result to create a dictionary of reduced datums. + (These aren't really "reduced datums" in the sense of being derived from the raw data, but a + temporary hacky workaround as these are the only things supported post-Target saving) + """ + + reduced_datums = { + 'num_obs': query_results.get('nObs'), + 'neo_score': query_results.get('neoScore'), + 'neo1km_score': query_results.get('neo1kmScore'), + 'pha_score': query_results.get('phaScore'), + 'ieo_score': query_results.get('ieoScore'), + 'geocentric_score': query_results.get('geocentricScore'), + 'impact_rating': query_results.get('rating'), + 'ca_dist': float(query_results.get('caDist')) if query_results.get('caDist') is not None else None, + 'arc': float(query_results.get('arc')) if query_results.get('arc') is not None else None, + 'rms': float(query_results.get('rmsN')) if query_results.get('rmsN') is not None else None, + 'uncertainty': float(query_results.get('unc')) if query_results.get('unc') is not None else None, + 'uncertainty_p1': float(query_results.get('uncP1')) if query_results.get('uncP1') is not None else None, + 'last_run': query_results.get('lastRun') + } + return reduced_datums + + def create_reduced_datums_from_query(self, target, data=None, data_type=None, **kwargs): + if data is not None and data_type == 'scout_detail': + scout_detail, created = ScoutDetail.objects.get_or_create(target=target, **data) + else: + scout_detail, created = None + return scout_detail, created diff --git a/tom_dataservices/migrations/0002_scoutdetail.py b/tom_dataservices/migrations/0002_scoutdetail.py new file mode 100644 index 000000000..e6e15a5ff --- /dev/null +++ b/tom_dataservices/migrations/0002_scoutdetail.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.11 on 2026-03-05 22:32 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_dataservices', '0001_initial'), + ('tom_targets', '0030_alter_basetarget_slope'), + ] + + operations = [ + migrations.CreateModel( + name='ScoutDetail', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('num_obs', models.IntegerField(blank=True, help_text='Number of observations', null=True)), + ('neo_score', models.IntegerField(blank=True, help_text='NEO digest score (0..100)', null=True)), + ('neo1km_score', models.IntegerField(blank=True, help_text='NEO >1km digest score (0..100)', null=True)), + ('pha_score', models.IntegerField(blank=True, help_text='PHA digest score (0..100)', null=True)), + ('ieo_score', models.IntegerField(blank=True, help_text='IEO digest score (0..100)', null=True)), + ('geocentric_score', models.IntegerField(blank=True, help_text='Geocentric digest score (0..100)', null=True)), + ('impact_rating', models.IntegerField(blank=True, choices=[(0, 'Negligible'), (1, 'Small'), (2, 'Modest'), (3, 'Moderate'), (4, 'Elevated')], help_text='Impact rating (0=negligible, 1=small, 2=modest, 3=moderate, 4=elevated)', null=True)), + ('ca_dist', models.FloatField(blank=True, help_text='Close approach distance (lunar distances)', null=True)), + ('arc', models.FloatField(blank=True, help_text='Arc length (days)', null=True)), + ('rms', models.FloatField(blank=True, help_text='RMS of the residuals to the orbit fit (arcsec)', null=True)), + ('uncertainty', models.FloatField(blank=True, help_text='1-sigma plane-of-sky positional uncertainty (arcmin)', null=True)), + ('uncertainty_p1', models.FloatField(blank=True, help_text='1-sigma plane-of-sky positional uncertainty at +1 day (arcmin)', null=True)), + ('last_run', models.DateTimeField(blank=True, help_text='Last time the data was updated from Scout', null=True)), + ('target', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='scout_detail', to='tom_targets.basetarget')), + ], + ), + ] From 7af7cae4570732305507771826dc071c2ce7d991 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 5 Mar 2026 16:43:13 -0800 Subject: [PATCH 21/29] Fix dumb jet-lag-induced bug. Add tests --- tom_dataservices/data_services/jpl.py | 10 ++- .../tests/data_services/test_jpl.py | 77 +++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 1f06f8007..be95cc42f 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -1,4 +1,6 @@ from math import sqrt, degrees +from dateutil.parser import parse +from dateutil.tz import tzutc from astropy.constants import GM_sun, au from django import forms @@ -258,7 +260,8 @@ def query_targets(self, query_parameters, **kwargs): target_data = self._fetch_target_data(result, query_parameters) if target_data is not None: reduced_datums = self._parse_detail_data(target_data) - target_data['reduced_datums']['scout_detail'] = reduced_datums + if reduced_datums is not None: + target_data['reduced_datums'] = {'scout_detail': reduced_datums} targets.append(target_data) return targets @@ -339,7 +342,8 @@ def _parse_detail_data(self, query_results, **kwargs): 'rms': float(query_results.get('rmsN')) if query_results.get('rmsN') is not None else None, 'uncertainty': float(query_results.get('unc')) if query_results.get('unc') is not None else None, 'uncertainty_p1': float(query_results.get('uncP1')) if query_results.get('uncP1') is not None else None, - 'last_run': query_results.get('lastRun') + 'last_run': parse(query_results.get('lastRun')).replace(tzinfo=tzutc()) if query_results.get('lastRun') + else None } return reduced_datums @@ -347,5 +351,5 @@ def create_reduced_datums_from_query(self, target, data=None, data_type=None, ** if data is not None and data_type == 'scout_detail': scout_detail, created = ScoutDetail.objects.get_or_create(target=target, **data) else: - scout_detail, created = None + scout_detail, created = None, None return scout_detail, created diff --git a/tom_dataservices/tests/data_services/test_jpl.py b/tom_dataservices/tests/data_services/test_jpl.py index 04448e9f2..04f9fabe8 100644 --- a/tom_dataservices/tests/data_services/test_jpl.py +++ b/tom_dataservices/tests/data_services/test_jpl.py @@ -1,3 +1,6 @@ +from datetime import datetime +from dateutil.tz import tzutc + from django.test import tag, SimpleTestCase, TestCase from unittest import mock @@ -11,11 +14,18 @@ def make_result(overrides=None): base = { 'objectName': 'ZTF10BL', 'neoScore': 100, + 'neo1kmScore': 0, 'phaScore': 0, + 'ieoScore': 0, 'geocentricScore': 1, 'rating': 2, 'unc': '1400', + 'uncP1': '1500', 'caDist': '0.98', + 'arc': '0.35', + 'nObs': 4, + 'rmsN': '0.12', + 'lastRun': '2026-02-11 22:45', } if overrides: base.update(overrides) @@ -327,6 +337,73 @@ def test_multiple_results_partial_filter(self, mock_qs): self.assertEqual(targets[0]['objectName'], 'ZTF10BL') +class TestParseDetailData(SimpleTestCase): + """Tests for ScoutDataService._parse_detail_data()""" + + def setUp(self): + self.ds = ScoutDataService() + + def test_returns_reduced_datums_dict_with_expected_keys(self): + detail_data = make_result_with_orbits() + reduced_datums = self.ds._parse_detail_data(detail_data) + + self.assertIsInstance(reduced_datums, dict) + + expected_keys = ['num_obs', 'neo_score', 'neo1km_score', 'pha_score', 'ieo_score', 'geocentric_score', + 'impact_rating', 'ca_dist', 'arc', 'rms', 'uncertainty', 'uncertainty_p1', 'last_run'] + + for datum in reduced_datums: + self.assertTrue(datum in expected_keys) + # Check that values are not still strings (e.g. '1400') but have been converted to appropriate + # types (e.g. float 1400.0) + self.assertNotEqual(str, type(reduced_datums[datum])) + + def test_convert_values(self): + detail_data = make_result_with_orbits() + + reduced_datums = self.ds._parse_detail_data(detail_data) + + expected_datums = {'num_obs': 4, + 'neo_score': 100, + 'neo1km_score': 0, + 'pha_score': 0, + 'ieo_score': 0, + 'geocentric_score': 1, + 'impact_rating': 2, + 'ca_dist': 0.98, + 'arc': 0.35, + 'rms': 0.12, + 'uncertainty': 1400.0, + 'uncertainty_p1': 1500.0, + 'last_run': datetime(2026, 2, 11, 22, 45, tzinfo=tzutc()) + } + + self.assertDictEqual(reduced_datums, expected_datums) + + def test_convert_values_more_nones(self): + detail_data = make_result_with_orbits({'rmsN': None, 'unc': None, 'uncP1': None, 'caDist': None, + 'lastRun': None, 'arc': None}) + + reduced_datums = self.ds._parse_detail_data(detail_data) + + expected_datums = {'num_obs': 4, + 'neo_score': 100, + 'neo1km_score': 0, + 'pha_score': 0, + 'ieo_score': 0, + 'geocentric_score': 1, + 'impact_rating': 2, + 'ca_dist': None, + 'arc': None, + 'rms': None, + 'uncertainty': None, + 'uncertainty_p1': None, + 'last_run': None + } + + self.assertDictEqual(reduced_datums, expected_datums) + + class TestScoutDataService(TestCase): """ Test the functionality of the JPL ScoutDataService From 406a905d57d157155f69fa8c23427775185a8b67 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 5 Mar 2026 17:02:11 -0800 Subject: [PATCH 22/29] Add Meta and __str__ method --- tom_dataservices/data_services/jpl.py | 6 ++++++ tom_dataservices/migrations/0002_scoutdetail.py | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index be95cc42f..66a753f1f 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -42,6 +42,12 @@ class ScoutImpactRating(models.IntegerChoices): help_text='1-sigma plane-of-sky positional uncertainty at +1 day (arcmin)') last_run = models.DateTimeField(null=True, blank=True, help_text='Last time the data was updated from Scout') + class Meta: + verbose_name = 'Scout Detail' + + def __str__(self): + return self.target.name + f' (Impact rating: {self.impact_rating})' + class ScoutForm(BaseQueryForm): tdes = forms.CharField(required=False, diff --git a/tom_dataservices/migrations/0002_scoutdetail.py b/tom_dataservices/migrations/0002_scoutdetail.py index e6e15a5ff..5e0605be4 100644 --- a/tom_dataservices/migrations/0002_scoutdetail.py +++ b/tom_dataservices/migrations/0002_scoutdetail.py @@ -1,14 +1,14 @@ -# Generated by Django 5.2.11 on 2026-03-05 22:32 +# Generated by Django 4.2.29 on 2026-03-06 01:01 -import django.db.models.deletion from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ + ('tom_targets', '0032_alter_basetarget_id_alter_persistentshare_id_and_more'), ('tom_dataservices', '0001_initial'), - ('tom_targets', '0030_alter_basetarget_slope'), ] operations = [ @@ -31,5 +31,8 @@ class Migration(migrations.Migration): ('last_run', models.DateTimeField(blank=True, help_text='Last time the data was updated from Scout', null=True)), ('target', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='scout_detail', to='tom_targets.basetarget')), ], + options={ + 'verbose_name': 'Scout Detail', + }, ), ] From cf016d420248b6f6a7947e34d9d258666eebe8a2 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Fri, 6 Mar 2026 09:16:57 -0800 Subject: [PATCH 23/29] Fix migration dependency on phantom 0032 dependency --- tom_dataservices/migrations/0002_scoutdetail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_dataservices/migrations/0002_scoutdetail.py b/tom_dataservices/migrations/0002_scoutdetail.py index 5e0605be4..81a417cb2 100644 --- a/tom_dataservices/migrations/0002_scoutdetail.py +++ b/tom_dataservices/migrations/0002_scoutdetail.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.29 on 2026-03-06 01:01 +# Generated by Django 4.2.29 on 2026-03-06 17:15 from django.db import migrations, models import django.db.models.deletion @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('tom_targets', '0032_alter_basetarget_id_alter_persistentshare_id_and_more'), + ('tom_targets', '0030_alter_basetarget_slope'), ('tom_dataservices', '0001_initial'), ] From ac9f6b42a226f8215012eb5c4d86128fbffe3e47 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Fri, 6 Mar 2026 09:49:57 -0800 Subject: [PATCH 24/29] Change canary test objects to older ones which (hopefully) won't evolve as much and break the tests --- .../tests/harvesters/test_jplhorizons.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/tom_catalogs/tests/harvesters/test_jplhorizons.py b/tom_catalogs/tests/harvesters/test_jplhorizons.py index 097406d07..26f8b734e 100644 --- a/tom_catalogs/tests/harvesters/test_jplhorizons.py +++ b/tom_catalogs/tests/harvesters/test_jplhorizons.py @@ -9,36 +9,37 @@ def setUp(self): self.broker = JPLHorizonsHarvester() def test_query_number_only(self): - self.broker.query('700000') + self.broker.query('69420') target = self.broker.to_target() target.save(names=getattr(target, 'extra_names', [])) # Only test things that are not likely to change (much) with time - self.assertEqual(target.name, '700000 (1994 UX10)') - self.assertEqual(target.names, ['700000 (1994 UX10)']) + self.assertEqual(target.name, '69420 (1995 YA1)') + self.assertEqual(target.names, ['69420 (1995 YA1)']) self.assertEqual(target.type, 'NON_SIDEREAL') self.assertEqual(target.scheme, 'MPC_MINOR_PLANET') self.assertEqual(target.ra, None) self.assertEqual(target.dec, None) - self.assertAlmostEqual(target.eccentricity, 0.093, places=3) - self.assertAlmostEqual(target.inclination, 4.1684084, places=3) - self.assertAlmostEqual(target.semimajor_axis, 2.657503, places=3) - self.assertAlmostEqual(target.abs_mag, 17.76, places=2) + self.assertAlmostEqual(target.eccentricity, 0.255042, places=4) + self.assertAlmostEqual(target.inclination, 13.208657, places=3) + self.assertAlmostEqual(target.semimajor_axis, 2.58835, places=3) + self.assertAlmostEqual(target.abs_mag, 15.09, places=2) self.assertAlmostEqual(target.slope, 0.15, places=2) def test_query_designation_only(self): - self.broker.query('2025 MB18') + self.broker.query('1995 YA1') target = self.broker.to_target() target.save(names=getattr(target, 'extra_names', [])) # Only test things that are not likely to change (much) with time - self.assertEqual(target.name, '(2025 MB18)') - self.assertEqual(target.names, ['(2025 MB18)']) + self.assertEqual(target.name, '69420 (1995 YA1)') + self.assertEqual(target.names, ['69420 (1995 YA1)']) self.assertEqual(target.type, 'NON_SIDEREAL') self.assertEqual(target.scheme, 'MPC_MINOR_PLANET') self.assertEqual(target.ra, None) self.assertEqual(target.dec, None) - self.assertAlmostEqual(target.eccentricity, 0.1386, places=4) - self.assertAlmostEqual(target.inclination, 19.2780, places=4) - self.assertAlmostEqual(target.abs_mag, 24.33, places=2) + self.assertAlmostEqual(target.eccentricity, 0.255042, places=4) + self.assertAlmostEqual(target.inclination, 13.208657, places=3) + self.assertAlmostEqual(target.semimajor_axis, 2.58835, places=3) + self.assertAlmostEqual(target.abs_mag, 15.09, places=2) self.assertAlmostEqual(target.slope, 0.15, places=2) def test_query_name(self): @@ -58,15 +59,15 @@ def test_query_name(self): self.assertAlmostEqual(target.slope, 0.60, places=2) def test_comet_query_desig(self): - self.broker.query('C/2025 A6') + self.broker.query('P/2012 B1') target = self.broker.to_target() target.save(names=getattr(target, 'extra_names', [])) # Only test things that are not likely to change (much) with time - self.assertEqual(target.name, 'Lemmon (C/2025 A6)') - self.assertEqual(target.names, ['Lemmon (C/2025 A6)']) + self.assertEqual(target.name, 'PANSTARRS (P/2012 B1)') + self.assertEqual(target.names, ['PANSTARRS (P/2012 B1)']) self.assertEqual(target.type, 'NON_SIDEREAL') self.assertEqual(target.scheme, 'MPC_COMET') self.assertEqual(target.ra, None) self.assertEqual(target.dec, None) - self.assertAlmostEqual(target.eccentricity, 0.9956, places=4) - self.assertAlmostEqual(target.inclination, 143.663531, places=3) + self.assertAlmostEqual(target.eccentricity, 0.410531, places=4) + self.assertAlmostEqual(target.inclination, 7.605125, places=3) From 9a1bdbfbf10a71f1c71d334986da344ed26759a5 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Fri, 6 Mar 2026 10:12:36 -0800 Subject: [PATCH 25/29] Change canary test objects to older ones which (hopefully) won't evolve as much and break the tests --- tom_catalogs/tests/harvesters/test_mpc.py | 28 +++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tom_catalogs/tests/harvesters/test_mpc.py b/tom_catalogs/tests/harvesters/test_mpc.py index 82135a6ae..6cf364409 100644 --- a/tom_catalogs/tests/harvesters/test_mpc.py +++ b/tom_catalogs/tests/harvesters/test_mpc.py @@ -185,37 +185,37 @@ def setUp(self): self.broker = MPCHarvester() def test_query_number_only(self): - self.broker.query('700000') + self.broker.query('69420') target = self.broker.to_target() target.save(names=getattr(target, 'extra_names', [])) # Only test things that are not likely to change (much) with time - self.assertEqual(target.name, '700000') - self.assertEqual(target.names, ['700000']) + self.assertEqual(target.name, '69420') + self.assertEqual(target.names, ['69420']) self.assertEqual(target.type, 'NON_SIDEREAL') self.assertEqual(target.scheme, 'MPC_MINOR_PLANET') self.assertEqual(target.ra, None) self.assertEqual(target.dec, None) - self.assertAlmostEqual(target.eccentricity, 0.092, places=3) - self.assertAlmostEqual(target.inclination, 4.1688, places=4) - self.assertAlmostEqual(target.mean_anomaly, 315.8420, places=4) - self.assertAlmostEqual(target.semimajor_axis, 2.6555, places=4) - self.assertAlmostEqual(target.abs_mag, 17.76, places=2) + self.assertAlmostEqual(target.eccentricity, 0.255042, places=4) + self.assertAlmostEqual(target.inclination, 13.208657, places=3) + self.assertAlmostEqual(target.semimajor_axis, 2.58835, places=3) + self.assertAlmostEqual(target.abs_mag, 15.09, places=2) self.assertAlmostEqual(target.slope, 0.15, places=2) def test_query_designation_only(self): - self.broker.query('2025 MB18') + self.broker.query('1998 WB2') target = self.broker.to_target() target.save(names=getattr(target, 'extra_names', [])) # Only test things that are not likely to change (much) with time - self.assertEqual(target.name, '2025 MB18') - self.assertEqual(target.names, ['2025 MB18']) + self.assertEqual(target.name, '1998 WB2') + self.assertEqual(target.names, ['1998 WB2']) self.assertEqual(target.type, 'NON_SIDEREAL') self.assertEqual(target.scheme, 'MPC_MINOR_PLANET') self.assertEqual(target.ra, None) self.assertEqual(target.dec, None) - self.assertAlmostEqual(target.eccentricity, 0.1398, places=4) - self.assertAlmostEqual(target.inclination, 19.3561, places=4) - self.assertAlmostEqual(target.abs_mag, 24.33, places=2) + self.assertAlmostEqual(target.eccentricity, 0.586622, places=4) + self.assertAlmostEqual(target.inclination, 2.37248, places=3) + self.assertAlmostEqual(target.semimajor_axis, 1.97247, places=3) + self.assertAlmostEqual(target.abs_mag, 21.85, places=2) self.assertAlmostEqual(target.slope, 0.15, places=2) def test_query_name(self): From 904c708490d22a5c61eeebdebec4f0c3fe2e17b2 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 19 Mar 2026 15:18:26 -0700 Subject: [PATCH 26/29] Restore functionality of advanced form after upstream changes --- tom_dataservices/data_services/jpl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 66a753f1f..1bb756662 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -1,4 +1,5 @@ from math import sqrt, degrees +from typing import List from dateutil.parser import parse from dateutil.tz import tzutc @@ -74,13 +75,16 @@ class ScoutForm(BaseQueryForm): pos_unc_max = forms.FloatField(required=False, label='Maximum positional uncertainty (arcmin)') + def simple_fields(self) -> List[str]: + return ['tdes', 'neo_score_min'] + class ScoutDataService(DataService): """ Docstring for ScoutDataService """ name = 'Scout' - app_version = '0.0.3' + app_version = '0.0.4' info_url = 'https://cneos.jpl.nasa.gov/scout/intro.html' query_results_table = 'tom_dataservices/scout/partials/scout_query_results_table.html' expected_signature = {'source': 'NASA/JPL Scout API', 'version': '1.3'} From 5e151badb4e266cc21b0ecfc97ee73b190913fc1 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 19 Mar 2026 16:17:44 -0700 Subject: [PATCH 27/29] Remove now unneeded partials --- tom_dataservices/data_services/jpl.py | 6 ------ .../scout/partials/scout_advanced_form.html | 7 ------- .../tom_dataservices/scout/partials/scout_simple_form.html | 3 --- 3 files changed, 16 deletions(-) delete mode 100644 tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html delete mode 100644 tom_dataservices/templates/tom_dataservices/scout/partials/scout_simple_form.html diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 1bb756662..1754ad531 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -102,12 +102,6 @@ def urls(cls, **kwargs) -> dict: urls['search_url'] = urls['base_url'] return urls - def get_simple_form_partial(self): - return 'tom_dataservices/scout/partials/scout_simple_form.html' - - def get_advanced_form_partial(self): - return 'tom_dataservices/scout/partials/scout_advanced_form.html' - def build_query_parameters(self, parameters, **kwargs): """ Args: diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html deleted file mode 100644 index ba17b533c..000000000 --- a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_advanced_form.html +++ /dev/null @@ -1,7 +0,0 @@ -{% load bootstrap4 %} -{% bootstrap_field form.pha_score_min %} -{% bootstrap_field form.geo_score_max %} -{% bootstrap_field form.impact_rating_min %} -{% bootstrap_field form.ca_dist_min %} -{% bootstrap_field form.pos_unc_min %} -{% bootstrap_field form.pos_unc_max %} diff --git a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_simple_form.html b/tom_dataservices/templates/tom_dataservices/scout/partials/scout_simple_form.html deleted file mode 100644 index 1f0ff9635..000000000 --- a/tom_dataservices/templates/tom_dataservices/scout/partials/scout_simple_form.html +++ /dev/null @@ -1,3 +0,0 @@ -{% load bootstrap4 %} -{% bootstrap_field form.tdes %} -{% bootstrap_field form.neo_score_min %} From ca41c4055afbbacd2a9f15f20cbc43dfd9de9ac4 Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 19 Mar 2026 16:21:36 -0700 Subject: [PATCH 28/29] Fix filtering logic bug that prevented filtering on geoScore=0. Update tests --- tom_dataservices/data_services/jpl.py | 5 ++- .../tests/data_services/test_jpl.py | 43 ++++++++++++++++--- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 1754ad531..20ba48efd 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -175,7 +175,8 @@ def _get_filter_thresholds(self): neo_score_min = p.get('neo_score_min', 0) or 0 pha_score_min = p.get('pha_score_min', 0) or 0 - geo_score_max = p.get('geo_score_max', 101) or 101 + # Needs to be different as code like above will treat 0 as "no value provided" and use the default, but 0 is a valid value for this parameter and should be used if provided. + geo_score_max = p['geo_score_max'] if p.get('geo_score_max') is not None else 101 default_pos_unc_max = 360 * 60 # 360 degrees (whole sky) as arcmin pos_unc_min = p.get('pos_unc_min', 0) or 0 @@ -224,7 +225,7 @@ def _passes_filters(self, result, pos_unc, ca_dist, thresholds): return ( result['neoScore'] >= thresholds['neo_score_min'] and result['phaScore'] >= thresholds['pha_score_min'] and - result['geocentricScore'] < thresholds['geo_score_max'] and + result['geocentricScore'] <= thresholds['geo_score_max'] and thresholds['pos_unc_min'] <= pos_unc <= thresholds['pos_unc_max'] and impact_ok and ca_dist_ok diff --git a/tom_dataservices/tests/data_services/test_jpl.py b/tom_dataservices/tests/data_services/test_jpl.py index 04f9fabe8..e0c1802d3 100644 --- a/tom_dataservices/tests/data_services/test_jpl.py +++ b/tom_dataservices/tests/data_services/test_jpl.py @@ -181,14 +181,18 @@ def test_fails_pha_score_below_minimum(self): result = make_result({'phaScore': 0}) self.assertFalse(self.ds._passes_filters(result, 0.0, None, self._thresholds({'pha_score_min': 5}))) - def test_fails_geocentric_score_at_or_above_maximum(self): - result = make_result({'geocentricScore': 5}) + def test_fails_geocentric_score_above_maximum(self): + result = make_result({'geocentricScore': 6}) self.assertFalse(self.ds._passes_filters(result, 0.0, None, self._thresholds({'geo_score_max': 5}))) def test_passes_geocentric_score_strictly_below_maximum(self): result = make_result({'geocentricScore': 4}) self.assertTrue(self.ds._passes_filters(result, 0.0, None, self._thresholds({'geo_score_max': 5}))) + def test_passes_geocentric_score_of_zero(self): + result = make_result({'geocentricScore': 0}) + self.assertTrue(self.ds._passes_filters(result, 0.0, None, self._thresholds({'geo_score_max': 5}))) + def test_fails_pos_unc_below_minimum(self): result = make_result() self.assertFalse(self.ds._passes_filters(result, 50.0, None, self._thresholds({'pos_unc_min': 100}))) @@ -308,8 +312,8 @@ def test_returns_empty_list_when_query_service_returns_none(self, mock_qs): @mock.patch('tom_dataservices.data_services.jpl.ScoutDataService.query_service') def test_result_excluded_by_geocentric_score_filter(self, mock_qs): - """A result with geocentricScore >= geo_score_max should be excluded.""" - result = make_result_with_orbits({'geocentricScore': 5}) # fails geo_score_max=5 + """A result with geocentricScore > geo_score_max should be excluded.""" + result = make_result_with_orbits({'geocentricScore': 6}) # fails geo_score_max=5 mock_qs.return_value = [result] targets = self.ds.query_targets(self.base_input_parameters) @@ -329,7 +333,7 @@ def test_result_included_when_all_filters_pass(self, mock_qs): def test_multiple_results_partial_filter(self, mock_qs): """Only results passing all filters should be returned from a multi-result response.""" passing = make_result_with_orbits({'objectName': 'ZTF10BL', 'geocentricScore': 1}) - failing = make_result_with_orbits({'objectName': 'ZTF99XX', 'geocentricScore': 5}) + failing = make_result_with_orbits({'objectName': 'ZTF99XX', 'geocentricScore': 6}) mock_qs.return_value = [passing, failing] targets = self.ds.query_targets(self.base_input_parameters) @@ -540,6 +544,33 @@ def test_query_targets_single(self, mock_client): else: self.assertEqual(target[key], expected_target_results[key]) + @mock.patch('tom_dataservices.data_services.jpl.ScoutDataService.query_service') + def test_query_targets_filter_on_geo0(self, mock_client): + first_result = self.scout_results[0].copy() + first_result['geocentricScore'] = 0 # passes geo_score_max=0 cut + second_result = self.scout_results[0].copy() + second_result['objectName'] = 'ZTF99XX' + mock_client.side_effect = [[first_result, second_result], ] + self.input_parameters['geo_score_max'] = 0 + + targets = self.jpl_ds.query_targets(self.input_parameters) + self.assertEqual(len(targets), 1) + expected_target_results = {'objectName': first_result['objectName'], + 'neoScore': first_result['neoScore'], + 'phaScore': first_result['phaScore'], + 'geocentricScore': first_result['geocentricScore'], + 'rating': first_result['rating'], + 'unc': first_result['unc'], + 'orbits': first_result['orbits'], + } + for target in targets: + for key in expected_target_results.keys(): + if key == 'orbits': + self.assertEqual(type(target[key]), type(expected_target_results[key])) + self.assertEqual(type(target[key]['data']), type(expected_target_results[key]['data'])) + else: + self.assertEqual(target[key], expected_target_results[key]) + def test_create_target_from_query(self): expected_target = self.test_target @@ -585,7 +616,7 @@ def test_query_service(self): self.assertIsNotNone(results) self.assertIsInstance(results, list) - for key in results[0].keys(): + for key in results[0].keys(): # type: ignore self.assertIn(key, self.expected_result_keys) def test_query_targets_single(self): From 84882f55604c6d41bccc5a458d61645d5ad9affa Mon Sep 17 00:00:00 2001 From: Tim Lister Date: Thu, 19 Mar 2026 16:24:36 -0700 Subject: [PATCH 29/29] Lint fixes --- tom_dataservices/data_services/jpl.py | 3 ++- tom_dataservices/tests/data_services/test_jpl.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tom_dataservices/data_services/jpl.py b/tom_dataservices/data_services/jpl.py index 20ba48efd..9337514a1 100644 --- a/tom_dataservices/data_services/jpl.py +++ b/tom_dataservices/data_services/jpl.py @@ -175,7 +175,8 @@ def _get_filter_thresholds(self): neo_score_min = p.get('neo_score_min', 0) or 0 pha_score_min = p.get('pha_score_min', 0) or 0 - # Needs to be different as code like above will treat 0 as "no value provided" and use the default, but 0 is a valid value for this parameter and should be used if provided. + # Needs to be different as code like above will treat 0 as "no value provided" and use the default, but 0 is a + # valid value for this parameter and should be used if provided. geo_score_max = p['geo_score_max'] if p.get('geo_score_max') is not None else 101 default_pos_unc_max = 360 * 60 # 360 degrees (whole sky) as arcmin diff --git a/tom_dataservices/tests/data_services/test_jpl.py b/tom_dataservices/tests/data_services/test_jpl.py index e0c1802d3..6adc8c806 100644 --- a/tom_dataservices/tests/data_services/test_jpl.py +++ b/tom_dataservices/tests/data_services/test_jpl.py @@ -616,7 +616,7 @@ def test_query_service(self): self.assertIsNotNone(results) self.assertIsInstance(results, list) - for key in results[0].keys(): # type: ignore + for key in results[0].keys(): # type: ignore self.assertIn(key, self.expected_result_keys) def test_query_targets_single(self):
POS Unc(') H Impact RatingCA Distance (LD) NEO PHA IEO{{ result.unc }} {{ result.H }} {{ result.rating }}{{ result.caDist }} {{ result.neoScore }} {{ result.phaScore }} {{ result.ieoScore }}