Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci_testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-13]
python-version: ["3.9", "3.10", "3.11"]
python-version: ["3.10", "3.11"]
defaults:
run:
shell: bash -l {0}
Expand Down
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ release.

## [Unreleased]

### Added
- Added optional boolean flag `remove_kernels` to the `props` dict in the `loads()` func to allow the user to add the `kernels` key in the output ISD. Added the `misc` kernel key for user-entered kernels. Format the kernels value to always be dict type in ISD generation. [#675](https://github.com/DOI-USGS/ale/pull/675)

## [1.0.2]

- Fixed bug where generic CH2 kernels were using for TMC-2 [#672](https://github.com/DOI-USGS/ale/pull/672)
Expand Down
14 changes: 13 additions & 1 deletion ale/base/data_naif.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,19 @@ def kernels(self):
try:
self._kernels = kernel_access.get_kernels_from_isis_pvl(self._props['kernels'])
except Exception as e:
self._kernels = self._props['kernels']
if isinstance(self._props['kernels'], list):
self._kernels = { "misc": self._props['kernels'] }
elif isinstance(self._props['kernels'], dict):
for k,v in self._props['kernels'].items():
# check that the keys are valid kernel types
if k not in ["ck", "spk", "tspk", "fk", "ik", "iak", "pck", "lsk", "sclk"]:
raise ValueError(f"Unknown kernel type: {k}")
# check that kernel values are in list format
if not isinstance(v, list):
raise ValueError(f"Kernels for {k} must be in list format.")
self._kernels = self._props['kernels']
else:
self._kernels = self._props['kernels']
elif self.search_kernels == True:
_, kernels = pyspiceql.searchForKernelsets([self.spiceql_mission, self.target_name, "base"], startTime=self.ephemeris_start_time, stopTime=self.ephemeris_stop_time, useWeb=self.use_web)
self._kernels = kernels
Expand Down
2 changes: 2 additions & 0 deletions ale/drivers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,8 @@ def load(label, props={}, formatter='ale', verbose=False, only_isis_spice=False,
res.instrument_id
with res as driver:
isd = formatter(driver)
if 'attach_kernels' in props and props['attach_kernels'] is False and 'kernels' in isd:
del isd['kernels']
if verbose:
logger.info(f"Success with: {driver.__class__.__name__}")
logger.info(f"ISD:\n{json.dumps(isd, indent=2, cls=AleJsonEncoder)}")
Expand Down
32 changes: 25 additions & 7 deletions ale/formatters/formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,12 +207,30 @@ def to_isd(driver):
raise Exception('No CSM sensor model name found!')

# remove extra qualities
# TODO: Rewuires SpiceQL API update to get relative kernels
# if driver.kernels and isinstance(driver.kernels, dict):
# isd["kernels"] = {k: v for k, v in driver.kernels.items() if not "_quality" in k or driver.spiceql_mission in k }
# elif driver.kernels and isinstance(driver.kernels, list):
# isd["kernels"] = driver.kernels
# else:
# isd["kernels"] = {}
if 'kernels' in driver_data and isinstance(driver.kernels, dict):
for k,v in driver.kernels.items():
# check that the keys are valid kernel types or quality types
if k not in ["ck", "spk", "tspk", "fk", "ik", "iak", "pck", "lsk", "sclk", "misc", driver.spiceql_mission+"_spk_quality", driver.spiceql_mission+"_ck_quality"]:
raise ValueError(f"Kernels key [{k}] is not valid.")
if k in ["ck", "spk", "tspk", "fk", "ik", "iak", "pck", "lsk", "sclk", "misc"]:
if not isinstance(v, list):
raise ValueError(f"Kernel [{k}] must be set to a list of kernels.")
isd["kernels"] = driver.kernels
elif k in [driver.spiceql_mission+"_spk_quality", driver.spiceql_mission+"_ck_quality"]:
if not isinstance(v, str):
raise ValueError(f"Kernel quality type [{k}] must be set to a string.")
isd["kernels"] = driver.kernels #{k: v for k, v in driver.kernels.items() if not "_quality" in k and not driver.spiceql_mission in k }
elif 'kernels' in driver_data and isinstance(driver.kernels, list):
kernels_dict = {}
for k in driver.kernels:
k_split = k.split('/')
k_type = k_split[2]
if k_type in kernels_dict and isinstance(kernels_dict[k_type], list):
kernels_dict[k_type].append(k)
else:
kernels_dict.update({k_type: [k]})
Comment on lines +224 to +231
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am guessing this is to try to identify the kernel types? Might be best to make that a function in SpiceQL, since SpiceQL can use the Config to identify them more effectively. Short term I think it's fine to leave this here.

isd["kernels"] = kernels_dict
else:
isd["kernels"] = {}

return isd
4 changes: 2 additions & 2 deletions environment.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: ale
# name: ale
channels:
- conda-forge

Expand All @@ -13,7 +13,7 @@ dependencies:
- networkx
- nlohmann_json
- numpy
- spiceql>=1.2.4
- spiceql=1.2.4=*_6
- pvl
- pytest
- pytest-cov
Expand Down
4 changes: 2 additions & 2 deletions tests/pytests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,10 +218,10 @@ def convert_kernels(kernels):
warnings.warn('Failed to convert transfer kernel, ' + kernel + ', skipping...')
else:
kernel = matches.group(1)
binary_kernels.append(kernel)
binary_kernels.append(os.path.abspath(kernel))
except:
raise Exception(f"Unable to convert {path} to binary kernel")
updated_kernels.append(kernel)
updated_kernels.append(os.path.abspath(kernel))

# Sort Kernels
# Ensure that the ISIS Addendum kernel is last in case it overrides
Expand Down
35 changes: 34 additions & 1 deletion tests/pytests/test_apollo_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,47 @@ def test_kernels():
def test_load(test_kernels):
label_file = get_image_label('AS15-M-1450', 'isis3')
compare_dict = get_isd("apollometric")
isd_str = ale.loads(label_file, props={'kernels': test_kernels}, verbose=True)
isd_str = ale.loads(label_file, props={'kernels': test_kernels, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj, indent=2))
print("======================")
print(json.dumps(compare_dict, indent=2))
assert compare_dicts(isd_obj, compare_dict) == []


def test_load_kernels(test_kernels):
label_file = get_image_label('AS15-M-1450', 'isis3')
compare_dict = get_isd("apollometric")
isd_str = ale.loads(label_file, props={'kernels': test_kernels}, verbose=True)
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj, indent=2))
print("======================")
print(json.dumps(compare_dict, indent=2))
assert isd_obj['kernels']
assert type(isd_obj['kernels']) == dict
assert isd_obj['kernels']['misc']
assert type(isd_obj['kernels']['misc']) == list


def test_bad_kernels():
label_file = get_image_label('AS15-M-1450', 'isis3')

# test invalid kernel type
kernels_dict = {"ck": "test_ck", "spk": "test_spk", "unknown": "test_unknown"}
with pytest.raises(Exception) as e:
isd_str = ale.loads(label_file, props={'kernels': kernels_dict}, verbose=True)

# test invalid kernel quality type
kernels_dict = {"ck": "test_ck", "spk": "test_spk", "abc_ck_quality": "test_unknown_quality"}
with pytest.raises(Exception) as e:
isd_str = ale.loads(label_file, props={'kernels': kernels_dict}, verbose=True)

# test invalid kernel value
kernels_dict = {"ck": "test_ck", "spk": "test_spk"}
with pytest.raises(Exception) as e:
isd_str = ale.loads(label_file, props={'kernels': kernels_dict}, verbose=True)


class test_isis3_naif(unittest.TestCase):
def setUp(self):
label = get_image_label("AS15-M-1450", "isis3")
Expand Down
8 changes: 4 additions & 4 deletions tests/pytests/test_cassini_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def test_load_pds(test_iss_kernels):
label_file = get_image_label("N1702360370_1")
compare_dict = get_isd("cassiniiss")

isd_str = ale.loads(label_file, props={'kernels': test_iss_kernels}, verbose=True)
isd_str = ale.loads(label_file, props={'kernels': test_iss_kernels, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj, indent=2))
assert compare_dicts(isd_obj, compare_dict) == []
Expand All @@ -42,7 +42,7 @@ def read_detached_table(table_label, cube):
return get_table_data("N1702360370_1", table_label["Name"])

with patch('ale.base.data_isis.read_table_data', side_effect=read_detached_table):
isd_str = ale.loads(label_file)
isd_str = ale.loads(label_file, props={'attach_kernels': False})
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj, indent=2))
x = compare_dicts(isd_obj, compare_dict)
Expand All @@ -52,7 +52,7 @@ def test_load_iss_isis_naif(test_iss_kernels):
label_file = get_image_label("N1702360370_1")
compare_dict = get_isd("cassiniiss")

isd_str = ale.loads(label_file, props={"kernels": test_iss_kernels})
isd_str = ale.loads(label_file, props={"kernels": test_iss_kernels, 'attach_kernels': False})
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj, indent=2))
x = compare_dicts(isd_obj, compare_dict)
Expand All @@ -62,7 +62,7 @@ def test_load_vims_isis_naif(test_vims_kernels):
label_file = get_image_label("v1514284191_1_vis", label_type="isis")
compare_dict = get_isd("cassinivims")

isd_str = ale.loads(label_file, props={"kernels": test_vims_kernels}, verbose=True)
isd_str = ale.loads(label_file, props={"kernels": test_vims_kernels, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj, indent=2))
x = compare_dicts(isd_obj, compare_dict)
Expand Down
6 changes: 3 additions & 3 deletions tests/pytests/test_chandrayaan_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def test_chandrayaan_load(m3_kernels):
label_file = get_image_label("M3T20090630T083407_V03_RDN", label_type="isis")
compare_dict = get_isd("chandrayannM3")
print("kernels: ", m3_kernels)
isd_str = ale.loads(label_file, props={"kernels": m3_kernels}, verbose=False)
isd_str = ale.loads(label_file, props={"kernels": m3_kernels, 'attach_kernels': False}, verbose=False)
isd_obj = json.loads(isd_str)
print(isd_str)
x = compare_dicts(isd_obj, compare_dict)
Expand All @@ -43,7 +43,7 @@ def test_chandrayaan_mrffr_load(mrffr_kernels):
label_file = get_image_label("fsb_00720_1cd_xhu_84n209_v1", label_type="isis3")
compare_dict = get_isd("chandrayaan_mrffr")

isd_str = ale.loads(label_file, props={"kernels": mrffr_kernels, "nadir": True}, verbose=True)
isd_str = ale.loads(label_file, props={"kernels": mrffr_kernels, "nadir": True, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
x = compare_dicts(isd_obj, compare_dict)
assert x == []
Expand All @@ -54,7 +54,7 @@ def test_chandrayaan_m3_pds_load(m3_kernels):

# Patch the full path of the timing table onto the driver
with patch("ale.drivers.chandrayaan_drivers.Chandrayaan1M3Pds3NaifSpiceDriver.utc_time_table", os.path.dirname(label_file)+"/M3T20090630T083407_V03_TIM_cropped.TAB"):
isd_str = ale.loads(label_file, props={"kernels": m3_kernels, "nadir": True})
isd_str = ale.loads(label_file, props={"kernels": m3_kernels, "nadir": True, 'attach_kernels': False})
isd_obj = json.loads(isd_str)
print(isd_obj)
print(compare_dicts(isd_obj, compare_dict))
Expand Down
8 changes: 4 additions & 4 deletions tests/pytests/test_clementine_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def test_uvvis_kernels():

def test_uvvis_load(test_uvvis_kernels):
label_file = get_image_label('LUA3107H.161', 'isis3')
isd_str = ale.loads(label_file, props={'kernels': test_uvvis_kernels, 'exact_ck_times': False})
isd_str = ale.loads(label_file, props={'kernels': test_uvvis_kernels, 'exact_ck_times': False, 'attach_kernels': False})
isd_obj = json.loads(isd_str)
compare_isd = get_isd('clem_uvvis')
assert compare_dicts(isd_obj, compare_isd) == []
Expand All @@ -35,7 +35,7 @@ def test_hires_kernels():

def test_hires_load(test_hires_kernels):
label_file = get_image_label('LHA0775Q.001', 'isis3')
isd_str = ale.loads(label_file, props={'kernels': test_hires_kernels, 'exact_ck_times': False}, verbose=True)
isd_str = ale.loads(label_file, props={'kernels': test_hires_kernels, 'exact_ck_times': False, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
compare_isd = get_isd('clem_hires')
assert compare_dicts(isd_obj, compare_isd) == []
Expand All @@ -50,7 +50,7 @@ def test_nir_kernels():

def test_nir_load(test_nir_kernels):
label_file = get_image_label('LNB4653M.093', 'isis3')
isd_str = ale.loads(label_file, props={'kernels': test_nir_kernels, 'exact_ck_times': False}, verbose=True)
isd_str = ale.loads(label_file, props={'kernels': test_nir_kernels, 'exact_ck_times': False, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
compare_isd = get_isd('clem_nir')
assert compare_dicts(isd_obj, compare_isd) == []
Expand All @@ -65,7 +65,7 @@ def test_lwir_kernels():

def test_lwir_load(test_lwir_kernels):
label_file = get_image_label('LLA5391Q.209', 'isis3')
isd_str = ale.loads(label_file, props={'kernels': test_lwir_kernels, 'exact_ck_times': False}, verbose=True)
isd_str = ale.loads(label_file, props={'kernels': test_lwir_kernels, 'exact_ck_times': False, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
compare_isd = get_isd('clem_lwir')
assert compare_dicts(isd_obj, compare_isd) == []
Expand Down
4 changes: 2 additions & 2 deletions tests/pytests/test_dawn_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def test_fc_load(fc_kernels, label_type):

label_file = get_image_label(label_prefix_file, label_type=label_type)

isd_str = ale.loads(label_file, props={'kernels': fc_kernels})
isd_str = ale.loads(label_file, props={'kernels': fc_kernels, 'attach_kernels': False})
isd_obj = json.loads(isd_str)
# print(json.dumps(isd_obj, indent=2))
assert compare_dicts(isd_obj, compare_dict) == []
Expand All @@ -79,7 +79,7 @@ def test_vir_load(vir_kernels):

compare_dict = get_isd("dawnvir")

isd_str = ale.loads(label_file, props={"kernels": vir_kernels, "nadir": False}, verbose=False)
isd_str = ale.loads(label_file, props={"kernels": vir_kernels, "nadir": False, 'attach_kernels': False}, verbose=False)
print(isd_str)
isd_obj = json.loads(isd_str)
x = compare_dicts(isd_obj, compare_dict)
Expand Down
2 changes: 1 addition & 1 deletion tests/pytests/test_galileo_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def test_galileo_load(test_kernels, label_type, kernel_type):
isd_str = ale.loads(label_file)
compare_isd = get_isd('galileossi_isis')
else:
isd_str = ale.loads(label_file, props={'kernels': test_kernels})
isd_str = ale.loads(label_file, props={'kernels': test_kernels, 'attach_kernels': False})
compare_isd = get_isd('galileossi')

isd_obj = json.loads(isd_str)
Expand Down
2 changes: 1 addition & 1 deletion tests/pytests/test_hayabusa2_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def test_kernels():
@pytest.mark.parametrize("label_type", ['isis3'])
def test_hayabusa_load(test_kernels, label_type):
label_file = get_image_label('hyb2_onc_20151203_084458_w2f_l2a', label_type)
isd_str = ale.loads(label_file, props={'kernels': test_kernels}, verbose=False)
isd_str = ale.loads(label_file, props={'kernels': test_kernels, 'attach_kernels': False}, verbose=False)
isd_obj = json.loads(isd_str)
compare_dict = get_isd('hayabusa2')
print(json.dumps(isd_obj, indent=2))
Expand Down
4 changes: 2 additions & 2 deletions tests/pytests/test_hayabusa_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def test_amica_load(test_amica_kernels):
label_file = get_image_label('st_2458542208_v', 'isis')
compare_dict = get_isd("hayabusaamica")

isd_str = ale.loads(label_file, props={'kernels': test_amica_kernels})
isd_str = ale.loads(label_file, props={'kernels': test_amica_kernels, 'attach_kernels': False})
isd_obj = json.loads(isd_str)
assert compare_dicts(isd_obj, compare_dict) == []

Expand Down Expand Up @@ -59,7 +59,7 @@ def test_nirs_load(test_nirs_kernels):
label_file = get_image_label('2392975548_lvl3_0', 'isis')
compare_dict = get_isd("hayabusanirs")

isd_str = ale.loads(label_file, props={'kernels': test_nirs_kernels}, verbose=True)
isd_str = ale.loads(label_file, props={'kernels': test_nirs_kernels, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
assert compare_dicts(isd_obj, compare_dict) == []

Expand Down
2 changes: 1 addition & 1 deletion tests/pytests/test_juno_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def test_kernels():
@pytest.mark.parametrize("label_type", ['isis3'])
def test_juno_load(test_kernels, label_type):
label_file = get_image_label('JNCR_2016240_01M06152_V01', label_type)
isd_str = ale.loads(label_file, props={'kernels': test_kernels})
isd_str = ale.loads(label_file, props={'kernels': test_kernels, 'attach_kernels': False})
isd_obj = json.loads(isd_str)
compare_dict = get_isd('juno')
# print(json.dumps(isd_obj, indent=2))
Expand Down
2 changes: 1 addition & 1 deletion tests/pytests/test_kaguya_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def test_kaguya_load(test_kernels, label_type, image):
compare_isd = get_isd(image_dict[image])
label_file = get_image_label(image, label_type)

isd_str = ale.loads(label_file, props={'kernels': test_kernels[image]}, verbose=False)
isd_str = ale.loads(label_file, props={'kernels': test_kernels[image], 'attach_kernels': False}, verbose=False)
isd_obj = json.loads(isd_str)

assert compare_dicts(isd_obj, compare_isd) == []
Expand Down
4 changes: 2 additions & 2 deletions tests/pytests/test_lo_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def test_high_load(test_high_kernels):
label_file = get_image_label('3133_high_res_1', 'isis')
compare_dict = get_isd("lohighcamera")

isd_str = ale.loads(label_file, props={'kernels': test_high_kernels}, verbose=False)
isd_str = ale.loads(label_file, props={'kernels': test_high_kernels, 'attach_kernels': False}, verbose=False)
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj, indent=2))
assert compare_dicts(isd_obj, compare_dict) == []
Expand Down Expand Up @@ -109,7 +109,7 @@ def test_medium_load(test_medium_kernels):
label_file = get_image_label('3133_med_res', 'isis')
compare_dict = get_isd("lomediumcamera")

isd_str = ale.loads(label_file, props={'kernels': test_medium_kernels}, verbose=True)
isd_str = ale.loads(label_file, props={'kernels': test_medium_kernels, 'attach_kernels': False}, verbose=True)
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj, indent=2))
assert compare_dicts(isd_obj, compare_dict) == []
Expand Down
8 changes: 4 additions & 4 deletions tests/pytests/test_lro_drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,11 @@ def test_kernels():
def test_load_lroc_nac(test_kernels, label_type, image, kernel_type):
if kernel_type == 'naif':
label_file = get_image_label(image, label_type)
isd_str = ale.loads(label_file, props={'kernels': test_kernels[image]}, verbose=False)
isd_str = ale.loads(label_file, props={'kernels': test_kernels[image], 'attach_kernels': False}, verbose=False)
compare_isd = image_dict[image]
else:
label_file = get_image(image)
isd_str = ale.loads(label_file)
isd_str = ale.loads(label_file, props={'attach_kernels': False})
compare_isd = get_isd('lro_isis')
print(isd_str)
isd_obj = json.loads(isd_str)
Expand All @@ -62,7 +62,7 @@ def test_load_lroc_nac(test_kernels, label_type, image, kernel_type):
@pytest.mark.parametrize("image", ['wac0000a1c4.uv.even'])
def test_load_lroc_wac(test_kernels, label_type, image, kernel_type):
label_file = get_image_label(image, label_type)
isd_str = ale.loads(label_file, props={'kernels': test_kernels[image]})
isd_str = ale.loads(label_file, props={'kernels': test_kernels[image], 'attach_kernels': False})
compare_isd = image_dict[image]
isd_obj = json.loads(isd_str)
print(json.dumps(isd_obj))
Expand All @@ -73,7 +73,7 @@ def test_load_lroc_wac(test_kernels, label_type, image, kernel_type):
# Test load of MiniRF labels
def test_load_minirf(test_kernels):
label_file = get_image_label('03821_16N196_S1', 'isis3')
isd_str = ale.loads(label_file, props={'kernels': test_kernels['03821_16N196_S1']})
isd_str = ale.loads(label_file, props={'kernels': test_kernels['03821_16N196_S1'], 'attach_kernels': False})
isd_obj = json.loads(isd_str)
comparison = compare_dicts(isd_obj, image_dict['03821_16N196_S1'])
assert comparison == []
Expand Down
Loading
Loading