From d26721d19d2656b4c686ea2405dd6b2bfbd135ac Mon Sep 17 00:00:00 2001 From: Masato Onodera Date: Fri, 13 Mar 2026 14:45:39 -1000 Subject: [PATCH 1/4] fix: fill NaN pmra/pmdec/parallax for Gaia filler targets In fixcols_gaiadb_to_targetdb(), Gaia sources with missing astrometry (common for faint stars) were passed through with NaN proper motion and parallax values. These NaNs propagated into get_fp_positions(), causing the coordinate transform to produce NaN focal plane positions, which then triggered a ValueError in TargetSelector.constructKDTree(). Apply the same NaN-filling used in other fetch functions: pmra/pmdec -> 0, parallax -> 1e-7 (effectively zero parallax). Co-Authored-By: Claude Sonnet 4.6 --- src/pfs_design_tool/pointing_utils/dbutils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pfs_design_tool/pointing_utils/dbutils.py b/src/pfs_design_tool/pointing_utils/dbutils.py index 0120134..4c62d62 100644 --- a/src/pfs_design_tool/pointing_utils/dbutils.py +++ b/src/pfs_design_tool/pointing_utils/dbutils.py @@ -598,6 +598,10 @@ def fixcols_gaiadb_to_targetdb( # df["priority"][tb["g_mag_ab"].value - 7 > 12] = 9999 # df["priority"][np.isnan(tb["g_mag_ab"])] = 9 + df.loc[df["pmra"].isna(), "pmra"] = 0.0 + df.loc[df["pmdec"].isna(), "pmdec"] = 0.0 + df.loc[df["parallax"].isna(), "parallax"] = 1.0e-7 + return df From 91addd20443c24e0faa6dc5d142db521e274c546 Mon Sep 17 00:00:00 2001 From: Masato Onodera Date: Fri, 13 Mar 2026 15:06:37 -1000 Subject: [PATCH 2/4] fix: copy parallax array before passing to ctrans to avoid read-only error pfs.utils.coordinates.Subaru_POPT2_PFS.radec2radecplxpm() modifies the parallax argument in-place (str_plx[...] = 0.00001). With Pandas 3.x, DataFrame.to_numpy() can return a read-only view due to Copy-on-Write, causing a ValueError when the function tries to write to it. Pass .copy() of the parallax array at both ctrans call sites in generate_guidestars_from_gaiadb() and generate_guidestars_from_csv(). Co-Authored-By: Claude Sonnet 4.6 --- src/pfs_design_tool/pointing_utils/designutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pfs_design_tool/pointing_utils/designutils.py b/src/pfs_design_tool/pointing_utils/designutils.py index b8bdca9..50d71fd 100644 --- a/src/pfs_design_tool/pointing_utils/designutils.py +++ b/src/pfs_design_tool/pointing_utils/designutils.py @@ -1041,7 +1041,7 @@ def generate_guidestars_from_gaiadb( pa=pa_deg, cent=np.array([ra_tel_deg, dec_tel_deg]).reshape((2, 1)), pm=np.stack([res[coldict["pmra"]], res[coldict["pmdec"]]], axis=0), - par=res[coldict["parallax"]], + par=res[coldict["parallax"]].copy(), time=observation_time, epoch=gaiadb_epoch, ) @@ -1272,7 +1272,7 @@ def generate_guidestars_from_csv( pa=pa_deg, cent=np.array([ra_tel_deg, dec_tel_deg]).reshape((2, 1)), pm=np.stack([res[coldict["pmra"]], res[coldict["pmdec"]]], axis=0), - par=res[coldict["parallax"]], + par=res[coldict["parallax"]].copy(), time=observation_time, epoch=gaiadb_epoch, ) From 62331c2c921af80572d0c636130110d68fa81f0f Mon Sep 17 00:00:00 2001 From: Masato Onodera Date: Fri, 13 Mar 2026 15:11:18 -1000 Subject: [PATCH 3/4] chore: pin pandas<3.0.0 until upstream PFS packages support CoW Pandas 3.0 enables Copy-on-Write by default, causing .to_numpy() to return read-only views. External PFS packages (pfs_utils, ics_cobraOps, etc.) have not been validated against Pandas 3.x and may have similar in-place modification issues. Pin to <3.0.0 until the ecosystem catches up. Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index af4b3e3..cc54a84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "loguru>=0.7.0", "matplotlib>=3.9.0", "numpy>=2.0.0", - "pandas>=2.2.0", + "pandas>=2.2.0,<3.0.0", "pfs-datamodel @ git+https://github.com/Subaru-PFS/datamodel.git", "pfs-utils @ git+https://github.com/Subaru-PFS/pfs_utils.git", "psycopg2-binary>=2.9.8", From 8a164b897f791fcaa0af0f14871692e124b4b8f5 Mon Sep 17 00:00:00 2001 From: Masato Onodera Date: Fri, 13 Mar 2026 17:11:55 -1000 Subject: [PATCH 4/4] fix: replace bare except and bool/None comparisons with specific exceptions and idiomatic checks - Use specific exception types (ValueError, FileNotFoundError, KeyError, TypeError, AttributeError) instead of bare except - Replace == True/False/None with idiomatic is None, not x, elif x, ~df[col], df[col] - Use .isin() for mixed-type is_medium_resolution column (str "L/M" + bool) - Fix E701 inline if statement in reconfigure_fibers_ppp.py Co-Authored-By: Claude Sonnet 4.6 --- src/pfs_design_tool/pointing_utils/dbutils.py | 4 +-- .../pointing_utils/designutils.py | 4 +-- src/pfs_design_tool/pointing_utils/nfutils.py | 6 ++-- src/pfs_design_tool/reconfigure_fibers_ppp.py | 33 +++++++++---------- .../subaru_fiber_allocation.py | 4 +-- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/pfs_design_tool/pointing_utils/dbutils.py b/src/pfs_design_tool/pointing_utils/dbutils.py index 4c62d62..a39c027 100644 --- a/src/pfs_design_tool/pointing_utils/dbutils.py +++ b/src/pfs_design_tool/pointing_utils/dbutils.py @@ -256,7 +256,7 @@ def generate_fluxstds_from_targetdb( AND teff_gspphot BETWEEN {min_teff} AND {max_teff} """ - except: + except ValueError: extra_where += "" if fluxstd_versions is not None: @@ -814,7 +814,7 @@ def fixcols_filler_targetdb( df_obs_filler_done = pd.read_csv( os.path.join(workDir, "ppp/df_obsfiller_done.csv") ) - except: + except FileNotFoundError: # query qaDB to get executed pfsdesign conn = connect_qadb(conf) cur = conn.cursor() diff --git a/src/pfs_design_tool/pointing_utils/designutils.py b/src/pfs_design_tool/pointing_utils/designutils.py index 50d71fd..3e33deb 100644 --- a/src/pfs_design_tool/pointing_utils/designutils.py +++ b/src/pfs_design_tool/pointing_utils/designutils.py @@ -362,7 +362,7 @@ def generate_pfs_design( np.nan, ] ) - except: + except (KeyError, TypeError): dict_of_flux_lists["psf_flux"][i_fiber] = np.array( [ np.nan, @@ -605,7 +605,7 @@ def generate_pfs_design( np.nan, ] ) - except: + except (TypeError, AttributeError): dict_of_flux_lists["psf_flux"][i_fiber] = np.array( [ np.nan, diff --git a/src/pfs_design_tool/pointing_utils/nfutils.py b/src/pfs_design_tool/pointing_utils/nfutils.py index b4d92cc..e5d4196 100644 --- a/src/pfs_design_tool/pointing_utils/nfutils.py +++ b/src/pfs_design_tool/pointing_utils/nfutils.py @@ -398,7 +398,7 @@ def fiber_allocation( # exit() - if (df_filler is not None) and (two_stage == False): + if (df_filler is not None) and not two_stage: print("[single-stage] Registering stars for fillers.") targets += register_objects( df_filler, target_class="sci", force_exptime=force_exptime @@ -610,7 +610,7 @@ def fiber_allocation( cobraSafetyMargin=cobraSafetyMargin, ) - if two_stage == False: + if not two_stage: return ( res[0], target_fppos[0], @@ -621,7 +621,7 @@ def fiber_allocation( bench, ) - elif two_stage == True: + elif two_stage: # get cobra ID of the 1st-stage assignment assign_1stage = {} for i, (vis, tp, tel) in enumerate(zip(res, target_fppos, telescopes)): diff --git a/src/pfs_design_tool/reconfigure_fibers_ppp.py b/src/pfs_design_tool/reconfigure_fibers_ppp.py index 21346cb..32651c3 100644 --- a/src/pfs_design_tool/reconfigure_fibers_ppp.py +++ b/src/pfs_design_tool/reconfigure_fibers_ppp.py @@ -550,7 +550,7 @@ def reconfigure_multiprocessing( design_ids = {} for i, pointing in enumerate(list_pointings): - if clearOutput == True: + if clearOutput: clear_output() # observation_time = Time.now().iso observation_time = str(dict_pointings[pointing.lower()]["observation_time"][0]) @@ -613,7 +613,8 @@ def reconfigure_multiprocessing( ) if conf["sfa"]["reduce_sky_targets"]: n_sky_target = conf["sfa"]["n_sky_random"] # this value can be tuned - if "CFHTLS" in ppc_code: n_sky_target=4000 + if "CFHTLS" in ppc_code: + n_sky_target = 4000 if len(df_sky) > n_sky_target: df_sky = df_sky.sample( n_sky_target, ignore_index=True, random_state=1 @@ -621,9 +622,9 @@ def reconfigure_multiprocessing( logger.info(f"Fetched sky target DataFrame: \n{df_sky}") # get filler targets (optional) - if conf["sfa"]["filler"] == False: + if not conf["sfa"]["filler"]: df_filler = None - if conf["sfa"]["dup_fluxstd_remove"] == True: + if conf["sfa"]["dup_fluxstd_remove"]: _, df_filler_nocut = dbutils.generate_fillers_from_targetdb( dict_pointings[pointing.lower()]["ra_center"], dict_pointings[pointing.lower()]["dec_center"], @@ -662,7 +663,7 @@ def reconfigure_multiprocessing( logger.info( f"Duplicates in fluxstds removed: {n_fluxstd_orig} --> {n_fluxstd_red}" ) - elif conf["sfa"]["filler"] == True: + elif conf["sfa"]["filler"]: """ df_filler_obs = dbutils.generate_targets_from_gaiadb( dict_pointings[pointing.lower()]["ra_center"], @@ -708,7 +709,7 @@ def reconfigure_multiprocessing( ) # remove duplicates in df_fluxstds with df_filler_usr & df_sci - if conf["sfa"]["dup_fluxstd_remove"] == True: + if conf["sfa"]["dup_fluxstd_remove"]: n_fluxstd_orig = len(df_fluxstds) # Build SkyCoord for df_filler_fluxstds coords_fluxstds = SkyCoord( @@ -740,12 +741,10 @@ def reconfigure_multiprocessing( if rsl_mode == "L": df_filler_usr = df_filler_usr[ - (df_filler_usr["is_medium_resolution"] == "L/M") - | (df_filler_usr["is_medium_resolution"] == False) + df_filler_usr["is_medium_resolution"].isin(["L/M", False]) ] df_filler_obs = df_filler_obs[ - (df_filler_obs["is_medium_resolution"] == "L/M") - | (df_filler_obs["is_medium_resolution"] == False) + df_filler_obs["is_medium_resolution"].isin(["L/M", False]) ] if ppc_backup: df_filler_usr = df_filler_usr[ @@ -753,12 +752,10 @@ def reconfigure_multiprocessing( ] elif rsl_mode == "M": df_filler_usr = df_filler_usr[ - (df_filler_usr["is_medium_resolution"] == "L/M") - | (df_filler_usr["is_medium_resolution"] == True) + df_filler_usr["is_medium_resolution"].isin(["L/M", True]) ] df_filler_obs = df_filler_obs[ - (df_filler_obs["is_medium_resolution"] == "L/M") - | (df_filler_obs["is_medium_resolution"] == True) + df_filler_obs["is_medium_resolution"].isin(["L/M", True]) ] if ppc_backup: df_filler_usr = df_filler_usr[ @@ -770,8 +767,8 @@ def reconfigure_multiprocessing( # --- case 1: too many user fillers → downsample user fillers only --- if len(df_filler_usr) >= n_fillers: - unobs_usr = df_filler_usr[df_filler_usr["observed"] == False] - obs_usr = df_filler_usr[df_filler_usr["observed"] == True] + unobs_usr = df_filler_usr[~df_filler_usr["observed"]] + obs_usr = df_filler_usr[df_filler_usr["observed"]] if len(unobs_usr) >= n_fillers: df_filler_usr = unobs_usr.sample(n_fillers, random_state=1, ignore_index=True) @@ -789,8 +786,8 @@ def reconfigure_multiprocessing( n_needed = n_fillers - len(df_filler_usr) # fill df_filler_obs using the same unobs-first rule - unobs_obs = df_filler_obs[df_filler_obs["observed"] == False] - obs_obs = df_filler_obs[df_filler_obs["observed"] == True] + unobs_obs = df_filler_obs[~df_filler_obs["observed"]] + obs_obs = df_filler_obs[df_filler_obs["observed"]] if len(unobs_obs) >= n_needed: df_filler_obs = unobs_obs.sample(n_needed, random_state=1, ignore_index=True) diff --git a/src/pfs_design_tool/subaru_fiber_allocation.py b/src/pfs_design_tool/subaru_fiber_allocation.py index d787a55..d457ada 100644 --- a/src/pfs_design_tool/subaru_fiber_allocation.py +++ b/src/pfs_design_tool/subaru_fiber_allocation.py @@ -497,7 +497,7 @@ def main(): df_targets = df_targets[~msk].reset_index() # degrade priority of science targets (default: no degradation) - if args.degrade_priority_proposal == None: + if args.degrade_priority_proposal is None: df_targets.priority += args.degrade_priority else: df_targets.priority[ @@ -617,7 +617,7 @@ def main(): try: cobra_safety_margin = conf["netflow"]["cobra_safety_margin"] - except: + except KeyError: cobra_safety_margin = 0.0 vis, tp, tel, tgt, tgt_class_dict, is_no_target, bench = nfutils.fiber_allocation(