From b91c2ea5589acf519da71b0e48424a087d43ada7 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Thu, 6 Mar 2025 10:48:42 -0800 Subject: [PATCH 1/7] Use recommended astropy.modeling.fitting.LMLSQFitter() --- spectractor/tools.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/spectractor/tools.py b/spectractor/tools.py index 957b36097..dde760568 100644 --- a/spectractor/tools.py +++ b/spectractor/tools.py @@ -681,12 +681,12 @@ def fit_poly2d(x, y, z, order): >>> assert np.isclose(fit.c1_1.value, -2) """ p_init = models.Polynomial2D(degree=order) - fit_p = fitting.LevMarLSQFitter() + fit_p = fitting.LMLSQFitter() with warnings.catch_warnings(): # Ignore model linearity warning from the fitter warnings.simplefilter('ignore') p = fit_p(p_init, x, y, z) - return p + return p def fit_poly1d_outlier_removal(x, y, order=2, sigma=3.0, niter=3): @@ -901,13 +901,13 @@ def fit_gauss2d_outlier_removal(x, y, z, sigma=3.0, niter=3, guess=None, bounds= with warnings.catch_warnings(): # Ignore model linearity warning from the fitter warnings.simplefilter('ignore') - fit = fitting.LevMarLSQFitter() + fit = fitting.LMLSQFitter() or_fit = fitting.FittingWithOutlierRemoval(fit, sigma_clip, niter=niter, sigma=sigma) # get fitted model and filtered data or_fitted_model, filtered_data = or_fit(gg_init, x, y, z) - my_logger.info(f'\n\t{or_fitted_model}') - # my_logger.debug(f'\n\t{fit.fit_info}') - return or_fitted_model + my_logger.info(f'\n\t{or_fitted_model}') + # my_logger.debug(f'\n\t{fit.fit_info}') + return or_fitted_model def fit_moffat2d_outlier_removal(x, y, z, sigma=3.0, niter=3, guess=None, bounds=None): @@ -998,13 +998,13 @@ def fit_moffat2d_outlier_removal(x, y, z, sigma=3.0, niter=3, guess=None, bounds with warnings.catch_warnings(): # Ignore model linearity warning from the fitter warnings.simplefilter('ignore') - fit = fitting.LevMarLSQFitter() + fit = fitting.LMLSQFitter() or_fit = fitting.FittingWithOutlierRemoval(fit, sigma_clip, niter=niter, sigma=sigma) # get fitted model and filtered data or_fitted_model, filtered_data = or_fit(gg_init, x, y, z) - my_logger.info(f'\n\t{or_fitted_model}') - # my_logger.debug(f'\n\t{fit.fit_info}') - return or_fitted_model + my_logger.info(f'\n\t{or_fitted_model}') + # my_logger.debug(f'\n\t{fit.fit_info}') + return or_fitted_model def fit_moffat1d_outlier_removal(x, y, sigma=3.0, niter=3, guess=None, bounds=None): @@ -1093,13 +1093,13 @@ def fit_moffat1d_outlier_removal(x, y, sigma=3.0, niter=3, guess=None, bounds=No with warnings.catch_warnings(): # Ignore model linearity warning from the fitter warnings.simplefilter('ignore') - fit = fitting.LevMarLSQFitter() + fit = fitting.LMLSQFitter() or_fit = fitting.FittingWithOutlierRemoval(fit, sigma_clip, niter=niter, sigma=sigma) # get fitted model and filtered data or_fitted_model, filtered_data = or_fit(gg_init, x, y) - my_logger.debug(f'\n\t{or_fitted_model}') - # my_logger.debug(f'\n\t{fit.fit_info}') - return or_fitted_model + my_logger.debug(f'\n\t{or_fitted_model}') + # my_logger.debug(f'\n\t{fit.fit_info}') + return or_fitted_model def fit_moffat1d(x, y, guess=None, bounds=None): @@ -1179,11 +1179,11 @@ def fit_moffat1d(x, y, guess=None, bounds=None): with warnings.catch_warnings(): # Ignore model linearity warning from the fitter warnings.simplefilter('ignore') - fit = fitting.LevMarLSQFitter() + fit = fitting.LMLSQFitter() fitted_model = fit(gg_init, x, y) - my_logger.info(f'\n\t{fitted_model}') - # my_logger.debug(f'\n\t{fit.fit_info}') - return fitted_model + my_logger.info(f'\n\t{fitted_model}') + # my_logger.debug(f'\n\t{fit.fit_info}') + return fitted_model def compute_fwhm(x, y, minimum=0, center=None, full_output=False, epsilon=1e-3): From 5501b466bf7671aff9ad7a687f89c3523bf61b5a Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 7 Mar 2025 08:31:01 -0800 Subject: [PATCH 2/7] Add logging for when SIMBAD is being accessed. --- spectractor/extractor/targets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spectractor/extractor/targets.py b/spectractor/extractor/targets.py index 9c4360f25..5d3a21192 100644 --- a/spectractor/extractor/targets.py +++ b/spectractor/extractor/targets.py @@ -288,6 +288,7 @@ def load(self): if getCalspec.is_calspec(self.label): calspec = getCalspec.Calspec(self.label) astroquery_label = calspec.Astroquery_Name + self.my_logger.info("Querying SIMBAD for %s", astroquery_label) self.simbad_table = simbadQuerier.query_object(astroquery_label) if self.simbad_table is not None: From e5f373c31955f47fbe59fe25a34181cfbe0ddf98 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 7 Mar 2025 08:37:50 -0800 Subject: [PATCH 3/7] Remove broken pytest section. --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 532062ace..aaa4789db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,2 @@ -[tool:pytest] -exclude=test_(extractor_ctio_planetary_nebula|astrometry|mcmc|multispectra) - [coverage:run] omit=spectractor/fit/mcmc.py From c21a459f0631254e727551a55a9b9d2da7e98894 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 7 Mar 2025 08:38:10 -0800 Subject: [PATCH 4/7] Fix figure number warning. --- spectractor/fit/mcmc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectractor/fit/mcmc.py b/spectractor/fit/mcmc.py index 4a83e6430..c461b7a85 100644 --- a/spectractor/fit/mcmc.py +++ b/spectractor/fit/mcmc.py @@ -428,7 +428,7 @@ def marginalize(self, pdfonly=False): # pragma: no cover def triangle_plots(self, output_filename=''): n = self.dim - fig = plt.figure(1, figsize=(16, 9)) + fig = plt.figure(figsize=(16, 9)) if parameters.PAPER: fig.set_size_inches(18, 13) fig.clf() From 83adf5d4e150384c4e607704df6c0932e7fc64fa Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 7 Mar 2025 12:09:05 -0800 Subject: [PATCH 5/7] Use specially cached simbad table when in jenkins context. --- spectractor/extractor/targets.py | 10 +- tests/data/hd111980_simbad.ecsv | 208 +++++++++++++++++++++++++++++++ tests/test_extractor.py | 8 ++ 3 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 tests/data/hd111980_simbad.ecsv diff --git a/spectractor/extractor/targets.py b/spectractor/extractor/targets.py index 5d3a21192..079832051 100644 --- a/spectractor/extractor/targets.py +++ b/spectractor/extractor/targets.py @@ -288,8 +288,14 @@ def load(self): if getCalspec.is_calspec(self.label): calspec = getCalspec.Calspec(self.label) astroquery_label = calspec.Astroquery_Name - self.my_logger.info("Querying SIMBAD for %s", astroquery_label) - self.simbad_table = simbadQuerier.query_object(astroquery_label) + if parameters.HD111980_OFFLINE_FILE != "" and astroquery_label == "HD 111980": + import astropy.table + + self.my_logger.info("Using cached SIMBAD query for ``%s``", astroquery_label) + self.simbad_table = astropy.table.Table.read(parameters.HD111980_OFFLINE_FILE) + else: + self.my_logger.info("Querying SIMBAD for ``%s``", astroquery_label) + self.simbad_table = simbadQuerier.query_object(astroquery_label) if self.simbad_table is not None: if self.verbose or True: diff --git a/tests/data/hd111980_simbad.ecsv b/tests/data/hd111980_simbad.ecsv new file mode 100644 index 000000000..29d29affe --- /dev/null +++ b/tests/data/hd111980_simbad.ecsv @@ -0,0 +1,208 @@ +# %ECSV 1.0 +# --- +# datatype: +# - name: main_id +# datatype: string +# description: Main identifier for an object +# meta: !!omap +# - {ucd: meta.id;meta.main} +# - {_votable_string_dtype: char} +# subtype: json +# - name: ra +# unit: deg +# datatype: float64 +# description: Right ascension +# meta: !!omap +# - {ucd: pos.eq.ra;meta.main} +# - name: dec +# unit: deg +# datatype: float64 +# description: Declination +# meta: !!omap +# - {ucd: pos.eq.dec;meta.main} +# - name: coo_err_maj +# unit: mas +# datatype: float32 +# description: Coordinate error major axis +# meta: !!omap +# - {ucd: phys.angSize.smajAxis;pos.errorEllipse;pos.eq} +# - name: coo_err_min +# unit: mas +# datatype: float32 +# description: Coordinate error minor axis +# meta: !!omap +# - {ucd: phys.angSize.sminAxis;pos.errorEllipse;pos.eq} +# - name: coo_err_angle +# unit: deg +# datatype: int16 +# description: Coordinate error angle +# meta: !!omap +# - {ucd: pos.posAng;pos.errorEllipse;pos.eq} +# - values: {'null': -32768} +# - name: coo_wavelength +# datatype: string +# description: Wavelength class for the origin of the coordinates (R,I,V,U,X,G) +# meta: !!omap +# - {ucd: instr.bandpass;pos.eq} +# - {_votable_string_dtype: char} +# - name: coo_bibcode +# datatype: string +# description: Coordinate reference +# meta: !!omap +# - {ucd: meta.bib.bibcode;pos.eq} +# - {_votable_string_dtype: char} +# subtype: json +# - name: U +# datatype: float64 +# description: U magnitude +# meta: !!omap +# - {ucd: phot.mag;em.opt.U} +# - name: R +# datatype: float64 +# description: R magnitude +# meta: !!omap +# - {ucd: phot.mag;em.opt.R} +# - name: V +# datatype: float64 +# description: V magnitude +# meta: !!omap +# - {ucd: phot.mag;em.opt.V} +# - name: I +# datatype: float64 +# description: I magnitude +# meta: !!omap +# - {ucd: phot.mag;em.opt.I} +# - name: J +# datatype: float64 +# description: J magnitude +# meta: !!omap +# - {ucd: phot.mag;em.IR.J} +# - name: B +# datatype: float64 +# description: B magnitude +# meta: !!omap +# - {ucd: phot.mag;em.opt.B} +# - name: rvz_redshift +# datatype: float64 +# description: redshift +# meta: !!omap +# - {ucd: src.redshift} +# - name: sp_type +# datatype: string +# description: MK spectral type +# meta: !!omap +# - {ucd: src.spType} +# - {_votable_string_dtype: char} +# subtype: json +# - name: pm_err_angle +# unit: deg +# datatype: int16 +# description: Proper motion error angle +# meta: !!omap +# - {ucd: pos.posAng;pos.errorEllipse;pos.pm} +# - values: {'null': -32768} +# - name: pmdec_prec +# datatype: int16 +# description: Proper motion in DEC precision +# meta: !!omap +# - values: {'null': -32768} +# - name: pmdec +# unit: mas / yr +# datatype: float64 +# description: Proper motion in DEC +# meta: !!omap +# - {ucd: pos.pm;pos.eq.dec} +# - name: pm_err_min_prec +# datatype: int16 +# description: Proper motion error minor axis precision +# meta: !!omap +# - values: {'null': -32768} +# - name: pmra_prec +# datatype: int16 +# description: Proper motion in RA precision +# meta: !!omap +# - values: {'null': -32768} +# - name: pm_err_maj +# unit: mas / yr +# datatype: float32 +# description: Proper motion error major axis +# meta: !!omap +# - {ucd: phys.angSize.smajAxis;pos.errorEllipse;pos.pm} +# - name: pm_err_min +# unit: mas / yr +# datatype: float32 +# description: Proper motion error minor axis +# meta: !!omap +# - {ucd: phys.angSize.sminAxis;pos.errorEllipse;pos.pm} +# - name: pmra +# unit: mas / yr +# datatype: float64 +# description: Proper motion in RA +# meta: !!omap +# - {ucd: pos.pm;pos.eq.ra} +# - name: pm_bibcode +# datatype: string +# description: Proper motion reference +# meta: !!omap +# - {ucd: meta.bib.bibcode;pos.pm} +# - {_votable_string_dtype: char} +# subtype: json +# - name: pm_err_maj_prec +# datatype: int16 +# description: Proper motion error major axis precision +# meta: !!omap +# - values: {'null': -32768} +# - name: pm_qual +# datatype: string +# description: Proper motion quality +# meta: !!omap +# - {ucd: meta.code.qual;pos.pm} +# - {_votable_string_dtype: char} +# - name: plx_prec +# datatype: int16 +# description: Parallax precision +# meta: !!omap +# - values: {'null': -32768} +# - name: plx_err +# unit: mas +# datatype: float32 +# description: Parallax error +# meta: !!omap +# - {ucd: stat.error;pos.parallax.trig} +# - name: plx_err_prec +# datatype: int16 +# description: Parallax error precision +# meta: !!omap +# - values: {'null': -32768} +# - name: plx_value +# unit: mas +# datatype: float64 +# description: Parallax +# meta: !!omap +# - {ucd: pos.parallax.trig} +# - name: plx_bibcode +# datatype: string +# description: Parallax reference +# meta: !!omap +# - {ucd: meta.bib.bibcode;pos.parallax.trig} +# - {_votable_string_dtype: char} +# subtype: json +# - name: plx_qual +# datatype: string +# description: Parallax quality +# meta: !!omap +# - {ucd: meta.code.qual;pos.parallax.trig} +# - {_votable_string_dtype: char} +# - name: matched_id +# datatype: string +# description: Identifier +# meta: !!omap +# - {ucd: meta.id} +# - {_votable_string_dtype: char} +# subtype: json +# meta: !!omap +# - {ID: result_S1741375520498} +# - {name: result_S1741375520498} +# schema: astropy-2.0 +main_id ra dec coo_err_maj coo_err_min coo_err_angle coo_wavelength coo_bibcode U R V I J B rvz_redshift sp_type pm_err_angle pmdec_prec pmdec pm_err_min_prec pmra_prec pm_err_maj pm_err_min pmra pm_bibcode pm_err_maj_prec pm_qual plx_prec plx_err plx_err_prec plx_value plx_bibcode plx_qual matched_id +"""HD 111980""" 193.31272069229124 -18.522225886039166 0.0225 0.0183 90 O """2020yCat.1350....0G""" 8.800000190734863 "" 8.380000114440918 "" 7.176000118255615 8.90999984741211 0.0005171580737759474 """F7V""" 90 3 -795.843 3 3 0.032 0.026 299.359 """2020yCat.1350....0G""" 3 A 4 0.0261 4 12.9277 """2020yCat.1350....0G""" A """HD 111980""" diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 8f4b457eb..793ebe35d 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -37,6 +37,14 @@ def test_extractor_ctio(): parameters.CCD_REBIN = 2 # rebin=1 to build tests/data spectrum apply_rebinning_to_parameters() + parameters.HD111980_OFFLINE_FILE = "" + if os.environ.get("JENKINS_HOME") is not None: + parameters.HD111980_OFFLINE_FILE = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + "data", + "hd111980_simbad.ecsv", + ) + for file_name in file_names: tag = file_name.split('/')[-1].replace("sim", "reduc") disperser_label, target_label, xpos, ypos = logbook.search_for_image(tag) From 25daeda3681796d8b2f90b889268ae05a446e955 Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Fri, 7 Mar 2025 12:36:13 -0800 Subject: [PATCH 6/7] Add default parameter and explicit format. --- spectractor/extractor/targets.py | 2 +- spectractor/parameters.py | 2 ++ tests/test_fullchain.py | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/spectractor/extractor/targets.py b/spectractor/extractor/targets.py index 079832051..9a55bd308 100644 --- a/spectractor/extractor/targets.py +++ b/spectractor/extractor/targets.py @@ -292,7 +292,7 @@ def load(self): import astropy.table self.my_logger.info("Using cached SIMBAD query for ``%s``", astroquery_label) - self.simbad_table = astropy.table.Table.read(parameters.HD111980_OFFLINE_FILE) + self.simbad_table = astropy.table.Table.read(parameters.HD111980_OFFLINE_FILE, format="ecsv") else: self.my_logger.info("Querying SIMBAD for ``%s``", astroquery_label) self.simbad_table = simbadQuerier.query_object(astroquery_label) diff --git a/spectractor/parameters.py b/spectractor/parameters.py index 0a8643c38..36b4cd6e7 100644 --- a/spectractor/parameters.py +++ b/spectractor/parameters.py @@ -143,3 +143,5 @@ def __getattr__(name): STYLE_PARAMETERS = ["VERBOSE", "DEBUG", "PAPER", "LINEWIDTH", "PLOT_DIR", "SAVE", "DEBUG_LOGGING", "DISPLAY", "PLOT_XLABEL", "PLOT_YLABEL", "PLOT_ROT_LABEL"] + +HD111980_OFFLINE_FILE = "" diff --git a/tests/test_fullchain.py b/tests/test_fullchain.py index ac9d3b586..4ec63f84c 100644 --- a/tests/test_fullchain.py +++ b/tests/test_fullchain.py @@ -106,6 +106,14 @@ def test_ctio_fullchain(): parameters.SPECTRACTOR_ATMOSPHERE_SIM = "libradtran" sim_image = "./tests/data/sim_20170530_134.fits" + parameters.HD111980_OFFLINE_FILE = "" + if os.environ.get("JENKINS_HOME") is not None: + parameters.HD111980_OFFLINE_FILE = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + "data", + "hd111980_simbad.ecsv", + ) + # load test and make image simulation if not os.path.isfile(sim_image): make_image() From ef0ffe56bf14244a7cd15d9988d56c56bdcfa3ca Mon Sep 17 00:00:00 2001 From: Eli Rykoff Date: Sat, 8 Mar 2025 11:25:21 -0800 Subject: [PATCH 7/7] Fix formatter name. --- spectractor/extractor/targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectractor/extractor/targets.py b/spectractor/extractor/targets.py index 9a55bd308..bd241e191 100644 --- a/spectractor/extractor/targets.py +++ b/spectractor/extractor/targets.py @@ -292,7 +292,7 @@ def load(self): import astropy.table self.my_logger.info("Using cached SIMBAD query for ``%s``", astroquery_label) - self.simbad_table = astropy.table.Table.read(parameters.HD111980_OFFLINE_FILE, format="ecsv") + self.simbad_table = astropy.table.Table.read(parameters.HD111980_OFFLINE_FILE, format="ascii.ecsv") else: self.my_logger.info("Querying SIMBAD for ``%s``", astroquery_label) self.simbad_table = simbadQuerier.query_object(astroquery_label)