diff --git a/autotest/autotest_notebooks.py b/autotest/autotest_notebooks.py index 41048487..b9f63fb1 100644 --- a/autotest/autotest_notebooks.py +++ b/autotest/autotest_notebooks.py @@ -23,7 +23,7 @@ def get_notebooks(): - return [f for f in os.listdir(nbdir) if f.endswith('.ipynb') and not "notest" in f] + return [f for f in os.listdir(nbdir) if f.endswith('.ipynb') and "notest" not in f] def run_notebook(fn): #pth = os.path.join(nbdir, fn) diff --git a/autotest/emulator_tests.py b/autotest/emulator_tests.py index de2c40e8..6b8d167c 100644 --- a/autotest/emulator_tests.py +++ b/autotest/emulator_tests.py @@ -15,6 +15,8 @@ mou_exe_path = get_exe_path("pestpp-mou") def dsi_freyberg(tmp_d,transforms=None,tag=""): + if not ies_exe_path: + pytest.skip("missing ies_exe_path") test_d = "ends_master" test_d = setup_tmp(test_d, tmp_d) @@ -92,6 +94,8 @@ def test_dsi_mixed(tmp_path): # @pytest.mark.timeout(method="thread", timeout=1000) def test_dsivc(tmp_path): + if not mou_exe_path: + pytest.skip("missing mou_exe_path") # basic quick as so can re-run here dsi_freyberg(tmp_path, transforms=None) # now test dsicv @@ -205,6 +209,7 @@ def plot_freyberg_dsi(): def lpfa_freyberg(tmp_d="temp",transforms=None): + pytest.importorskip("sklearn") test_d = "ends_master" test_d = setup_tmp(test_d, tmp_d) @@ -331,6 +336,8 @@ def test_lpfa_std(tmp_path): def gpr_compare_invest(): + if not mou_exe_path: + pytest.skip("missing mou_exe_path") import numpy as np from sklearn.gaussian_process import GaussianProcessRegressor case = "zdt1" @@ -492,6 +499,8 @@ def gpr_compare_invest(): def gpr_constr_invest(): + if not mou_exe_path: + pytest.skip("missing mou_exe_path") import numpy as np from sklearn.gaussian_process import GaussianProcessRegressor case = "constr" @@ -812,14 +821,13 @@ def gpr_zdt1_test(): gpr_t_d = os.path.join(case + "_gpr_template") data, input_names, output_names = collate_training_data(pst,m_d,case) - from pyemu.emulators.gpr import GPR gpr = GPR(data=data.copy(), input_names=input_names, output_names=output_names, #transforms=transforms, #kernel=gp_kernel, n_restarts_optimizer=20, - ); + ) gpr.fit() gpr.prepare_pestpp(m_d,case,gpr_t_d=gpr_t_d) diff --git a/autotest/en_tests.py b/autotest/en_tests.py index a80d088e..5e0a1472 100644 --- a/autotest/en_tests.py +++ b/autotest/en_tests.py @@ -349,7 +349,6 @@ def from_parfiles_test(tmp_path): def triangular_draw_test(): import os - import matplotlib.pyplot as plt import pyemu pst = pyemu.Pst(os.path.join("pst","pest.pst")) @@ -577,7 +576,6 @@ def emp_cov_draw_test(): def mixed_par_draw_test(): import os - import matplotlib.pyplot as plt import pandas as pd import pyemu @@ -591,6 +589,7 @@ def mixed_par_draw_test(): pe2 = pyemu.ParameterEnsemble.from_mixed_draws(pst, {},default="uniform",num_reals=num_reals) pe3 = pyemu.ParameterEnsemble.from_mixed_draws(pst, {}, default="triangular", num_reals=num_reals) + # import matplotlib.pyplot as plt # ax = plt.subplot(111) # pe1.loc[:,pname].hist(ax=ax,alpha=0.5,bins=25) # pe2.loc[:, pname].hist(ax=ax,alpha=0.5,bins=25) @@ -697,7 +696,6 @@ def binary_test(tmp_path): def mixed_par_draw_2_test(): import os - import matplotlib.pyplot as plt import pandas as pd import pyemu @@ -711,6 +709,7 @@ def mixed_par_draw_2_test(): pe2 = pyemu.ParameterEnsemble.from_mixed_draws(pst, {},default="uniform",num_reals=num_reals) pe3 = pyemu.ParameterEnsemble.from_mixed_draws(pst, {}, default="triangular", num_reals=num_reals) + # import matplotlib.pyplot as plt # ax = plt.subplot(111) # pe1.loc[:,pname].hist(ax=ax,alpha=0.5,bins=25) # pe2.loc[:, pname].hist(ax=ax,alpha=0.5,bins=25) diff --git a/autotest/full_meal_deal_tests_ignore.py b/autotest/full_meal_deal_tests_ignore.py index e33cd7ab..be897882 100644 --- a/autotest/full_meal_deal_tests_ignore.py +++ b/autotest/full_meal_deal_tests_ignore.py @@ -60,7 +60,7 @@ def _get_port(): def freyberg_spinup(tmp_path): try: import flopy - except Exception as e: + except Exception: return import pyemu @@ -248,7 +248,7 @@ def freyberg_kl_pp_compare(): import pandas as pd try: import flopy - except Exception as e: + except Exception: return import pyemu @@ -319,7 +319,7 @@ def freyberg_diff_obs_test(tmp_path): import pandas as pd try: import flopy - except Exception as e: + except Exception: return import pyemu diff --git a/autotest/plot_tests.py b/autotest/plot_tests.py index e949be45..8f41f4c6 100644 --- a/autotest/plot_tests.py +++ b/autotest/plot_tests.py @@ -3,14 +3,11 @@ import numpy as np import pandas as pd import pyemu +import pytest def plot_summary_test(tmp_path): - - try: - import matplotlib.pyplot as plt - except: - return + plt = pytest.importorskip("matplotlib.pyplot") par_df_fname = os.path.join("utils","freyberg_pp.par.usum.csv") shutil.copy(par_df_fname, tmp_path) par_df_fname = os.path.join(tmp_path, "freyberg_pp.par.usum.csv") @@ -38,10 +35,7 @@ def plot_summary_test(tmp_path): def pst_plot_test(tmp_path): - try: - import matplotlib.pyplot as plt - except: - return + plt = pytest.importorskip("matplotlib.pyplot") pstfile = os.path.join("pst", "pest.pst") shutil.copy(pstfile, os.path.join(tmp_path, "pest.pst")) @@ -98,10 +92,7 @@ def pst_plot_test(tmp_path): def ensemble_plot_test(tmp_path): - try: - import matplotlib.pyplot as plt - except: - return + plt = pytest.importorskip("matplotlib.pyplot") pstfile = os.path.join("pst", "pest.pst") shutil.copy(pstfile, os.path.join(tmp_path, "pest.pst")) @@ -165,10 +156,7 @@ def ensemble_plot_test(tmp_path): def ensemble_1to1_test(tmp_path): - try: - import matplotlib.pyplot as plt - except: - return + plt = pytest.importorskip("matplotlib.pyplot") shutil.copy(os.path.join("pst","pest.pst"), os.path.join(tmp_path, "pest.pst")) shutil.copy(os.path.join("pst","pest.rei"), @@ -259,10 +247,7 @@ def ensemble_1to1_test(tmp_path): def ensemble_summary_test(tmp_path): - try: - import matplotlib.pyplot as plt - except: - return + plt = pytest.importorskip("matplotlib.pyplot") shutil.copy(os.path.join("pst", "pest.pst"), os.path.join(tmp_path, "pest.pst")) @@ -295,10 +280,7 @@ def ensemble_summary_test(tmp_path): ) # def cov_test(): -# try: -# import matplotlib.pyplot as plt -# except: -# return +# plt = pytest.importorskip("matplotlib.pyplot") # # import os # import numpy as np @@ -323,7 +305,7 @@ def ensemble_summary_test(tmp_path): def ensemble_change_test(tmp_path): - import matplotlib.pyplot as plt + plt = pytest.importorskip("matplotlib.pyplot") os.chdir(tmp_path) pst = pyemu.Pst.from_par_obs_names(par_names=["p1","p2"]) cov = pyemu.Cov(np.array([[1.0,0.0],[0.0,1.0]]),names=["p1","p2"]) diff --git a/autotest/pst_from_tests.py b/autotest/pst_from_tests.py index 20e4eeb7..f759c5fa 100644 --- a/autotest/pst_from_tests.py +++ b/autotest/pst_from_tests.py @@ -73,18 +73,14 @@ def setup_tmp(od, tmp_d, sub=None): def test_freyberg(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd - from pyemu import PyemuWarning + from pyemu.pyemu_warnings import PyemuWarning pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return - org_model_ws = os.path.join("..", "examples", "freyberg_sfr_update") tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -296,15 +292,12 @@ def test_freyberg(tmp_path): def test_freyberg_prior_build(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join("..", "examples", "freyberg_sfr_update") tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -607,7 +600,7 @@ def add_py_function_test(tmp_path): def test_mf6_freyberg(tmp_path): # import sys # sys.path.insert(0, os.path.join("..", "..", "pypestutils")) - import pypestutils as ppu + ppu = pytest.importorskip("pypestutils") pf,sim = setup_freyberg_mf6(tmp_path) m = sim.get_model() @@ -870,7 +863,7 @@ def test_mf6_freyberg(tmp_path): list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper), index_cols=[0, 1, 2], + pargp="twel_mlt", index_cols=[0, 1, 2], use_cols=[3], upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs, mfile_skip=1) @@ -885,7 +878,7 @@ def test_mf6_freyberg(tmp_path): list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper), index_cols=[0, 1, 2], + pargp="twel_mlt", index_cols=[0, 1, 2], use_cols=[3], upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs, mfile_skip='#') @@ -899,7 +892,7 @@ def test_mf6_freyberg(tmp_path): list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper), index_cols=['#k', 'i', 'j'], + pargp="twel_mlt", index_cols=['#k', 'i', 'j'], use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) # add temporally indep, but spatially correlated wel flux pars @@ -912,7 +905,7 @@ def test_mf6_freyberg(tmp_path): list_file = "freyberg6.wel_stress_period_data_{0}.txt".format(kper+1) pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper), index_cols=['#k', 'i', 'j'], + pargp="twel_mlt", index_cols=['#k', 'i', 'j'], use_cols=['flux'], upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs, mfile_skip=1) @@ -928,7 +921,7 @@ def test_mf6_freyberg(tmp_path): kper = int(list_file.split(".")[1].split('_')[-1]) - 1 # add spatially constant, but temporally correlated wel flux pars pf.add_parameters(filenames=list_file,par_type="constant",par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper),index_cols=[0,1,2],use_cols=[3], + pargp="twel_mlt",index_cols=[0,1,2],use_cols=[3], upper_bound=1.5,lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) # add temporally indep, but spatially correlated wel flux pars @@ -1279,15 +1272,12 @@ def mf6_freyberg_shortnames_test(tmp_path): def mf6_freyberg_da_test(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6_da') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -1355,7 +1345,7 @@ def mf6_freyberg_da_test(tmp_path): kper = int(list_file.split(".")[1].split('_')[-1]) - 1 # add spatially constant, but temporally correlated wel flux pars pf.add_parameters(filenames=list_file,par_type="constant",par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper),index_cols=[0,1,2],use_cols=[3], + pargp="twel_mlt",index_cols=[0,1,2],use_cols=[3], upper_bound=1.5,lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs) # add temporally indep, but spatially correlated wel flux pars @@ -1445,10 +1435,7 @@ def mf6_freyberg_da_test(tmp_path): def setup_freyberg_mf6(wd, model="freyberg_mf6", **kwargs): - try: - import flopy - except: - return + flopy = pytest.importorskip("flopy") # try: # model = request.param # except AttributeError: @@ -1783,7 +1770,7 @@ def direct_arraypars_test(tmp_path): def direct_listpars_test(tmp_path): - import flopy + flopy = pytest.importorskip("flopy") pf, sim = setup_freyberg_mf6(tmp_path) m = sim.get_model() template_ws = pf.new_d @@ -1930,16 +1917,12 @@ def direct_listpars_test(tmp_path): @pytest.mark.skip("now broken down into parts to avoid some mem issues") def mf6_freyberg_direct_test(tmp_path): - + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -2270,15 +2253,12 @@ def mf6_freyberg_direct_test(tmp_path): def mf6_freyberg_varying_idomain(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -2429,15 +2409,12 @@ def mf6_freyberg_varying_idomain(tmp_path): def xsec_test(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join('..', 'examples', 'xsec') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -2478,16 +2455,12 @@ def xsec_test(tmp_path): def mf6_freyberg_short_direct_test(tmp_path): - + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -2965,7 +2938,7 @@ def test_add_list_parameters(self): par_name_base="xywel", index_cols={'lay': 0, 'x': 4, 'y': 5}, use_cols=[3], - pargp=f'xywel', + pargp='xywel', geostruct=self.grid_gs, rebuild_pst=True ) @@ -3265,16 +3238,13 @@ def invest(): def pstfrom_profile(): + flopy = pytest.importorskip("flopy") import cProfile import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join("..", "examples", "freyberg_sfr_update") nam_file = "freyberg.nam" @@ -3389,17 +3359,12 @@ def pstfrom_profile(): def mf6_freyberg_arr_obs_and_headerless_test(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return - - org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -3526,19 +3491,16 @@ def mf6_freyberg_arr_obs_and_headerless_test(tmp_path): def test_mf6_freyberg_pp_locs(tmp_path): + # flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - #try: - import flopy - #except: - # return # import sys # sys.path.insert(0,os.path.join("..","..","pypestutils")) - import pypestutils as ppu + ppu = pytest.importorskip("pypestutils") pf, sim = setup_freyberg_mf6(tmp_path) m = sim.get_model() @@ -3664,9 +3626,9 @@ def test_mf6_freyberg_pp_locs(tmp_path): def usg_freyberg_test(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd - import flopy import pyemu pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) @@ -3874,7 +3836,7 @@ def usg_freyberg_test(tmp_path): def mf6_add_various_obs_test(tmp_path): - import flopy + flopy = pytest.importorskip("flopy") org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -3957,19 +3919,16 @@ def _add_big_obsffile(pf, profile=False, nchar=50000): def mf6_subdir_test(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return # import sys # sys.path.insert(0, os.path.join("..", "..", "pypestutils")) - import pypestutils as ppu + ppu = pytest.importorskip("pypestutils") org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') sd = "sub_dir" @@ -4076,7 +4035,7 @@ def mf6_subdir_test(tmp_path): ) pf.add_parameters(filenames=list_file, par_type="constant", par_name_base="twel_mlt_{0}".format(kper), - pargp="twel_mlt".format(kper), index_cols=[0, 1, 2], + pargp="twel_mlt", index_cols=[0, 1, 2], use_cols=[3], upper_bound=1.5, lower_bound=0.5, datetime=dts[kper], geostruct=rch_temporal_gs, mfile_skip=1) @@ -4261,16 +4220,13 @@ def mf6_subdir_test(tmp_path): def shortname_conversion_test(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd import re pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return tmp_model_ws = Path(tmp_path, "temp_pst_from") if os.path.exists(tmp_model_ws): @@ -4574,15 +4530,12 @@ def test_vertex_grid(tmp_path): return def test_defaults(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -4694,19 +4647,15 @@ def list_float_int_index_test(tmp_path): #def mf6_freyberg_thresh_invest(setup_freyberg_mf6): def mf6_freyberg_thresh_test(tmp_path): - + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd # pd.set_option('display.max_rows', 500) # pd.set_option('display.max_columns', 500) # pd.set_option('display.width', 1000) - # try: - import flopy - # except: - # return # import sys # sys.path.insert(0,os.path.join("..","..","pypestutils")) - import pypestutils as ppu + ppu = pytest.importorskip("pypestutils") org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') @@ -5014,7 +4963,7 @@ def mf6_freyberg_thresh_test(tmp_path): def plot_thresh(m_d): - import flopy + flopy = pytest.importorskip("flopy") sim = flopy.mf6.MFSimulation.load(sim_ws=m_d) dis = sim.get_model().dis @@ -5453,6 +5402,7 @@ def hyperpars_test(tmp_path): def mf6_freyberg_ppu_hyperpars_invest(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd import os @@ -5461,15 +5411,11 @@ def mf6_freyberg_ppu_hyperpars_invest(tmp_path): import pyemu # sys.path.insert(0,os.path.join("..","..","pypestutils")) - import pypestutils as ppu + ppu = pytest.importorskip("pypestutils") pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -5665,20 +5611,15 @@ def mf6_freyberg_ppu_hyperpars_invest(tmp_path): def mf6_freyberg_ppu_hyperpars_thresh_invest(tmp_path): - + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd # pd.set_option('display.max_rows', 500) # pd.set_option('display.max_columns', 500) # pd.set_option('display.width', 1000) - # try: - import flopy - # except: - # return import sys sys.path.insert(0,os.path.join("..","..","pypestutils")) - - import pypestutils as ppu + ppu = pytest.importorskip("pypestutils") org_model_ws = os.path.join('..', 'examples', 'freyberg_mf6') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -6053,7 +5994,7 @@ def mf6_freyberg_ppu_hyperpars_thresh_invest(tmp_path): def arrayskip_test(tmp_path): from pathlib import Path - sr = pyemu.SpatialReference(delr=[1000]*81, delc=[1000]*57, xll=841955, yll=2208285) + sr = pyemu.helpers.SpatialReference(delr=[1000]*81, delc=[1000]*57, xll=841955, yll=2208285) pf = pyemu.utils.PstFrom(Path('utils','arrayskip'), Path(tmp_path, "template"), spatial_reference=sr) shutil.copy(Path(pf.new_d, "AWC_subset.txt"), Path(pf.new_d, "AWC_subset_1.txt")) @@ -6178,7 +6119,7 @@ def test_dup_idxs(tmp_path): index_cols=['i', 'j'], use_cols='q', ofile_sep=',') - except Exception as e: + except Exception: pass else: raise Exception("Expected error not raised") @@ -6204,7 +6145,7 @@ def test_dup_idxs(tmp_path): def invest_vertexpp_setup_speed(): - import flopy + flopy = pytest.importorskip("flopy") from flopy.utils.gridgen import Gridgen from pathlib import Path from shapely import Polygon @@ -6250,15 +6191,12 @@ def invest_vertexpp_setup_speed(): def xsec_pars_as_obs_test(tmp_path): + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd pd.set_option('display.max_rows', 500) pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) - try: - import flopy - except: - return org_model_ws = os.path.join('..', 'examples', 'xsec') tmp_model_ws = setup_tmp(org_model_ws, tmp_path) @@ -6325,7 +6263,7 @@ def xsec_pars_as_obs_test(tmp_path): def draw_consistency_test(tmp_path): - import flopy + flopy = pytest.importorskip("flopy") org_d = os.path.join("tests",'synthdewater') tmp_d = os.path.join(os.path.join(tmp_path,'tmp')) diff --git a/autotest/pst_tests.py b/autotest/pst_tests.py index 33ba7f4c..3f620e40 100644 --- a/autotest/pst_tests.py +++ b/autotest/pst_tests.py @@ -3,6 +3,7 @@ import time from pathlib import Path import numpy as np +import pytest def setup_tmp(od, tmp_path): @@ -180,8 +181,8 @@ def load_test(tmp_path): for pst_file in pst_files: if "pest_tied_tester" not in pst_file: continue - if pst_file.endswith(".pst") and not "comments" in pst_file and \ - not "missing" in pst_file: + if pst_file.endswith(".pst") and "comments" not in pst_file and \ + "missing" not in pst_file: print(pst_file) try: p = Pst(os.path.join(pst_dir, pst_file)) @@ -574,6 +575,7 @@ def res_stats_test(): def write_tables_test(tmp_path): + pytest.importorskip("jinja2") import os import pyemu @@ -1648,7 +1650,7 @@ def results_ies_3_test(): pst.add_results(m_d2) try: pst.add_results(m_d2) - except Exception as e: + except Exception: pass else: raise Exception("should have failed...") diff --git a/autotest/pst_tests_ignore.py b/autotest/pst_tests_ignore.py index 12d14448..59499394 100644 --- a/autotest/pst_tests_ignore.py +++ b/autotest/pst_tests_ignore.py @@ -909,7 +909,7 @@ def results_ies_3_test(): pst.add_results(m_d2) try: pst.add_results(m_d2) - except Exception as e: + except Exception: pass else: raise Exception("should have failed...") diff --git a/autotest/utils_tests.py b/autotest/utils_tests.py index 5c96f7ba..63896ed8 100644 --- a/autotest/utils_tests.py +++ b/autotest/utils_tests.py @@ -173,7 +173,6 @@ def covariance_matrix_test(): def setup_ppcov_simple(tmp_path): import os - import platform exe_file = os.path.join("utils","ppcov.exe") print(platform.platform()) @@ -233,7 +232,6 @@ def ppcov_simple_test(): def setup_ppcov_complex(tmp_path): import os - import platform exe_file = os.path.join("utils","ppcov.exe") print(platform.platform()) @@ -330,12 +328,9 @@ def tpl_to_dataframe_test(tmp_path): # risk=0.975) def setup_pp_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import pyemu - try: - import flopy - except: - return o_model_ws = os.path.join("..","examples","Freyberg","extra_crispy") model_ws = os.path.join(tmp_path, "extra_crispy") shutil.copytree(o_model_ws, model_ws) @@ -512,15 +507,12 @@ def zero_order_regul_test(tmp_path): def kl_test(tmp_path): + flopy = pytest.importorskip("flopy") + plt = pytest.importorskip("matplotlib.pyplot") + import os import numpy as np import pyemu - import matplotlib.pyplot as plt - try: - import flopy - except: - print("flopy not imported...") - return o_model_ws = os.path.join("..","verification","Freyberg","extra_crispy") model_ws = Path(tmp_path, "extra_crispy") shutil.copytree(o_model_ws, model_ws) @@ -614,12 +606,7 @@ def ok_test(tmp_path): def ok_grid_test(tmp_path): - - try: - import flopy - except: - return - + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd import pyemu @@ -647,12 +634,7 @@ def ok_grid_test(tmp_path): def ok_grid_zone_test(tmp_path): - - try: - import flopy - except: - return - + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd import pyemu @@ -687,13 +669,10 @@ def ok_grid_zone_test(tmp_path): def ppk2fac_verf_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import numpy as np import pyemu - try: - import flopy - except: - return ws = os.path.join("..","verification","Freyberg") filedict = dict(gspc_file="grid.spc", pp_file="pp_00_pp.dat", @@ -744,15 +723,11 @@ def ppk2fac_verf_test(tmp_path): def mflist_budget_test(tmp_path): + flopy = pytest.importorskip("flopy") import pyemu import os import pandas as pd import shutil - try: - import flopy - except: - print("no flopy...") - return model_ws = os.path.join("..", "examples", "Freyberg_transient") shutil.copytree(model_ws, Path(tmp_path, "Freyberg_transient")) model_ws = Path(tmp_path, "Freyberg_transient") @@ -784,14 +759,10 @@ def mflist_budget_test(tmp_path): def mtlist_budget_test(tmp_path): + flopy = pytest.importorskip("flopy") import pyemu import shutil import os - try: - import flopy - except: - print("no flopy...") - return list_filenames = [Path("utils","mt3d.list"), Path("utils", "mt3d_imm_sor.lst")] _ = [shutil.copy(list_filename, Path(tmp_path, list_filename.name)) @@ -936,11 +907,7 @@ def geostat_draws_test(tmp_path): # def linearuniversal_krige_test(): -# try: -# import flopy -# except: -# return -# +# flopy = pytest.importorskip("flopy") # import numpy as np # import pandas as pd # import pyemu @@ -1011,14 +978,12 @@ def load_sgems_expvar_test(): def read_hydmod_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import numpy as np import pandas as pd import pyemu - try: - import flopy - except: - return + df, outfile = pyemu.gw_utils.modflow_read_hydmod_file(os.path.join('utils','freyberg.hyd.bin'), os.path.join(tmp_path,'freyberg.hyd.bin.dat')) df = pd.read_csv(os.path.join(tmp_path, 'freyberg.hyd.bin.dat'), sep=r"\s+") @@ -1028,13 +993,10 @@ def read_hydmod_test(tmp_path): def make_hydmod_insfile_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import shutil import pyemu - try: - import flopy - except: - return shutil.copy2(os.path.join('utils','freyberg.hyd.bin'),os.path.join(tmp_path,'freyberg.hyd.bin')) bd = os.getcwd() os.chdir(tmp_path) @@ -1049,14 +1011,11 @@ def make_hydmod_insfile_test(tmp_path): def plot_summary_test(tmp_path): + plt = pytest.importorskip("matplotlib.pyplot") import os import pandas as pd import pyemu - try: - import matplotlib.pyplot as plt - except: - return par_df = pd.read_csv(os.path.join("utils","freyberg_pp.par.usum.csv"), index_col=0) @@ -1082,13 +1041,10 @@ def plot_summary_test(tmp_path): def hds_timeseries_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import shutil import numpy as np - try: - import flopy - except: - return import pyemu model_ws =os.path.join("..","examples","Freyberg_transient") @@ -1162,14 +1118,11 @@ def hds_timeseries_test(tmp_path): def grid_obs_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import shutil import numpy as np import pandas as pd - try: - import flopy - except: - return import pyemu m_ws = os.path.join("..", "examples", "freyberg_sfr_update") @@ -1297,14 +1250,11 @@ def grid_obs_test(tmp_path): def postprocess_inactive_conc_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import shutil import numpy as np import pandas as pd - try: - import flopy - except: - return import pyemu bd = os.getcwd() model_ws = os.path.join("..", "examples", "Freyberg_transient") @@ -1359,10 +1309,10 @@ def gw_sft_ins_test(tmp_path): def sfr_helper_test(tmp_path): # TODO: need attention to move IO to tmp_path (particularly writing) + flopy = pytest.importorskip("flopy") import os import pandas as pd import pyemu - import flopy import numpy as np #setup the process @@ -1527,9 +1477,9 @@ def sfr_helper_test(tmp_path): # TODO: need attention to move IO to tmp_path (p def sfr_obs_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import pyemu - import flopy [shutil.copy(os.path.join("utils",f"freyberg.{ext}"), tmp_path) for ext in ["sfr.out", "nam", "dis", "bas"]] @@ -1550,9 +1500,9 @@ def sfr_obs_test(tmp_path): os.chdir(bd) def sfr_reach_obs_test(tmp_path): + flopy = pytest.importorskip("flopy") import os import pyemu - import flopy import pandas as pd import numpy as np [shutil.copy(os.path.join("utils", f"freyberg.{ext}"), tmp_path) @@ -1666,8 +1616,8 @@ def write_jactest_test(tmp_path): def plot_id_bar_test(tmp_path): + plt = pytest.importorskip("matplotlib.pyplot") import pyemu - # import matplotlib.pyplot as plt w_dir = "la" shutil.copy(os.path.join(w_dir, "pest.jcb"), os.path.join(tmp_path, "pest.jcb")) shutil.copy(os.path.join(w_dir, "pest.pst"), os.path.join(tmp_path, "pest.pst")) @@ -1709,11 +1659,8 @@ def jco_from_pestpp_runstorage_test(tmp_path): def hfb_test(tmp_path): + flopy = pytest.importorskip("flopy") import os - try: - import flopy - except: - return import pyemu org_model_ws = os.path.join("..", "examples", "freyberg_sfr_update") @@ -1746,11 +1693,8 @@ def hfb_test(tmp_path): def hfb_zn_mult_test(tmp_path): + flopy = pytest.importorskip("flopy") import os - try: - import flopy - except: - return import pyemu import pandas as pd @@ -1888,13 +1832,11 @@ def smp_dateparser_test(tmp_path): def fieldgen_dev(tmp_path): + flopy = pytest.importorskip("flopy") + plt = pytest.importorskip("matplotlib.pyplot") import shutil import numpy as np import pandas as pd - try: - import flopy - except: - return import pyemu from pyemu.legacy import PstFromFlopyModel @@ -1919,7 +1861,6 @@ def fieldgen_dev(tmp_path): struct_dict = {gs:["hk","ss"]} df = pyemu.helpers.run_fieldgen(m,10,struct_dict,cwd=new_model_ws) - import matplotlib.pyplot as plt i = df.index.map(lambda x: int(x.split('_')[0])) j = df.index.map(lambda x: int(x.split('_')[1])) arr = np.zeros((m.nrow,m.ncol)) @@ -1929,12 +1870,7 @@ def fieldgen_dev(tmp_path): def ok_grid_invest(tmp_path): - - try: - import flopy - except: - return - + flopy = pytest.importorskip("flopy") import numpy as np import pandas as pd import pyemu @@ -1964,11 +1900,7 @@ def ok_grid_invest(tmp_path): assert diff < 1.0e-10 def specsim_test(): - try: - import flopy - except: - return - + flopy = pytest.importorskip("flopy") import numpy as np import pyemu num_reals = 100 @@ -1987,7 +1919,7 @@ def specsim_test(): try: ss = pyemu.geostats.SpecSim2d(geostruct=gs,delx=broke_delr,dely=delc) - except Exception as e: + except Exception: pass else: raise Exception("should have failed") @@ -1996,14 +1928,14 @@ def specsim_test(): gs = pyemu.geostats.GeoStruct(variograms=variograms, transform="none", nugget=nugget) try: ss = pyemu.geostats.SpecSim2d(geostruct=gs,delx=broke_delr,dely=delc) - except Exception as e: + except Exception: pass else: raise Exception("should have failed") try: ss = pyemu.geostats.SpecSim2d(geostruct=gs,delx=delr,dely=broke_delc) - except Exception as e: + except Exception: pass else: raise Exception("should have failed") @@ -2046,12 +1978,8 @@ def specsim_test(): assert np.abs(mean - mean_value) < 0.1 def aniso_invest(): - - try: - import flopy - except: - return - + flopy = pytest.importorskip("flopy") + plt = pytest.importorskip("matplotlib.pyplot") import numpy as np import pandas as pd import pyemu @@ -2075,7 +2003,6 @@ def aniso_invest(): ss = pyemu.geostats.SpecSim2d(geostruct=gs, delx=delr, dely=delc) reals2 = ss.draw_arrays(num_reals=num_reals, mean_value=mean_value) - import matplotlib.pyplot as plt fig,axes = plt.subplots(1,2,figsize=(6,3)) axes[0].imshow(reals2[0]) axes[1].imshow(reals1[0]) @@ -2085,7 +2012,6 @@ def aniso_invest(): def run_test(): import pyemu - import platform if "window" in platform.platform().lower(): pyemu.os_utils.run("echo test") @@ -2099,7 +2025,6 @@ def run_test(): raise Exception("should have failed") def run_sp_success_test(): - import platform if "window" in platform.platform().lower(): pyemu.os_utils.run("echo test", use_sp=True, shell=True) else: @@ -2112,7 +2037,6 @@ def run_sp_failure_test(): shell=False, logfile=False) def run_sp_capture_output_test(tmp_path): - import platform if platform.system() == "Windows": shell = True else: @@ -2127,7 +2051,6 @@ def run_sp_capture_output_test(tmp_path): assert "Hello World" in content def run_sp_verbose_test(capsys): - import platform if platform.system() == "Windows": shell = True else: @@ -2305,10 +2228,10 @@ def geostat_prior_builder2_test(tmp_path): def temporal_draw_invest(): + plt = pytest.importorskip("matplotlib.pyplot") import numpy as np import pandas as pd import pyemu - import matplotlib.pyplot as plt from datetime import datetime v = pyemu.geostats.ExpVario(contribution=1.0,a=500) gs = pyemu.geostats.GeoStruct(variograms=v) @@ -2495,6 +2418,8 @@ def ac_draw_test(tmp_path): def test_fake_frun(tmp_path): from pst_from_tests import ies_exe_path, setup_freyberg_mf6 + if not ies_exe_path: + pytest.skip("missing ies_exe_path") pf, sim = setup_freyberg_mf6(tmp_path) v = pyemu.geostats.ExpVario(contribution=1.0, a=500) gs = pyemu.geostats.GeoStruct(variograms=v, transform='log') @@ -2609,22 +2534,20 @@ def thresh_pars_test(): def test_ppu_import(): - import pypestutils as ppu + ppu = pytest.importorskip("pypestutils") @pytest.mark.timeout(method="thread") def ppu_geostats_test(tmp_path): + flopy = pytest.importorskip("flopy") + plt = pytest.importorskip("matplotlib.pyplot") import sys import os import numpy as np - import matplotlib.pyplot as plt import pyemu - - import flopy sys.path.insert(0,os.path.join("..","..","pypestutils")) - - import pypestutils as ppu + ppu = pytest.importorskip("pypestutils") o_model_ws = os.path.join("..","examples","Freyberg","extra_crispy") model_ws = os.path.join(tmp_path, "extra_crispy") @@ -2707,6 +2630,8 @@ def ppw_worker(id_num,case,t_d,host,port,frun): @pytest.mark.timeout(method="thread") def test_pypestworker(tmp_path): + if not mou_exe_path: + pytest.skip("missing mou_exe_path") from datetime import datetime import numpy as np import subprocess as sp @@ -2812,6 +2737,9 @@ def test_pypestworker(tmp_path): def gpr_compare_invest(): + if not mou_exe_path: + pytest.skip("missing mou_exe_path") + plt = pytest.importorskip("matplotlib.pyplot") import numpy as np from sklearn.gaussian_process import GaussianProcessRegressor case = "zdt1" @@ -2875,8 +2803,6 @@ def gpr_compare_invest(): o1 = o1.loc[o1.is_feasible == True, :] o1 = o1.loc[o1.nsga2_front == 1, :] - - import matplotlib.pyplot as plt o2 = pd.read_csv(os.path.join(gpr_m_d, case + ".{0}.obs_pop.csv".format(max(0, gpst.control_data.noptmax)))) fig,ax = plt.subplots(1,1,figsize=(5,5)) ax.scatter(o1.obj_1,o1.obj_2,c="r",s=10) @@ -2973,6 +2899,8 @@ def gpr_compare_invest(): def gpr_constr_invest(): + if not mou_exe_path: + pytest.skip("missing mou_exe_path") import numpy as np from sklearn.gaussian_process import GaussianProcessRegressor case = "constr" @@ -3155,6 +3083,8 @@ def gpr_constr_invest(): def gpr_zdt1_invest(): + if not mou_exe_path: + pytest.skip("missing mou_exe_path") import numpy as np import subprocess as sp import multiprocessing as mp diff --git a/examples/Freyberg/run.py b/examples/Freyberg/run.py index fc234717..5802270e 100644 --- a/examples/Freyberg/run.py +++ b/examples/Freyberg/run.py @@ -38,7 +38,7 @@ def process(): m2s_exe = os.path.join("mod2smp.exe") else: m2s_exe = None - print m2s_exe + "< "+os.path.join("misc","mod2smp.in") + print(m2s_exe + "< "+os.path.join("misc","mod2smp.in")) os.system(m2s_exe + "< "+os.path.join("misc","mod2smp.in")) # sw-gw exchange diff --git a/examples/PST_to_pyemu_to_Schur_Overview.ipynb b/examples/PST_to_pyemu_to_Schur_Overview.ipynb index dae4481e..81bd099e 100644 --- a/examples/PST_to_pyemu_to_Schur_Overview.ipynb +++ b/examples/PST_to_pyemu_to_Schur_Overview.ipynb @@ -7,7 +7,8 @@ "outputs": [], "source": [ "import pyemu\n", - "import os, shutil\n", + "import os\n", + "import shutil\n", "import pandas as pd\n", "import flopy\n", "import numpy as np\n", diff --git a/examples/pstfrom_mf6.ipynb b/examples/pstfrom_mf6.ipynb index c2e21f67..63683920 100644 --- a/examples/pstfrom_mf6.ipynb +++ b/examples/pstfrom_mf6.ipynb @@ -911,10 +911,10 @@ ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# load the mf6 model with flopy to get the spatial reference\n", "sim = flopy.mf6.MFSimulation.load(sim_ws=tmp_model_ws)\n", @@ -1033,7 +1033,7 @@ " # add spatially constant, but temporally correlated parameter\n", " pf.add_parameters(filenames=list_file,par_type=\"constant\",\n", " \t\t\t\t par_name_base=\"twel_mlt_{0}\".format(kper),\n", - " pargp=\"twel_mlt\".format(kper),index_cols=[0,1,2],\n", + " pargp=\"twel_mlt\",index_cols=[0,1,2],\n", " use_cols=[3],upper_bound=1.5,lower_bound=0.5,\n", " datetime=dts[kper], geostruct=temporal_gs)\n", "\n", @@ -1092,11 +1092,11 @@ ] }, { - "metadata": {}, "cell_type": "code", - "outputs": [], "execution_count": null, - "source": "" + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/pyemu/emulators/dsi.py b/pyemu/emulators/dsi.py index 456458e1..07a3ee50 100755 --- a/pyemu/emulators/dsi.py +++ b/pyemu/emulators/dsi.py @@ -268,7 +268,7 @@ def prepare_pestpp(self, t_d=None, observation_data=None): pname = "dsi_par{0:04d}".format(i) dsi_pnames.append(pname) fin.write("{0},0.0\n".format(pname)) - ftpl.write("{0},~ {0} ~\n".format(pname, pname)) + ftpl.write("{0},~ {0} ~\n".format(pname, )) fin.close() ftpl.close() self.logger.log("creating tpl files") @@ -448,7 +448,7 @@ def prepare_dsivc(self, decvar_names, t_d=None, pst=None, oe=None, track_stack=F self.dsi_args = dsi_args out_files = [] - self.logger.statement(f"preparing stack stats observations...") + self.logger.statement("preparing stack stats observations...") assert isinstance(oe, ObservationEnsemble), "oe must be an ObservationEnsemble" if oe.index.name is None: id_vars="index" @@ -459,7 +459,7 @@ def prepare_dsivc(self, decvar_names, t_d=None, pst=None, oe=None, track_stack=F stack_stats['obsnme'] = stack_stats.apply(lambda x: x.variable+"_stat:"+x.stat,axis=1) stack_stats.set_index("obsnme",inplace=True) stack_stats = stack_stats.obsval - self.logger.statement(f"stack osb recorded to dsi.stack_stats.csv...") + self.logger.statement("stack osb recorded to dsi.stack_stats.csv...") out_file = os.path.join(t_d,"dsi.stack_stats.csv") out_files.append(out_file) stack_stats.to_csv(out_file,float_format="%.6e") @@ -481,7 +481,7 @@ def prepare_dsivc(self, decvar_names, t_d=None, pst=None, oe=None, track_stack=F - self.logger.statement(f"prepare DSIVC template files...") + self.logger.statement("prepare DSIVC template files...") dsi_in_file = os.path.join(t_d, "dsivc_pars.csv") dsi_tpl_file = dsi_in_file + ".tpl" ftpl = open(dsi_tpl_file, 'w') @@ -497,10 +497,10 @@ def prepare_dsivc(self, decvar_names, t_d=None, pst=None, oe=None, track_stack=F ftpl.close() - self.logger.statement(f"building DSIVC control file...") + self.logger.statement("building DSIVC control file...") pst_dsivc = Pst.from_io_files([dsi_tpl_file],[dsi_in_file],[i+".ins" for i in out_files],out_files,pst_path=".") - self.logger.statement(f"setting dec var bounds...") + self.logger.statement("setting dec var bounds...") par = pst_dsivc.parameter_data # set all parameters fixed par.loc[:,"partrans"] = "fixed" @@ -511,12 +511,12 @@ def prepare_dsivc(self, decvar_names, t_d=None, pst=None, oe=None, track_stack=F par.loc[decvar_names,"parlbnd"] = self.data.loc[:,decvar_names].min() par.loc[decvar_names,"parval1"] = self.data.loc[:,decvar_names].quantile(.5) - self.logger.statement(f"zero-weighting observation data...") + self.logger.statement("zero-weighting observation data...") # prepemtpively set obs weights 0.0 obs = pst_dsivc.observation_data obs.loc[:,"weight"] = 0.0 - self.logger.statement(f"getting obs metadata from DSI observation_data...") + self.logger.statement("getting obs metadata from DSI observation_data...") obsorg = pst.observation_data.copy() columns = [i for i in obsorg.columns if i !='obsnme'] for o in obsorg.obsnme.values: @@ -528,7 +528,7 @@ def prepare_dsivc(self, decvar_names, t_d=None, pst=None, oe=None, track_stack=F #obs.loc[stack.index,"obgnme"] = "stack" - self.logger.statement(f"building dsivc_forward_run.py...") + self.logger.statement("building dsivc_forward_run.py...") pst_dsivc.model_command = "python dsivc_forward_run.py" from pyemu.utils.helpers import dsivc_forward_run function_source = inspect.getsource(dsivc_forward_run) @@ -538,14 +538,14 @@ def prepare_dsivc(self, decvar_names, t_d=None, pst=None, oe=None, track_stack=F file.write("if __name__ == \"__main__\":\n") file.write(f" {function_source.split('(')[0].split('def ')[1]}(ies_exe_path='{ies_exe_path}')\n") - self.logger.statement(f"preparing nominal initial population...") + self.logger.statement("preparing nominal initial population...") if mou_population_size is None: # set the population size to 2 * number of decision variables # this is a good rule of thumb for MOU mou_population_size = 2 * len(decvar_names) # these should generally be twice the number of decision variables if mou_population_size < 2 * len(decvar_names): - self.logger.statement(f"mou population is less than 2x number of decision variables, this may be too small...") + self.logger.statement("mou population is less than 2x number of decision variables, this may be too small...") # sample 160 sets of decision variables from a unform distribution dvpop = ParameterEnsemble.from_uniform_draw(pst_dsivc,num_reals=mou_population_size) # record to external file for PESTPP-MOU @@ -562,7 +562,7 @@ def prepare_dsivc(self, decvar_names, t_d=None, pst=None, oe=None, track_stack=F pst_dsivc.write(os.path.join(t_d,"dsivc.pst"),version=2) # updating the DSI pst control file - self.logger.statement(f"updating DSI pst control file...") + self.logger.statement("updating DSI pst control file...") self.logger.statement("overwriting dsi.pst file...") pst.observation_data.loc[decvar_names, "weight"] = dsi_args["decvar_weight"] pst.control_data.noptmax = dsi_args["noptmax"] diff --git a/pyemu/en.py b/pyemu/en.py index 37e51ce0..f42d4a0d 100644 --- a/pyemu/en.py +++ b/pyemu/en.py @@ -522,10 +522,8 @@ def _gaussian_draw( covname = "trouble_{0}.cov".format(grp_name) cov_grp.to_ascii(covname) raise Exception( - "error inverting cov for group '{0}'," - + "saved trouble cov to {1}".format( - grp_name, covname - ) + f"error inverting cov for group '{grp_name}', " + f"saved trouble cov to {covname}" ) a, i = Ensemble._get_eigen_projection_matrix(cov_grp.as_2d) diff --git a/pyemu/la.py b/pyemu/la.py index a3ef8f6d..5cb7ce7b 100644 --- a/pyemu/la.py +++ b/pyemu/la.py @@ -430,7 +430,7 @@ def __load_predictions(self): else: try: pred_mat = self.__fromfile(arg, astype=Matrix) - except Exception as e: + except Exception: raise Exception( "forecast argument: " + arg @@ -695,7 +695,7 @@ def fehalf(self): parameter covariance matrix """ - if self.__fehalf != None: + if self.__fehalf is not None: return self.__fehalf self.log("fehalf") self.__fehalf = self.parcov.u * (self.parcov.s ** (0.5)) @@ -711,7 +711,7 @@ def qhalf(self): `pyemu.Matrix`: square root of the cofactor matrix """ - if self.__qhalf != None: + if self.__qhalf is not None: return self.__qhalf self.log("qhalf") self.__qhalf = self.obscov ** (-0.5) diff --git a/pyemu/legacy/pstfromflopy.py b/pyemu/legacy/pstfromflopy.py index f6498552..6faad5cc 100644 --- a/pyemu/legacy/pstfromflopy.py +++ b/pyemu/legacy/pstfromflopy.py @@ -1611,7 +1611,7 @@ def _pp_prep(self, mlt_df): self.log("processing pp_prefix:{0}".format(pp_prefix)) if pp_prefix not in pp_array_file.keys(): self.logger.lraise( - "{0} not in self.pp_array_file.keys()".format( + "{0} not in self.pp_array_file.keys() {1}".format( pp_prefix, ",".join(pp_array_file.keys()) ) ) diff --git a/pyemu/mat/mat_handler.py b/pyemu/mat/mat_handler.py index 1786e910..8a268efe 100644 --- a/pyemu/mat/mat_handler.py +++ b/pyemu/mat/mat_handler.py @@ -1745,7 +1745,7 @@ def drop(self, names, axis): if len(keep_names) != self.__x.shape[0]: raise Exception( "shape-name mismatch:" - + "{0}:{0}".format(len(keep_names), self.__x.shape) + + "{0}:{1}".format(len(keep_names), self.__x.shape) ) self.row_names = keep_names self.col_names = copy.deepcopy(keep_names) @@ -1761,7 +1761,7 @@ def drop(self, names, axis): if len(keep_names) != self.__x.shape[0]: raise Exception( "shape-name mismatch:" - + "{0}:{0}".format(len(keep_names), self.__x.shape) + + "{0}:{1}".format(len(keep_names), self.__x.shape) ) self.row_names = keep_names self.col_names = copy.deepcopy(keep_names) @@ -2148,7 +2148,7 @@ def read_dense(filename, forgive=False, close=True, only_rows=None): f.seek(offset) try: slen = np.fromfile(f, Matrix.integer, 1)[0] - except Exception as e: + except Exception: break try: name = ( @@ -2232,7 +2232,7 @@ def get_dense_binary_info(filename): curr_pos = f.tell() try: slen = np.fromfile(f, Matrix.integer, 1)[0] - except Exception as e: + except Exception: break try: name = ( @@ -2461,7 +2461,7 @@ def from_fortranfile(filename): """ try: from scipy.io import FortranFile - except Exception as e: + except Exception: raise Exception("Matrix.from_fortranfile requires scipy") f = FortranFile(filename, mode="r") itemp1, itemp2 = f.read_ints() @@ -2613,7 +2613,7 @@ def read_ascii(filename): for r in raw: try: x.append(float(r)) - except Exception as e: + except Exception: # overflow if "+" in r: x.append(1.0e30) diff --git a/pyemu/pst/pst_controldata.py b/pyemu/pst/pst_controldata.py index 1592e82c..780a67a4 100644 --- a/pyemu/pst/pst_controldata.py +++ b/pyemu/pst/pst_controldata.py @@ -310,12 +310,12 @@ def _parse_value(value): v = int(value) t = np.int32 f = IFMT - except Exception as e: + except Exception: try: v = float(value) t = np.float64 f = FFMT - except Exception as ee: + except Exception: #string with list to lower and space stripped v = value.lower().replace(" ", "") t = str diff --git a/pyemu/pst/pst_handler.py b/pyemu/pst/pst_handler.py index 897a65d0..5eff4d84 100644 --- a/pyemu/pst/pst_handler.py +++ b/pyemu/pst/pst_handler.py @@ -181,7 +181,7 @@ def __getattr__(self,tag): elif tag.startswith("r"): try: idx = int(tag[1:]) - except Exception as e: + except Exception: pass else: return self.results[self.result_dirs[idx]] @@ -1994,13 +1994,13 @@ def _write_version1(self, new_filename): for group in self.obs_groups: try: group = group.decode() - except Exception as e: + except Exception: pass f_out.write(pst_utils.SFMT(str(group)) + "\n") for group in self.prior_groups: try: group = group.decode() - except Exception as e: + except Exception: pass f_out.write(pst_utils.SFMT(str(group)) + "\n") self._write_df( @@ -3450,7 +3450,7 @@ def write_par_summary_table( if filename.lower().endswith("xlsx") or filename.lower().endswith("xls"): try: pargp_df.to_excel(filename, index=None) - except Exception as e: + except Exception: if filename.lower().endswith("xlsx"): print( "could not export {0} in Excel format. Try installing xlrd".format( @@ -3475,7 +3475,7 @@ def write_par_summary_table( f.write(pargp_df.style.hide(axis='index').to_latex( None, environment='longtable') ) - except (TypeError, AttributeError) as e: + except (TypeError, AttributeError): pargp_df.to_latex(index=False, longtable=True) f.write("\\end{landscape}\n") f.write("\\end{center}\n") @@ -3567,7 +3567,7 @@ def write_obs_summary_table(self, filename=None, group_names=None): if filename.lower().endswith("xlsx") or filename.lower().endswith("xls"): try: obsg_df.to_excel(filename, index=None) - except Exception as e: + except Exception: if filename.lower().endswith("xlsx"): print( "could not export {0} in Excel format. Try installing xlrd".format( @@ -3595,7 +3595,7 @@ def write_obs_summary_table(self, filename=None, group_names=None): f.write(obsg_df.style.hide(axis='index').to_latex( None, environment='longtable') ) - except (TypeError, AttributeError) as e: + except (TypeError, AttributeError): obsg_df.to_latex(index=False, longtable=True) f.write("\\end{landscape}\n") f.write("\\end{center}\n") @@ -3874,7 +3874,7 @@ def try_parse_name_metadata(self): meta_dict = pd.DataFrame(list(meta_dict), index=meta_dict.index) unique_keys = meta_dict.columns.difference(fieldnames) df[unique_keys] = meta_dict[unique_keys] - except Exception as e: + except Exception: print("error parsing metadata from '{0}', continuing".format(name)) def rename_parameters(self, name_dict, pst_path=".", tplmap=None): diff --git a/pyemu/pst/pst_utils.py b/pyemu/pst/pst_utils.py index f9525b66..d330e63f 100644 --- a/pyemu/pst/pst_utils.py +++ b/pyemu/pst/pst_utils.py @@ -1482,7 +1482,7 @@ def _execute_ins_line(self, ins_line, ins_lcount): elif i1 == "l": # if start of instruction is line advance try: nlines = int(ins[1:]) # try and get advance number - except Exception as e: + except Exception: self.throw_ins_error( "casting line advance to int for " "instruction '{0}'".format(ins), @@ -1550,7 +1550,7 @@ def _execute_ins_line(self, ins_line, ins_lcount): val_str = rline[cursor_pos:es] try: val = float(val_str) - except Exception as e: + except Exception: if oname != "dum": self.throw_out_error( "casting string '{0}' to float for instruction '{1}'".format( @@ -1593,7 +1593,7 @@ def _execute_ins_line(self, ins_line, ins_lcount): raw = raw.split(":") try: s_idx = int(raw[0]) - 1 - except Exception as e: + except Exception: self.throw_ins_error( "error converting '{0}' to integer in semi-fixed instruction: '{1}'".format( raw[0], ins @@ -1602,7 +1602,7 @@ def _execute_ins_line(self, ins_line, ins_lcount): ) try: e_idx = int(raw[1]) - except Exception as e: + except Exception: self.throw_ins_error( "error converting '{0}' to integer in semi-fixed instruction: '{1}'".format( raw[1], ins @@ -1639,7 +1639,7 @@ def _execute_ins_line(self, ins_line, ins_lcount): val_str = line[rs_idx:re_idx] try: val = float(val_str) - except Exception as e: + except Exception: if oname != "dum": self.throw_out_error( "casting string '{0}' to float for instruction '{1}'".format( @@ -1664,7 +1664,7 @@ def _execute_ins_line(self, ins_line, ins_lcount): raw = raw.split(":") try: s_idx = int(raw[0]) - 1 - except Exception as e: + except Exception: self.throw_ins_error( "error converting '{0}' to integer in fixed instruction: '{1}'".format( raw[0], ins @@ -1673,7 +1673,7 @@ def _execute_ins_line(self, ins_line, ins_lcount): ) try: e_idx = int(raw[1]) - except Exception as e: + except Exception: self.throw_ins_error( "error converting '{0}' to integer in fixed instruction: '{1}'".format( raw[1], ins @@ -1698,7 +1698,7 @@ def _execute_ins_line(self, ins_line, ins_lcount): val_str = line[s_idx:e_idx] try: val = float(val_str) - except Exception as e: + except Exception: if oname != "dum": self.throw_out_error( "casting string '{0}' to float for instruction '{1}'".format( diff --git a/pyemu/pst/result_handler.py b/pyemu/pst/result_handler.py index e3e56a12..c50c00ef 100644 --- a/pyemu/pst/result_handler.py +++ b/pyemu/pst/result_handler.py @@ -178,7 +178,7 @@ def get_files(self,tag): len(os.path.split(f)[1].split(".")) == 4: try: itr = int(os.path.split(f)[1].split(".")[1]) - except Exception as e: + except Exception: pass else: files.append(f) @@ -391,7 +391,7 @@ def get_files(self, tag): os.path.split(f)[1].lower().startswith(self.case+"."): try: itr = int(os.path.split(f)[1].split(".")[1]) - except Exception as e: + except Exception: pass else: files.append(f) diff --git a/pyemu/sc.py b/pyemu/sc.py index 0008af2e..24576bdb 100644 --- a/pyemu/sc.py +++ b/pyemu/sc.py @@ -354,8 +354,8 @@ def get_conditional_instance(self, parameter_names): if not isinstance(parameter_names, list): parameter_names = [parameter_names] - for iname, name in enumerate(parameter_names): - name = str(name).lower() + for iname, pname in enumerate(parameter_names): + name = str(pname).lower() parameter_names[iname] = name assert name in self.jco.col_names, ( "contribution parameter " + name + " not found jco" @@ -813,7 +813,7 @@ def get_removed_obs_importance(self, obslist_dict=None, reset_zero_weight=None): else: try: self.pst - except Exception as e: + except Exception: raise Exception( "'obslist_dict' not passed and self.pst is not available" ) diff --git a/pyemu/utils/geostats.py b/pyemu/utils/geostats.py index cddda728..ea0baa7f 100644 --- a/pyemu/utils/geostats.py +++ b/pyemu/utils/geostats.py @@ -963,7 +963,7 @@ def calc_factors_grid( try: from pypestutils.pestutilslib import PestUtilsLib use_ppu = True - except Exception as e: + except Exception: pass @@ -2243,7 +2243,7 @@ def read_sgems_variogram_xml(xml_file, return_type=GeoStruct): try: import xml.etree.ElementTree as ET - except Exception as e: + except Exception: print("error import elementtree, skipping...") VARTYPE = {1: SphVario, 2: ExpVario, 3: GauVario, 4: None} assert os.path.exists(xml_file) diff --git a/pyemu/utils/gw_utils.py b/pyemu/utils/gw_utils.py index cb5d71c9..fcc50e9f 100644 --- a/pyemu/utils/gw_utils.py +++ b/pyemu/utils/gw_utils.py @@ -1390,7 +1390,7 @@ def try_cast(x): # normalize for c in df.columns: # print(c) - if not "node" in c: + if "node" not in c: df.loc[:, c] = df.loc[:, c].apply(try_cast) # print(df.loc[df.loc[:,c].apply(lambda x : type(x) == str),:]) if df.dtypes[c] == float: @@ -1436,7 +1436,7 @@ def setup_sfr_seg_parameters( try: import flopy - except Exception as e: + except Exception: return if par_cols is None: par_cols = ["flow", "runoff", "hcond1", "pptsw"] @@ -1676,7 +1676,7 @@ def setup_sfr_reach_parameters(nam_file, model_ws=".", par_cols=["strhc1"]): """ try: import flopy - except Exception as e: + except Exception: return if par_cols is None: par_cols = ["strhc1"] @@ -2341,7 +2341,7 @@ def setup_sfr_reach_obs( os.path.split(ins_file)[-1], os.path.split(sfr_out_file + ".processed")[-1] ) - except Exception as e: + except Exception: pass os.chdir(bd) if df is not None: diff --git a/pyemu/utils/helpers.py b/pyemu/utils/helpers.py index e5e10d48..ad686f92 100644 --- a/pyemu/utils/helpers.py +++ b/pyemu/utils/helpers.py @@ -99,7 +99,7 @@ def _try_pdcol_numeric(x, first=True, intadj=0, **kwargs): x = pd.Series([xx + intadj if isinstance(xx, (int, np.integer)) else xx for xx in x]) else: x = x + intadj if isinstance(x, (int, np.integer)) else x - except ValueError as e: + except ValueError: if first: x = x.apply(_try_pdcol_numeric, first=False, intadj=intadj, **kwargs) else: @@ -319,7 +319,7 @@ def draw_by_group(pst, num_reals=100, sigma_range=6, use_specsim=False, # (setup through add_parameters) for geostruct, par_df_l in struct_dict.items(): par_df = pd.concat(par_df_l) # force to single df - if not 'partype' in par_df.columns: + if 'partype' not in par_df.columns: logger.warn( f"draw() error: use_specsim is {use_specsim} but no column named" f"'partype' to indicate grid based pars in geostruct {geostruct}" @@ -382,12 +382,12 @@ def draw_by_group(pst, num_reals=100, sigma_range=6, use_specsim=False, ) logger.log(f"Drawing {len(subset)} non-specsim pars") if gr_par_pe is not None: - logger.log(f"Joining specsim and non-specsim pars") + logger.log("Joining specsim and non-specsim pars") exist = gr_par_pe.columns.intersection(pe.columns) pe = pe._df.drop(exist, axis=1) # specsim par take precedence pe = pd.concat([pe, gr_par_pe], axis=1) pe = pyemu.ParameterEnsemble(pst=pst, df=pe) - logger.log(f"Joining specsim and non-specsim pars") + logger.log("Joining specsim and non-specsim pars") else: pe = pyemu.ParameterEnsemble(pst=pst, df=gr_par_pe) logger.log("drawing realizations") @@ -4612,6 +4612,7 @@ def gpr_pyworker(pst,host,port,input_df=None,mdf=None,gpr=False): print("WARNING: using legacy gpr_pyworker function, which is deprecated") gpr_pyworker_legacy(pst,host,port,input_df=input_df,mdf=mdf) else: + from pyemu.emulators import GPR if gpr is True: gpr = GPR.load("gpr_emulator.pkl") diff --git a/pyemu/utils/optimization.py b/pyemu/utils/optimization.py index 926704cb..b0155734 100644 --- a/pyemu/utils/optimization.py +++ b/pyemu/utils/optimization.py @@ -1,5 +1,6 @@ from __future__ import print_function, division -import os, sys +import os +import sys import numpy as np import pandas as pd from pyemu import Matrix, Pst, Schur, Cov diff --git a/pyemu/utils/os_utils.py b/pyemu/utils/os_utils.py index 201fec8d..4eff1225 100644 --- a/pyemu/utils/os_utils.py +++ b/pyemu/utils/os_utils.py @@ -11,7 +11,6 @@ import multiprocessing as mp import warnings import socket -import time from datetime import datetime import random import logging @@ -54,7 +53,6 @@ def _istextfile(filename, blocksize=512): are NUL ('\x00') bytes in the block, assume this is a binary file. """ - import sys PY3 = sys.version_info[0] == 3 @@ -825,7 +823,7 @@ def connect(self,is_reconnect=False): except ConnectionRefusedError: continue - except Exception as e: + except Exception: continue self.net_pack = NetPack(timeout=self.timeout,verbose=self.verbose) @@ -982,13 +980,13 @@ def get_parameters(self): self._lock.release() time.sleep(self.timeout) if len(pars) != len(self.par_names): - raise Exception("len(par vals) {0} != len(par names)".format(len(pars),len(self.par_names))) + raise Exception("len(par vals) {0} != len(par names) {1}".format(len(pars),len(self.par_names))) return pd.Series(data=pars,index=self.par_names) def send_observations(self,obsvals,parvals=None,request_more_pars=True): if len(obsvals) != len(self.obs_names): - raise Exception("len(obs vals) {0} != len(obs names)".format(len(obsvals), len(self.obs_names))) + raise Exception("len(obs vals) {0} != len(obs names) {1}".format(len(obsvals), len(self.obs_names))) if isinstance(obsvals,np.ndarray): _obsvals = obsvals elif isinstance(obsvals,pd.Series): diff --git a/pyemu/utils/pp_utils.py b/pyemu/utils/pp_utils.py index e9593e62..b1012c69 100644 --- a/pyemu/utils/pp_utils.py +++ b/pyemu/utils/pp_utils.py @@ -304,7 +304,7 @@ def setup_pilotpoints_grid( "unrecognized field type in par_info:{0}:{1}".format(name, dtype) ) - except Exception as e: + except Exception: raise Exception( "unrecognized field type in par_info:{0}:{1}".format(name, dtype) ) @@ -486,7 +486,7 @@ def write_pp_shapfile(pp_df, shapename=None): "unrecognized field type in par_info:{0}:{1}".format(name, dtype) ) - except Exception as e: + except Exception: raise Exception( "unrecognized field type in par_info:{0}:{1}".format(name, dtype) ) @@ -647,7 +647,7 @@ def get_zoned_ppoints_for_vertexgrid(spacing, zone_array, mg, zone_number=None, # get zone cell x,y xc, yc = mg.xcellcenters, mg.ycellcenters - if zone_number != None: + if zone_number is not None: xc_zone = xc[np.where(zone_array==zone_number)[0]] yc_zone = yc[np.where(zone_array==zone_number)[0]] else: @@ -670,7 +670,7 @@ def _get_ppoints(): # get vertices for model grid/zone polygon verts = [mg.get_cell_vertices(cellid) for cellid in range(mg.ncpl)] - if zone_number != None: + if zone_number is not None: # select zone area verts = [verts[i] for i in np.where(zone_array==zone_number)[0]] # dissolve @@ -688,7 +688,7 @@ def _get_ppoints(): # # # get vertices for model grid/zone polygon # verts = [mg.get_cell_vertices(cellid) for cellid in range(mg.ncpl)] - # if zone_number != None: + # if zone_number is not None: # # select zone area # verts = [verts[i] for i in np.where(zone_array == zone_number)[0]] # # dissolve diff --git a/pyemu/utils/pst_from.py b/pyemu/utils/pst_from.py index ed07b512..f3793d77 100644 --- a/pyemu/utils/pst_from.py +++ b/pyemu/utils/pst_from.py @@ -379,7 +379,7 @@ def _dict_get_xy(self, arg, **kwargs): arg_len = None try: arg_len = len(arg) - except Exception as e: + except Exception: self.logger.lraise( "Pstfrom._dict_get_xy() error getting xy from arg:'{0}' - no len support".format( arg @@ -670,11 +670,11 @@ def draw(self, num_reals=100, sigma_range=6, use_specsim=False, scale_offset=Tru delc = None try: delr = self.spatial_reference.delr - except Exception as e: + except Exception: pass try: delc = self.spatial_reference.delc - except Exception as e: + except Exception: pass # method moved to helpers @@ -2295,8 +2295,8 @@ def add_parameters( if len(use_cols) != len(ult_lbound) != len(ult_ubound): self.logger.lraise( "mismatch in number of columns to use {0} " - "and number of ultimate lower {0} or upper " - "{1} par bounds defined" + "and number of ultimate lower {1} or upper " + "{2} par bounds defined" "".format(len(use_cols), len(ult_lbound), len(ult_ubound)) ) diff --git a/pyproject.toml b/pyproject.toml index 88ecb16f..bf4a5031 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ test = [ "coveralls", "pytest", "pytest-cov", + "pytest-timeout", "pytest-xdist", "flaky", "nbmake", diff --git a/verification/Freyberg/debug.py b/verification/Freyberg/debug.py index 09e3a29f..be308c54 100644 --- a/verification/Freyberg/debug.py +++ b/verification/Freyberg/debug.py @@ -1,4 +1,5 @@ import os +import sys import pyemu pst = pyemu.Pst(os.path.join("freyberg.pst"))