From 3b66ba98c417dd37165533fa0c52cdb30b44e26f Mon Sep 17 00:00:00 2001 From: noman404 Date: Thu, 23 Jan 2025 08:04:46 +0800 Subject: [PATCH 01/29] running on python 3.13.0 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 65339062..13c3116e 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ general_requirements = [ "pytest>=8,<9", - "numpy~=1.26.4", + "numpy~=2.0.0", "sortedcontainers<3", "numexpr<3", "dpath<3", @@ -60,6 +60,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Information Analysis", ], description="Core microsimulation engine enabling country-specific policy models.", From a29dc5ba39c36bafb1c482574210385e8d988ff0 Mon Sep 17 00:00:00 2001 From: noman404 Date: Thu, 23 Jan 2025 18:15:15 +0800 Subject: [PATCH 02/29] upgrade numpy 2.1.0 --- Makefile | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 33da57ef..9120f20a 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ format: install: pip install -e ".[dev]" --config-settings editable_mode=compat - pip install policyengine-us - pip install policyengine-uk +# pip install policyengine-us +# pip install policyengine-uk test-country-template: policyengine-core test policyengine_core/country_template/tests -c policyengine_core.country_template diff --git a/setup.py b/setup.py index 13c3116e..5dbdbea6 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ general_requirements = [ "pytest>=8,<9", - "numpy~=2.0.0", + "numpy~=2.1.0", "sortedcontainers<3", "numexpr<3", "dpath<3", From 990644307e4222f04eae69f93ae4472a29e3a195 Mon Sep 17 00:00:00 2001 From: noman404 Date: Thu, 23 Jan 2025 18:17:45 +0800 Subject: [PATCH 03/29] enable policyengine dependencies for build test --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9120f20a..33da57ef 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ format: install: pip install -e ".[dev]" --config-settings editable_mode=compat -# pip install policyengine-us -# pip install policyengine-uk + pip install policyengine-us + pip install policyengine-uk test-country-template: policyengine-core test policyengine_core/country_template/tests -c policyengine_core.country_template From 446bc1140b165b5db3842473225db35c1eb069d8 Mon Sep 17 00:00:00 2001 From: noman404 Date: Sun, 26 Jan 2025 22:18:26 +0800 Subject: [PATCH 04/29] updated to float64, round and inf --- policyengine_core/populations/group_population.py | 4 ++-- policyengine_core/taxscales/marginal_rate_tax_scale.py | 8 ++++---- policyengine_core/taxscales/rate_tax_scale_like.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/policyengine_core/populations/group_population.py b/policyengine_core/populations/group_population.py index e201d0f5..fd6b1bee 100644 --- a/policyengine_core/populations/group_population.py +++ b/policyengine_core/populations/group_population.py @@ -232,7 +232,7 @@ def max(self, array: ArrayLike, role: Role = None) -> ArrayLike: return self.reduce( array, reducer=numpy.maximum, - neutral_element=-numpy.infty, + neutral_element=-numpy.inf, role=role, ) @@ -256,7 +256,7 @@ def min(self, array: ArrayLike, role: Role = None) -> ArrayLike: return self.reduce( array, reducer=numpy.minimum, - neutral_element=numpy.infty, + neutral_element=numpy.inf, role=role, ) diff --git a/policyengine_core/taxscales/marginal_rate_tax_scale.py b/policyengine_core/taxscales/marginal_rate_tax_scale.py index 2da91c53..3453f019 100644 --- a/policyengine_core/taxscales/marginal_rate_tax_scale.py +++ b/policyengine_core/taxscales/marginal_rate_tax_scale.py @@ -66,12 +66,12 @@ def calc( # # numpy.finfo(float_).eps thresholds1 = numpy.outer( - factor + numpy.finfo(numpy.float_).eps, + factor + numpy.finfo(numpy.float64).eps, numpy.array(self.thresholds + [numpy.inf]), ) if round_base_decimals is not None: - thresholds1 = numpy.round_(thresholds1, round_base_decimals) + thresholds1 = numpy.round(thresholds1, round_base_decimals) a = numpy.maximum( numpy.minimum(base1, thresholds1[:, 1:]) - thresholds1[:, :-1], 0 @@ -82,8 +82,8 @@ def calc( else: r = numpy.tile(self.rates, (len(tax_base), 1)) - b = numpy.round_(a, round_base_decimals) - return numpy.round_(r * b, round_base_decimals).sum(axis=1) + b = numpy.round(a, round_base_decimals) + return numpy.round(r * b, round_base_decimals).sum(axis=1) def combine_bracket( self, diff --git a/policyengine_core/taxscales/rate_tax_scale_like.py b/policyengine_core/taxscales/rate_tax_scale_like.py index b598f447..79d39728 100644 --- a/policyengine_core/taxscales/rate_tax_scale_like.py +++ b/policyengine_core/taxscales/rate_tax_scale_like.py @@ -175,12 +175,12 @@ def bracket_indices( # # numpy.finfo(float_).eps thresholds1 = numpy.outer( - +factor + numpy.finfo(numpy.float_).eps, + +factor + numpy.finfo(numpy.float64).eps, numpy.array(self.thresholds), ) if round_decimals is not None: - thresholds1 = numpy.round_(thresholds1, round_decimals) + thresholds1 = numpy.round(thresholds1, round_decimals) return (base1 - thresholds1 >= 0).sum(axis=1) - 1 From ee7a428239de2c68b569b0f3124981405861e38a Mon Sep 17 00:00:00 2001 From: noman404 Date: Sun, 26 Jan 2025 23:52:54 +0800 Subject: [PATCH 05/29] updated numpy.select default types --- .../parameters/vectorial_parameter_node_at_instant.py | 3 ++- policyengine_core/tools/simulation_dumper.py | 3 ++- tests/core/parameters_fancy_indexing/test_fancy_indexing.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/policyengine_core/parameters/vectorial_parameter_node_at_instant.py b/policyengine_core/parameters/vectorial_parameter_node_at_instant.py index dbd1d14b..8d220df4 100644 --- a/policyengine_core/parameters/vectorial_parameter_node_at_instant.py +++ b/policyengine_core/parameters/vectorial_parameter_node_at_instant.py @@ -203,13 +203,14 @@ def __getitem__(self, key: str) -> Any: enum = type(key[0]) key = numpy.select( [key == item for item in enum], - [item.name for item in enum], + [str(item.name) for item in enum], ) elif isinstance(key, EnumArray): enum = key.possible_values key = numpy.select( [key == item.index for item in enum], [item.name for item in enum], + default="unknown" ) else: key = key.astype("str") diff --git a/policyengine_core/tools/simulation_dumper.py b/policyengine_core/tools/simulation_dumper.py index dd0c0ad6..1a1d6529 100644 --- a/policyengine_core/tools/simulation_dumper.py +++ b/policyengine_core/tools/simulation_dumper.py @@ -95,7 +95,8 @@ def _dump_entity(population, directory): else: encoded_roles = np.select( [population.members_role == role for role in flattened_roles], - [role.key for role in flattened_roles], + [str(role.key) for role in flattened_roles], + default="unknown" ) np.save(os.path.join(path, "members_role.npy"), encoded_roles) diff --git a/tests/core/parameters_fancy_indexing/test_fancy_indexing.py b/tests/core/parameters_fancy_indexing/test_fancy_indexing.py index d753586f..33904903 100644 --- a/tests/core/parameters_fancy_indexing/test_fancy_indexing.py +++ b/tests/core/parameters_fancy_indexing/test_fancy_indexing.py @@ -114,5 +114,5 @@ class TypesZone(Enum): z1 = "Zone 1" z2 = "Zone 2" - zone = np.asarray([TypesZone.z1, TypesZone.z2, TypesZone.z2, TypesZone.z1]) + zone = np.asarray([z.name for z in [TypesZone.z1, TypesZone.z2, TypesZone.z2, TypesZone.z1]]) assert_near(P.single.owner[zone], [100, 200, 200, 100]) From 5fd2a36aa9f28eaaad3a2aaccc2a092aa75b4352 Mon Sep 17 00:00:00 2001 From: noman404 Date: Mon, 27 Jan 2025 00:14:57 +0800 Subject: [PATCH 06/29] fix concat issue --- policyengine_core/commons/formulas.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/policyengine_core/commons/formulas.py b/policyengine_core/commons/formulas.py index 4695cf9c..7d316c36 100644 --- a/policyengine_core/commons/formulas.py +++ b/policyengine_core/commons/formulas.py @@ -92,6 +92,11 @@ def concat(this: ArrayLike[str], that: ArrayLike[str]) -> ArrayType[str]: array(['this1.0', 'that2.5']...) """ + if isinstance(this, tuple): + raise TypeError("First argument must not be a tuple.") + + if isinstance(that, tuple): + raise TypeError("Second argument must not be a tuple.") if isinstance(this, numpy.ndarray) and not numpy.issubdtype( this.dtype, numpy.str_ From 87b06f147831d326e8f599aa698c18aa916aa20b Mon Sep 17 00:00:00 2001 From: noman404 Date: Mon, 27 Jan 2025 00:16:00 +0800 Subject: [PATCH 07/29] fix enum passing issue --- policyengine_core/enums/enum_array.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/policyengine_core/enums/enum_array.py b/policyengine_core/enums/enum_array.py index 1bf5148b..c468a2c7 100644 --- a/policyengine_core/enums/enum_array.py +++ b/policyengine_core/enums/enum_array.py @@ -96,7 +96,8 @@ def decode_to_str(self) -> numpy.str_: """ return numpy.select( [self == item.index for item in self.possible_values], - [item.name for item in self.possible_values], + [str(item.name) for item in self.possible_values], + default="unknown" ) def __repr__(self) -> str: From 14ecf64ecbeda73e14d8df1bbed149c2bc91f67c Mon Sep 17 00:00:00 2001 From: noman404 Date: Mon, 27 Jan 2025 00:16:19 +0800 Subject: [PATCH 08/29] fix tracer line issue --- tests/core/test_tracers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_tracers.py b/tests/core/test_tracers.py index 266b7982..c538346a 100644 --- a/tests/core/test_tracers.py +++ b/tests/core/test_tracers.py @@ -403,7 +403,7 @@ def test_log_aggregate(tracer): lines = tracer.computation_log.lines(aggregate=True) assert ( - lines[0] == " A<2017, (default)> = {'avg': 1.0, 'max': 1, 'min': 1}" + lines[0].strip() == "A<2017, (default)> = {'avg': np.float64(1.0), 'max': np.int64(1), 'min': np.int64(1)}" ) From 66d1b3c84eb3dd7f59893ed560bab9c3538b548d Mon Sep 17 00:00:00 2001 From: noman404 Date: Thu, 30 Jan 2025 00:22:53 +0800 Subject: [PATCH 09/29] fix unknow dtype issue for PE us --- .../parameters/vectorial_parameter_node_at_instant.py | 1 + 1 file changed, 1 insertion(+) diff --git a/policyengine_core/parameters/vectorial_parameter_node_at_instant.py b/policyengine_core/parameters/vectorial_parameter_node_at_instant.py index 8d220df4..3dc3d25f 100644 --- a/policyengine_core/parameters/vectorial_parameter_node_at_instant.py +++ b/policyengine_core/parameters/vectorial_parameter_node_at_instant.py @@ -204,6 +204,7 @@ def __getitem__(self, key: str) -> Any: key = numpy.select( [key == item for item in enum], [str(item.name) for item in enum], + default="unknown" ) elif isinstance(key, EnumArray): enum = key.possible_values From 5b92473fc82bee604be4574f6bd69ac62964cbf6 Mon Sep 17 00:00:00 2001 From: noman404 Date: Thu, 30 Jan 2025 02:46:33 +0800 Subject: [PATCH 10/29] applied format --- policyengine_core/enums/enum_array.py | 2 +- .../parameters/vectorial_parameter_node_at_instant.py | 4 ++-- policyengine_core/tools/simulation_dumper.py | 2 +- .../core/parameters_fancy_indexing/test_fancy_indexing.py | 7 ++++++- tests/core/test_tracers.py | 3 ++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/policyengine_core/enums/enum_array.py b/policyengine_core/enums/enum_array.py index c468a2c7..c518add8 100644 --- a/policyengine_core/enums/enum_array.py +++ b/policyengine_core/enums/enum_array.py @@ -97,7 +97,7 @@ def decode_to_str(self) -> numpy.str_: return numpy.select( [self == item.index for item in self.possible_values], [str(item.name) for item in self.possible_values], - default="unknown" + default="unknown", ) def __repr__(self) -> str: diff --git a/policyengine_core/parameters/vectorial_parameter_node_at_instant.py b/policyengine_core/parameters/vectorial_parameter_node_at_instant.py index 3dc3d25f..0b50d928 100644 --- a/policyengine_core/parameters/vectorial_parameter_node_at_instant.py +++ b/policyengine_core/parameters/vectorial_parameter_node_at_instant.py @@ -204,14 +204,14 @@ def __getitem__(self, key: str) -> Any: key = numpy.select( [key == item for item in enum], [str(item.name) for item in enum], - default="unknown" + default="unknown", ) elif isinstance(key, EnumArray): enum = key.possible_values key = numpy.select( [key == item.index for item in enum], [item.name for item in enum], - default="unknown" + default="unknown", ) else: key = key.astype("str") diff --git a/policyengine_core/tools/simulation_dumper.py b/policyengine_core/tools/simulation_dumper.py index 1a1d6529..4e2f8a18 100644 --- a/policyengine_core/tools/simulation_dumper.py +++ b/policyengine_core/tools/simulation_dumper.py @@ -96,7 +96,7 @@ def _dump_entity(population, directory): encoded_roles = np.select( [population.members_role == role for role in flattened_roles], [str(role.key) for role in flattened_roles], - default="unknown" + default="unknown", ) np.save(os.path.join(path, "members_role.npy"), encoded_roles) diff --git a/tests/core/parameters_fancy_indexing/test_fancy_indexing.py b/tests/core/parameters_fancy_indexing/test_fancy_indexing.py index 33904903..476a4f38 100644 --- a/tests/core/parameters_fancy_indexing/test_fancy_indexing.py +++ b/tests/core/parameters_fancy_indexing/test_fancy_indexing.py @@ -114,5 +114,10 @@ class TypesZone(Enum): z1 = "Zone 1" z2 = "Zone 2" - zone = np.asarray([z.name for z in [TypesZone.z1, TypesZone.z2, TypesZone.z2, TypesZone.z1]]) + zone = np.asarray( + [ + z.name + for z in [TypesZone.z1, TypesZone.z2, TypesZone.z2, TypesZone.z1] + ] + ) assert_near(P.single.owner[zone], [100, 200, 200, 100]) diff --git a/tests/core/test_tracers.py b/tests/core/test_tracers.py index c538346a..66410692 100644 --- a/tests/core/test_tracers.py +++ b/tests/core/test_tracers.py @@ -403,7 +403,8 @@ def test_log_aggregate(tracer): lines = tracer.computation_log.lines(aggregate=True) assert ( - lines[0].strip() == "A<2017, (default)> = {'avg': np.float64(1.0), 'max': np.int64(1), 'min': np.int64(1)}" + lines[0].strip() + == "A<2017, (default)> = {'avg': np.float64(1.0), 'max': np.int64(1), 'min': np.int64(1)}" ) From 107b12061e920aacdcfd8481b573ac7aea064a4b Mon Sep 17 00:00:00 2001 From: noman404 Date: Thu, 30 Jan 2025 02:48:21 +0800 Subject: [PATCH 11/29] added change log --- changelog_entry.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29b..ab83c9f9 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,6 @@ +- bump: major + changes: + changed: + - python 3.13.0 + - numpy 2.1.0 + date: 2025-01-30 02:47:17 \ No newline at end of file From 9350086544b8c8222f36d49e39ced3c2f6d4e2a8 Mon Sep 17 00:00:00 2001 From: noman404 Date: Fri, 31 Jan 2025 07:40:50 +0800 Subject: [PATCH 12/29] updated change log, fix lint issue --- changelog_entry.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index ab83c9f9..7ed71d79 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -2,5 +2,4 @@ changes: changed: - python 3.13.0 - - numpy 2.1.0 - date: 2025-01-30 02:47:17 \ No newline at end of file + - numpy 2.1.0 \ No newline at end of file From 929def7d5fe1a1868de15084d846cba0ae44d990 Mon Sep 17 00:00:00 2001 From: noman404 Date: Fri, 31 Jan 2025 09:10:30 +0800 Subject: [PATCH 13/29] added new tests --- tests/core/commons/test_formulas.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/core/commons/test_formulas.py b/tests/core/commons/test_formulas.py index db22fc47..ecd3e182 100644 --- a/tests/core/commons/test_formulas.py +++ b/tests/core/commons/test_formulas.py @@ -79,3 +79,13 @@ def test_switch_when_values_are_empty(): with pytest.raises(AssertionError): assert commons.switch(conditions, value_by_condition) + + +def test_concat_tuple_inputs(): + with pytest.raises(TypeError, match="First argument must not be a tuple."): + commons.concat(("a", "b"), numpy.array(["c", "d"])) + + with pytest.raises( + TypeError, match="Second argument must not be a tuple." + ): + commons.concat(numpy.array(["a", "b"]), ("c", "d")) From f8103782197a79ca95ee5f23588089c5fd8acc41 Mon Sep 17 00:00:00 2001 From: noman404 Date: Fri, 31 Jan 2025 09:15:17 +0800 Subject: [PATCH 14/29] added GH workflow test for python 3.13 --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b8cef923..94c57b15 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -39,10 +39,10 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v3 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} - name: Install package run: make install - name: Run tests From bf721bb262966ad44876d93f5b75e5672e7b888d Mon Sep 17 00:00:00 2001 From: noman404 Date: Fri, 7 Feb 2025 10:06:28 +0800 Subject: [PATCH 15/29] point to updated PE-US, added windows in GH workflow --- .github/workflows/pr.yaml | 1 + Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 94c57b15..3d21f0b1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,6 +34,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.12", "3.13" ] fail-fast: false runs-on: ${{ matrix.os }} steps: diff --git a/Makefile b/Makefile index 33da57ef..957e2adb 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ format: install: pip install -e ".[dev]" --config-settings editable_mode=compat - pip install policyengine-us + pip install git+https://github.com/noman404/policyengine-us.git@noman404/python3.13 pip install policyengine-uk test-country-template: From 3df7162514ab523e3f9cf3a91bebe5ada207cbdb Mon Sep 17 00:00:00 2001 From: noman404 Date: Fri, 7 Feb 2025 10:24:22 +0800 Subject: [PATCH 16/29] added long path ignore for windows --- .github/workflows/pr.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3d21f0b1..94d08903 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -40,6 +40,12 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v3 + - name: Enable long paths in Git (Windows Only) + if: runner.os == 'Windows' + run: git config --system core.longpaths true + - name: Enable Win32 long paths via registry (Windows Only) + if: runner.os == 'Windows' + run: reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem" /v LongPathsEnabled /t REG_DWORD /d 1 /f - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From c46184cd7824860039d8c043222359d20fb42516 Mon Sep 17 00:00:00 2001 From: noman404 Date: Fri, 7 Feb 2025 22:17:48 +0800 Subject: [PATCH 17/29] added standard-imghdr --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5dbdbea6..5f564066 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ "pyvis>=0.3.2", "microdf_python>=0.4.3", "huggingface_hub>=0.25.1", + "standard-imghdr" ] dev_requirements = [ From 646125c67c49036888a5a0cc35bb57be83aad5c4 Mon Sep 17 00:00:00 2001 From: noman404 Date: Fri, 7 Feb 2025 22:19:35 +0800 Subject: [PATCH 18/29] imghdr fix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f564066..19f5773d 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ "pyvis>=0.3.2", "microdf_python>=0.4.3", "huggingface_hub>=0.25.1", - "standard-imghdr" + "standard-imghdr", ] dev_requirements = [ From 719c61163fd756641ce3b4f6d2f1668cfec5c343 Mon Sep 17 00:00:00 2001 From: noman404 Date: Fri, 21 Feb 2025 10:38:08 +0800 Subject: [PATCH 19/29] argsort fix --- policyengine_core/populations/population.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/policyengine_core/populations/population.py b/policyengine_core/populations/population.py index a04b8de6..92198b16 100644 --- a/policyengine_core/populations/population.py +++ b/policyengine_core/populations/population.py @@ -253,7 +253,11 @@ def get_rank( # We double-argsort all lines of the matrix. # Double-argsorting gets the rank of each value once sorted # For instance, if x = [3,1,6,4,0], y = numpy.argsort(x) is [4, 1, 0, 3, 2] (because the value with index 4 is the smallest one, the value with index 1 the second smallest, etc.) and z = numpy.argsort(y) is [2, 1, 4, 3, 0], the rank of each value. - sorted_matrix = numpy.argsort(numpy.argsort(matrix)) + + first_argsort = numpy.argsort(matrix, axis=1, kind="stable") + # because of the infinities the first sort creates positional indices + # The second argsort converts these positions to ranks + sorted_matrix = numpy.argsort(first_argsort, axis=1, kind="stable") # Build the result vector by taking for each person the value in the right line (corresponding to its household id) and the right column (corresponding to its position) result = sorted_matrix[ids, positions] From b39d99229b8ac0834b05d6f69dac40557413d4a2 Mon Sep 17 00:00:00 2001 From: noman404 Date: Sat, 22 Feb 2025 08:08:43 +0800 Subject: [PATCH 20/29] python build error fix for workflow --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 94d08903..11b0f47d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,7 +34,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, windows-latest ] - python-version: [ "3.12", "3.13" ] + python-version: [ "3.12", "3.13.2" ] fail-fast: false runs-on: ${{ matrix.os }} steps: From c141b10388b7c86929f05a4376a3de7dee8aa145 Mon Sep 17 00:00:00 2001 From: noman404 Date: Tue, 25 Feb 2025 01:00:25 +0800 Subject: [PATCH 21/29] added code comment for sorting --- .github/workflows/pr.yaml | 2 +- policyengine_core/populations/population.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 11b0f47d..94d08903 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,7 +34,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest, windows-latest ] - python-version: [ "3.12", "3.13.2" ] + python-version: [ "3.12", "3.13" ] fail-fast: false runs-on: ${{ matrix.os }} steps: diff --git a/policyengine_core/populations/population.py b/policyengine_core/populations/population.py index 92198b16..a6d992cb 100644 --- a/policyengine_core/populations/population.py +++ b/policyengine_core/populations/population.py @@ -254,9 +254,9 @@ def get_rank( # Double-argsorting gets the rank of each value once sorted # For instance, if x = [3,1,6,4,0], y = numpy.argsort(x) is [4, 1, 0, 3, 2] (because the value with index 4 is the smallest one, the value with index 1 the second smallest, etc.) and z = numpy.argsort(y) is [2, 1, 4, 3, 0], the rank of each value. - first_argsort = numpy.argsort(matrix, axis=1, kind="stable") # because of the infinities the first sort creates positional indices - # The second argsort converts these positions to ranks + # The second argsort converts these positions to ranks, thus fixes the broken sort issue + first_argsort = numpy.argsort(matrix, axis=1, kind="stable") sorted_matrix = numpy.argsort(first_argsort, axis=1, kind="stable") # Build the result vector by taking for each person the value in the right line (corresponding to its household id) and the right column (corresponding to its position) From ca1910fd69bd9f10a794a916276d2f6c58c32daf Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Mon, 5 May 2025 22:59:00 -0700 Subject: [PATCH 22/29] Removed country package installs --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 957e2adb..adefd27a 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,6 @@ format: install: pip install -e ".[dev]" --config-settings editable_mode=compat - pip install git+https://github.com/noman404/policyengine-us.git@noman404/python3.13 - pip install policyengine-uk test-country-template: policyengine-core test policyengine_core/country_template/tests -c policyengine_core.country_template From 4c62dce0a86d6825237a7ab0d4bee0f3b67c9db6 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Mon, 5 May 2025 23:04:47 -0700 Subject: [PATCH 23/29] Moved smoke test for country package to PR workflow --- .github/workflows/pr.yaml | 24 ++++++++++++++++++++++++ tests/{ => smoke}/test_us.py | 9 +++++++++ 2 files changed, 33 insertions(+) rename tests/{ => smoke}/test_us.py (55%) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 94d08903..9be3c6fe 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -59,3 +59,27 @@ jobs: run: make build - name: Test documentation builds run: make documentation + SmokeTestForMultipleVersions: + name: Test Core and Country Compatibility on (${{ matrix.os }}, py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + needs: Lint + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ['3.10', '3.11', '3.12'] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install -core package ONLY (no dev deps) + run: python -m pip install . + - name: Install -us package from PyPI + run: python -m pip install policyengine-us + - name: Run smoke tests only + run: pytest -m smoke + env: + RUN_SMOKE_TESTS: "1" \ No newline at end of file diff --git a/tests/test_us.py b/tests/smoke/test_us.py similarity index 55% rename from tests/test_us.py rename to tests/smoke/test_us.py index 8ad7f372..65d32868 100644 --- a/tests/test_us.py +++ b/tests/smoke/test_us.py @@ -1,3 +1,12 @@ +import os +import pytest + + +@pytest.mark.smoke +@pytest.mark.skipif( + os.getenv("RUN_SMOKE_TESTS") != "1", + reason="Skip smoke tests unless explicitly enabled", +) def test_policyengine_us_microsimulation_runs(): from policyengine_us import Microsimulation From ed108018329b50b20d53dac4177beb46e391f3e2 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Mon, 5 May 2025 23:09:57 -0700 Subject: [PATCH 24/29] Added changelog --- changelog_entry.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 7ed71d79..89ad2064 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,5 +1,5 @@ -- bump: major +- bump: minor changes: changed: - - python 3.13.0 - - numpy 2.1.0 \ No newline at end of file + - added support for python 3.13.0 + - upgraded dependency to numpy 2.1.0 \ No newline at end of file From dc4abe618a3c7042ef85d25dd2ccf87db9487765 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Tue, 6 May 2025 13:12:26 -0700 Subject: [PATCH 25/29] added pytest config for custom marker --- pytest.ini | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..8da4f3d6 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ + +[pytest] +markers = + smoke: mark a test as part of the smoke suite + From bcf0f768e5d0ecd23dcc02d7d50e511fa0c2b443 Mon Sep 17 00:00:00 2001 From: SakshiKekre Date: Tue, 6 May 2025 13:18:30 -0700 Subject: [PATCH 26/29] PR workflow modified to verify if setuptools fixes distutils module unavailability issue --- .github/workflows/pr.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9be3c6fe..fca1146d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -75,8 +75,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install -core package ONLY (no dev deps) - run: python -m pip install . + - name: Install -core package with dev deps + # run: python -m pip install . + # Installing with dev deps to see if setuptools provides distutils (removed in Python 3.12) + run: python -m pip install ".[dev]" - name: Install -us package from PyPI run: python -m pip install policyengine-us - name: Run smoke tests only From 1379b521b9e0107b9637ebac71a49c5e3a276e11 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 17:18:43 -0400 Subject: [PATCH 27/29] Add Python 3.13 to smoke test matrix and merge latest main --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fca1146d..86c30c31 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -67,7 +67,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - name: Checkout repo uses: actions/checkout@v4 From a0d73d8737f23489ab5847e57304856a8d32390c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 17:28:27 -0400 Subject: [PATCH 28/29] Skip policyengine-us smoke tests for Python 3.13 This resolves the circular dependency issue where: - policyengine-core needs to support Python 3.13 first - policyengine-us can't be updated until policyengine-core supports 3.13 - But CI fails because policyengine-us doesn't support 3.13 yet The smoke tests for Python 3.13 will be re-enabled once policyengine-us is updated. --- .github/workflows/pr.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 86c30c31..99ae35b0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -80,8 +80,12 @@ jobs: # Installing with dev deps to see if setuptools provides distutils (removed in Python 3.12) run: python -m pip install ".[dev]" - name: Install -us package from PyPI + # Skip policyengine-us installation for Python 3.13 until it's updated + if: matrix.python-version != '3.13' run: python -m pip install policyengine-us - name: Run smoke tests only + # Only run smoke tests if policyengine-us was installed + if: matrix.python-version != '3.13' run: pytest -m smoke env: RUN_SMOKE_TESTS: "1" \ No newline at end of file From db04b45630503c461fc45524169f2d2f01e40c4a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Jul 2025 17:39:39 -0400 Subject: [PATCH 29/29] Complete Python 3.13 upgrade with additional improvements - Update push.yaml workflow to use Python 3.13 for builds and deployments - Standardize all GitHub Actions to latest versions (v4/v5) - Remove obsolete Python 2 compatibility check in parameter_node_at_instant.py - Update README.md to explicitly list supported Python versions (3.10-3.13) - Ensure consistency across all workflows --- .github/workflows/pr.yaml | 10 ++++----- .github/workflows/push.yaml | 22 +++++++++---------- README.md | 2 +- .../parameters/parameter_node_at_instant.py | 2 -- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 99ae35b0..316052ff 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -6,7 +6,7 @@ jobs: Lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check formatting uses: "lgeiger/black-action@master" with: @@ -15,13 +15,13 @@ jobs: name: Check version runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.ref }} - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.12" - name: Build changelog @@ -47,14 +47,14 @@ jobs: if: runner.os == 'Windows' run: reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem" /v LongPathsEnabled /t REG_DWORD /d 1 /f - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install package run: make install - name: Run tests run: make test - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 - name: Build package run: make build - name: Test documentation builds diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 9cf4a8cf..8239a1d4 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -9,7 +9,7 @@ jobs: (github.repository == 'PolicyEngine/policyengine-core') && (github.event.head_commit.message == 'Update PolicyEngine Core') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check formatting uses: "lgeiger/black-action@master" with: @@ -22,15 +22,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.ref }} token: ${{ secrets.POLICYENGINE_GITHUB }} - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - name: Build changelog run: pip install yaml-changelog && make changelog - name: Preview changelog update @@ -49,16 +49,16 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine Core') steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - name: Install package run: make install - name: Run tests run: make test - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 - name: Generate documentation run: make documentation - name: Deploy documentation @@ -75,11 +75,11 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine Core') steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - name: Publish a git tag run: ".github/publish-git-tag.sh || true" - name: Install package diff --git a/README.md b/README.md index c67bd756..bd8fcafd 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This package, a fork of [OpenFisca-Core](https://github.com/OpenFisca/OpenFisca- # Prerequisites -Python 3.10 or beyond is required. +Python 3.10, 3.11, 3.12, or 3.13 is required. # Setting Up diff --git a/policyengine_core/parameters/parameter_node_at_instant.py b/policyengine_core/parameters/parameter_node_at_instant.py index 64e3d356..67f4695e 100644 --- a/policyengine_core/parameters/parameter_node_at_instant.py +++ b/policyengine_core/parameters/parameter_node_at_instant.py @@ -72,6 +72,4 @@ def __repr__(self) -> str: for name, value in self._children.items() ] ) - if sys.version_info < (3, 0): - return result return result