From f7aa13d9ba5047f327a7e44c70b138fbeb5777ea Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Fri, 24 Oct 2025 19:42:56 +0300 Subject: [PATCH 01/10] Bug fix --- Lib/test/test_dict.py | 15 +++++++++++++++ Objects/dictobject.c | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 60c62430370e96..258255a015f5f8 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1601,6 +1601,21 @@ def __hash__(self): with self.assertRaises(KeyError): d.get(key2) + def test_clear_at_lookup(self): + d = {} + + class X(object): + def __hash__(self): + return 1 + def __eq__(self, other): + nonlocal d + d.clear() + + for _ in range(10): + d[X()] = None + + self.assertEqual(len(d), 1) + class CAPITest(unittest.TestCase): diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 73d4db4cac7963..e245d48e8cbca4 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1899,6 +1899,14 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (ix == DKIX_ERROR) goto Fail; + // gh-140551: If dict was cleaned in _Py_dict_lookup, + // we have to resize one more time to force general key kind. + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { + if (insertion_resize(mp, 0) < 0) + goto Fail; + assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + } + if (ix == DKIX_EMPTY) { assert(!_PyDict_HasSplitTable(mp)); /* Insert into new slot. */ From f1b6853a520cf27066b1efac76364c4034d94a5c Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Fri, 24 Oct 2025 20:42:41 +0300 Subject: [PATCH 02/10] Add NEWS --- .../2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst new file mode 100644 index 00000000000000..18e221117833eb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst @@ -0,0 +1,2 @@ +Fixed crash in ``dict`` if ``clear`` is called at the lookup stage. Patch by +Mikhail Efimov. From 3b5eacee43a22c1cd8cdc5b84177ee0632182816 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Fri, 24 Oct 2025 21:06:23 +0300 Subject: [PATCH 03/10] Update Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst Co-authored-by: Sergey Miryanov --- .../2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst index 18e221117833eb..599097dffbc809 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst @@ -1,2 +1,2 @@ -Fixed crash in ``dict`` if ``clear`` is called at the lookup stage. Patch by +Fixed crash in :class:`dict` if :meth:`clear` is called at the lookup stage. Patch by Mikhail Efimov. From 726f52bcb4db59dee683dcfb1855367b0b75c282 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Fri, 24 Oct 2025 21:41:52 +0300 Subject: [PATCH 04/10] Fixes for NEWS. Add setdefault test and fix --- Lib/test/test_dict.py | 9 +++++++-- ...025-10-24-20-42-33.gh-issue-140551.-9swrl.rst | 4 ++-- Objects/dictobject.c | 16 ++++++++-------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 258255a015f5f8..31c647f747ad28 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1602,8 +1602,6 @@ def __hash__(self): d.get(key2) def test_clear_at_lookup(self): - d = {} - class X(object): def __hash__(self): return 1 @@ -1611,11 +1609,18 @@ def __eq__(self, other): nonlocal d d.clear() + d = {} for _ in range(10): d[X()] = None self.assertEqual(len(d), 1) + d = {} + for _ in range(10): + d.setdefault(X(), None) + + self.assertEqual(len(d), 1) + class CAPITest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst index 599097dffbc809..b47ea84e1db437 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst @@ -1,2 +1,2 @@ -Fixed crash in :class:`dict` if :meth:`clear` is called at the lookup stage. Patch by -Mikhail Efimov. +Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup +stage. Patch by Mikhail Efimov. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e245d48e8cbca4..8274d6005e9ce5 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1775,6 +1775,14 @@ static inline int insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, Py_hash_t hash, PyObject *key, PyObject *value) { + // gh-140551: If dict was cleaned in _Py_dict_lookup, + // we have to resize one more time to force general key kind. + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { + if (insertion_resize(mp, 0) < 0) + return -1; + assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + } + if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ if (insertion_resize(mp, 1) < 0) { @@ -1899,14 +1907,6 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (ix == DKIX_ERROR) goto Fail; - // gh-140551: If dict was cleaned in _Py_dict_lookup, - // we have to resize one more time to force general key kind. - if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(mp, 0) < 0) - goto Fail; - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); - } - if (ix == DKIX_EMPTY) { assert(!_PyDict_HasSplitTable(mp)); /* Insert into new slot. */ From 3538e1bb76182455bb018389dc61691120687331 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Sun, 26 Oct 2025 21:57:33 +0300 Subject: [PATCH 05/10] Update Lib/test/test_dict.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 31c647f747ad28..2e6c2bbdf19409 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1602,7 +1602,7 @@ def __hash__(self): d.get(key2) def test_clear_at_lookup(self): - class X(object): + class X: def __hash__(self): return 1 def __eq__(self, other): From 3d8bb6ddae47fd046b9782e1ce452597466b72e8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Oct 2025 18:41:03 +0900 Subject: [PATCH 06/10] do not convert split/unicode table before _Py_dict_lookup --- Objects/dictobject.c | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 8274d6005e9ce5..0324bd0e432cdb 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1882,13 +1882,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, ASSERT_DICT_LOCKED(mp); - if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(mp, 0) < 0) - goto Fail; - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); - } - - if (_PyDict_HasSplitTable(mp)) { + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { insert_split_value(interp, mp, key, value, ix); @@ -1908,9 +1902,10 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, goto Fail; if (ix == DKIX_EMPTY) { - assert(!_PyDict_HasSplitTable(mp)); - /* Insert into new slot. */ - assert(old_value == NULL); + // insert_combined_dict() will convert from non DICT_KEYS_GENERAL table + // into DICT_KEYS_GENERAL table if key is not Unicode. + // We don't convert it before _Py_dict_lookup because non-Unicode key + // may change generic table into Unicode/split table. if (insert_combined_dict(interp, mp, hash, key, value) < 0) { goto Fail; } @@ -4417,16 +4412,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu return 0; } - if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(mp, 0) < 0) { - if (result) { - *result = NULL; - } - return -1; - } - } - - if (_PyDict_HasSplitTable(mp)) { + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { PyObject *value = mp->ma_values->values[ix]; @@ -4447,8 +4433,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } } - assert(!_PyDict_HasSplitTable(mp)); - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); if (ix == DKIX_ERROR) { if (result) { @@ -4458,7 +4442,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } if (ix == DKIX_EMPTY) { - assert(!_PyDict_HasSplitTable(mp)); value = default_value; if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { From 1555aaf0024cc32ed41b9ff512b0d13fd5a9b83a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Oct 2025 19:03:31 +0900 Subject: [PATCH 07/10] skip unnecessary _Py_dict_lookup when split table is full. --- Objects/dictobject.c | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 0324bd0e432cdb..3bd9385fec9fd7 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1879,28 +1879,26 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { PyObject *old_value; + Py_ssize_t ix; ASSERT_DICT_LOCKED(mp); if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { - Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { insert_split_value(interp, mp, key, value, ix); Py_DECREF(key); Py_DECREF(value); return 0; } - - /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(mp, 1) < 0) { + // No space in shared keys. Go to insert_combined_dict() below. + } + else { + ix = _Py_dict_lookup(mp, key, hash, &old_value); + if (ix == DKIX_ERROR) goto Fail; - } } - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) - goto Fail; - if (ix == DKIX_EMPTY) { // insert_combined_dict() will convert from non DICT_KEYS_GENERAL table // into DICT_KEYS_GENERAL table if key is not Unicode. @@ -4377,6 +4375,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; + Py_ssize_t ix; PyInterpreterState *interp = _PyInterpreterState_GET(); ASSERT_DICT_LOCKED(d); @@ -4413,7 +4412,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { - Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { PyObject *value = mp->ma_values->values[ix]; int already_present = value != NULL; @@ -4426,19 +4425,16 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } return already_present; } - - /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(mp, 1) < 0) { - goto error; - } + // No space in shared keys. Go to insert_combined_dict() below. } - - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) { - if (result) { - *result = NULL; + else { + ix = _Py_dict_lookup(mp, key, hash, &value); + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; } - return -1; } if (ix == DKIX_EMPTY) { @@ -4468,12 +4464,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu *result = incref_result ? Py_NewRef(value) : value; } return 1; - -error: - if (result) { - *result = NULL; - } - return -1; } int From 080467d3202d16e33470eca0dbd7d888a0736316 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Mon, 27 Oct 2025 17:39:30 +0300 Subject: [PATCH 08/10] Review addressed (comment change) --- Objects/dictobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 3bd9385fec9fd7..1e7e88f3084fd4 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1903,7 +1903,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, // insert_combined_dict() will convert from non DICT_KEYS_GENERAL table // into DICT_KEYS_GENERAL table if key is not Unicode. // We don't convert it before _Py_dict_lookup because non-Unicode key - // may change generic table into Unicode/split table. + // may change generic table into Unicode table. if (insert_combined_dict(interp, mp, hash, key, value) < 0) { goto Fail; } @@ -4440,6 +4440,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu if (ix == DKIX_EMPTY) { value = default_value; + // See comment to this function in insertdict. if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { Py_DECREF(key); Py_DECREF(value); From 863b9dbef3f81fff04778757bc5d07d122a139c6 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Mon, 27 Oct 2025 18:20:44 +0300 Subject: [PATCH 09/10] Little NEWS update --- .../2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst index b47ea84e1db437..8fd9b46c0aeabe 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst @@ -1,2 +1,2 @@ Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup -stage. Patch by Mikhail Efimov. +stage. Patch by Mikhail Efimov and Inada Naoki. From 12c74d740e8537bbc2abd09c7fd9222359a552a9 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Tue, 28 Oct 2025 11:22:07 +0300 Subject: [PATCH 10/10] Typo --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 1e7e88f3084fd4..65eed151c2829d 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1775,7 +1775,7 @@ static inline int insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, Py_hash_t hash, PyObject *key, PyObject *value) { - // gh-140551: If dict was cleaned in _Py_dict_lookup, + // gh-140551: If dict was cleared in _Py_dict_lookup, // we have to resize one more time to force general key kind. if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { if (insertion_resize(mp, 0) < 0)