From 2ece07d4c45fd3e089d04bbcd6b8da0e5faf6703 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 29 Oct 2025 12:06:23 +0300 Subject: [PATCH 1/2] gh-140551: Fix `dict` crash if `clear` is called at `lookup` stage (#140558) Co-authored-by: Inada Naoki --- Lib/test/test_dict.py | 20 +++++ ...-10-24-20-42-33.gh-issue-140551.-9swrl.rst | 2 + Objects/dictobject.c | 86 ++++++++----------- 3 files changed, 56 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 4c095464cbbc54..5c3889550953dd 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1654,6 +1654,26 @@ def make_pairs(): self.assertEqual(d.get(key3_3), 44) self.assertGreaterEqual(eq_count, 1) + def test_clear_at_lookup(self): + class X: + def __hash__(self): + return 1 + def __eq__(self, other): + nonlocal d + d.clear() + + d = {} + for _ in range(10): + d[X()] = None + + self.assertEqual(len(d), 1) + + d = {} + for _ in range(10): + d.setdefault(X(), None) + + self.assertEqual(len(d), 1) + class CAPITest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst b/Misc/NEWS.d/next/Core and Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst new file mode 100644 index 00000000000000..8fd9b46c0aeabe --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst @@ -0,0 +1,2 @@ +Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup +stage. Patch by Mikhail Efimov and Inada Naoki. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 3ab080b3e73449..0b60bbd84d4de8 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1724,6 +1724,14 @@ static inline int insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, Py_hash_t hash, PyObject *key, PyObject *value) { + // gh-140551: If dict was cleared in _Py_dict_lookup, + // we have to resize one more time to force general key kind. + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { + if (insertion_resize(interp, mp, 0) < 0) + return -1; + assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + } + if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ if (insertion_resize(interp, mp, 1) < 0) { @@ -1825,40 +1833,33 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { PyObject *old_value; + Py_ssize_t ix; ASSERT_DICT_LOCKED(mp); - if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(interp, mp, 0) < 0) - goto Fail; - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); - } - - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { + ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { insert_split_value(interp, mp, key, value, ix); Py_DECREF(key); Py_DECREF(value); return 0; } - - /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { - goto Fail; - } + // No space in shared keys. Go to insert_combined_dict() below. } + else { + ix = _Py_dict_lookup(mp, key, hash, &old_value); + if (ix == DKIX_ERROR) + goto Fail; - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) - goto Fail; - - MAINTAIN_TRACKING(mp, key, value); + MAINTAIN_TRACKING(mp, key, value); + } if (ix == DKIX_EMPTY) { - assert(!_PyDict_HasSplitTable(mp)); - /* Insert into new slot. */ - assert(old_value == NULL); + // insert_combined_dict() will convert from non DICT_KEYS_GENERAL table + // into DICT_KEYS_GENERAL table if key is not Unicode. + // We don't convert it before _Py_dict_lookup because non-Unicode key + // may change generic table into Unicode table. if (insert_combined_dict(interp, mp, hash, key, value) < 0) { goto Fail; } @@ -4256,6 +4257,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; + Py_ssize_t ix; PyInterpreterState *interp = _PyInterpreterState_GET(); ASSERT_DICT_LOCKED(d); @@ -4290,17 +4292,8 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu return 0; } - if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(interp, mp, 0) < 0) { - if (result) { - *result = NULL; - } - return -1; - } - } - - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { + ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { PyObject *value = mp->ma_values->values[ix]; int already_present = value != NULL; @@ -4313,27 +4306,22 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } return already_present; } - - /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { - goto error; - } + // No space in shared keys. Go to insert_combined_dict() below. } - - assert(!_PyDict_HasSplitTable(mp)); - - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) { - if (result) { - *result = NULL; + else { + ix = _Py_dict_lookup(mp, key, hash, &value); + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; } - return -1; } if (ix == DKIX_EMPTY) { - assert(!_PyDict_HasSplitTable(mp)); value = default_value; + // See comment to this function in insertdict. if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { Py_DECREF(key); Py_DECREF(value); @@ -4359,12 +4347,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu *result = incref_result ? Py_NewRef(value) : value; } return 1; - -error: - if (result) { - *result = NULL; - } - return -1; } int From 15e538abd3ef8ae09fd728f2341119244646b6ea Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 29 Oct 2025 15:13:30 +0300 Subject: [PATCH 2/2] Move MAINTAIN_TRACKING --- Objects/dictobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 0b60bbd84d4de8..4a88e08d1da52e 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1837,6 +1837,8 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, ASSERT_DICT_LOCKED(mp); + MAINTAIN_TRACKING(mp, key, value); + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { @@ -1852,7 +1854,6 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (ix == DKIX_ERROR) goto Fail; - MAINTAIN_TRACKING(mp, key, value); } if (ix == DKIX_EMPTY) {