Skip to content

Commit efeeb6c

Browse files
eendebakptclaude
andauthored
gh-132042: Speed up slot lookup during class creation (#159479)
* gh-132042: Speed up slot lookup during class creation Resolve the special method slots of a new class with one pass over each class dict along the MRO instead of one lookup per slot name and per class. A small static table maps slot names to slotdef indices; it also replaces the temporary dict in _PyType_InitSlotDefs() and the linear scan in has_slotdef(). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Replace dead defensive checks in collect_slot_descrs() with asserts * Use the slot name table in update_slot() * Share the slot name table probe loop * review comments * gh-132042: Allow dict subclasses as tp_dict in the slot scan An extension may initialize tp_dict with a dict subclass before calling PyType_Ready(), so assert PyDict_Check() like find_name_in_mro() does. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * gh-132042: Test subclassing a type whose tp_dict is a dict subclass Add _testcapi.MyListDictSubclass, a static type like MyList whose tp_dict is initialized with a dict subclass before PyType_Ready(), as extensions may do, and check that subclasses created from Python still inherit its special methods. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * gh-132042: Ignore the MyListDictSubclass_Type static type in check-c-globals Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
1 parent 3735833 commit efeeb6c

6 files changed

Lines changed: 296 additions & 77 deletions

File tree

‎Lib/test/test_capi/test_type.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,19 @@ class MyList(list):
485485
self.assertFalse(type_issubtype(frozendict, dict))
486486
self.assertFalse(type_issubtype(MyList, tuple))
487487

488+
def test_subclass_type_with_dict_subclass_tp_dict(self):
489+
# gh-132042: MyListDictSubclass initializes tp_dict with a dict
490+
# subclass before PyType_Ready(), as extensions may do. Creating a
491+
# subclass must still find the special methods in that dict.
492+
class Sub(_testcapi.MyListDictSubclass):
493+
pass
494+
self.assertEqual(len(Sub([1, 2, 3])), 3)
495+
496+
class Sub2(_testcapi.MyListDictSubclass):
497+
def __len__(self):
498+
return 5
499+
self.assertEqual(len(Sub2([1, 2, 3])), 5)
500+
488501
def test_type_modified(self):
489502
# Test PyType_Modified()
490503
type_modified = _testlimitedcapi.type_modified

‎Lib/test/test_free_threading/test_type.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,74 @@ class B(A):
401401
with threading_helper.start_threads(threads):
402402
pass
403403

404+
def test_class_creation_with_concurrent_setattr(self):
405+
# Creating a class resolves its slots from the dicts along the MRO
406+
# while holding the type lock, before the class is reachable from
407+
# its bases. If that resolution blocked on a base's dict mutex, the
408+
# type lock would be released while blocked, and another thread
409+
# could then assign a special method on a base that was already
410+
# resolved. The new class would end up with a slot that does not
411+
# match its MRO.
412+
N = 300
413+
414+
class B2:
415+
pass
416+
417+
class B1(B2):
418+
pass
419+
420+
lock = threading.Lock()
421+
mismatches = []
422+
done = False
423+
424+
def custom_repr(self):
425+
return "custom repr"
426+
427+
def setter():
428+
# Toggle B1.__repr__, so a stale resolution of B1 gives the new
429+
# class object.__repr__ while B1.__repr__ is custom_repr.
430+
barrier.wait()
431+
while not done:
432+
with lock:
433+
B1.__repr__ = custom_repr
434+
with lock:
435+
del B1.__repr__
436+
437+
def churner():
438+
# type.__setattr__ takes B2's dict mutex before the type lock
439+
# and holds it while waiting for the type lock, so B2's dict
440+
# mutex is contended whenever creator holds the type lock.
441+
barrier.wait()
442+
n = 0
443+
while not done:
444+
n += 1
445+
B2.attr = n
446+
447+
def creator():
448+
nonlocal done
449+
# Mark the base dicts as shared, so lookups from this thread
450+
# never need to take their mutexes.
451+
getattr(B1, "nope", None)
452+
barrier.wait()
453+
try:
454+
for _ in range(N):
455+
C = type("C", (B1,), {})
456+
c = C()
457+
with lock:
458+
slot_result = repr(c)
459+
mro_result = C.__repr__(c)
460+
if slot_result != mro_result:
461+
mismatches.append((slot_result, mro_result))
462+
finally:
463+
done = True
464+
465+
targets = (setter, churner, churner, creator)
466+
barrier = threading.Barrier(len(targets))
467+
threads = [Thread(target=target) for target in targets]
468+
with threading_helper.start_threads(threads):
469+
pass
470+
self.assertEqual(mismatches, [])
471+
404472

405473
if __name__ == "__main__":
406474
unittest.main()
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Speed up class creation by resolving the special method slots of a new class
2+
with one pass over each class dict along the MRO instead of one lookup per
3+
slot name and per class.

‎Modules/_testcapimodule.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3307,6 +3307,17 @@ static PyTypeObject MyList_Type = {
33073307
MyList_new, /* tp_new */
33083308
};
33093309

3310+
/* Like MyList, but tp_dict is initialized with a dict subclass before
3311+
PyType_Ready(), as extension modules may do (e.g. ctypes before 3.13). */
3312+
static PyTypeObject MyListDictSubclass_Type = {
3313+
PyVarObject_HEAD_INIT(NULL, 0)
3314+
"MyListDictSubclass",
3315+
sizeof(MyListObject),
3316+
.tp_dealloc = MyList_dealloc,
3317+
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
3318+
.tp_new = MyList_new,
3319+
};
3320+
33103321
/* Test PEP 560 */
33113322

33123323
typedef struct {
@@ -3623,6 +3634,28 @@ _testcapi_exec(PyObject *m)
36233634
Py_INCREF(&MyList_Type);
36243635
PyModule_AddObject(m, "MyList", (PyObject *)&MyList_Type);
36253636

3637+
MyListDictSubclass_Type.tp_base = &PyList_Type;
3638+
if (MyListDictSubclass_Type.tp_dict == NULL) {
3639+
PyObject *cls = PyObject_CallFunction((PyObject *)&PyType_Type,
3640+
"s(O){}", "DictSubclass",
3641+
&PyDict_Type);
3642+
if (cls == NULL) {
3643+
return -1;
3644+
}
3645+
MyListDictSubclass_Type.tp_dict = PyObject_CallNoArgs(cls);
3646+
Py_DECREF(cls);
3647+
if (MyListDictSubclass_Type.tp_dict == NULL) {
3648+
return -1;
3649+
}
3650+
}
3651+
/* PyModule_AddType() would skip PyType_Ready() as tp_dict is set. */
3652+
if (PyType_Ready(&MyListDictSubclass_Type) < 0) {
3653+
return -1;
3654+
}
3655+
if (PyModule_AddType(m, &MyListDictSubclass_Type) < 0) {
3656+
return -1;
3657+
}
3658+
36263659
if (PyType_Ready(&GenericAlias_Type) < 0)
36273660
return -1;
36283661
Py_INCREF(&GenericAlias_Type);

0 commit comments

Comments
 (0)