gh-161184: Use a managed dict and weakref list for the _io types

Lib/test/test_io/test_general.py

@@ -895,6 +895,41 @@ def test_immutable_types(self):
                 with self.assertRaisesRegex(TypeError, "immutable"):
                     tp.foo = "bar"
 
+    def test_managed_dict(self):
+        # The _io types use Py_TPFLAGS_MANAGED_DICT: there is no explicit
+        # dict slot and instance attributes are stored as inline values.
+        _testinternalcapi = import_helper.import_module("_testinternalcapi")
+        _io = self._io
+        no_dict = {_io.IncrementalNewlineDecoder, _io._BytesIOBuffer}
+        for tp in self.types:
+            with self.subTest(tp=tp):
+                if tp in no_dict:
+                    self.assertEqual(tp.__dictoffset__, 0)
+                else:
+                    self.assertEqual(tp.__dictoffset__, -1)
+
+        # Use fresh subclasses: each class has its own shared keys, which
+        # may already be exhausted on the base types by earlier tests.
+        class BytesIO(_io.BytesIO): pass
+        class StringIO(_io.StringIO): pass
+        class BufferedReader(_io.BufferedReader): pass
+        class TextIOWrapper(_io.TextIOWrapper): pass
+        class RawIO(_io._RawIOBase):
+            def __init__(self):
+                self.attr = 1
+        instances = [
+            BytesIO(b"data"),
+            StringIO("data"),
+            BufferedReader(_io.BytesIO(b"data")),
+            TextIOWrapper(_io.BytesIO()),
+            RawIO(),
+        ]
+        for obj in instances:
+            with self.subTest(obj=type(obj).__name__):
+                obj.attr = 1
+                self.assertTrue(_testinternalcapi.has_inline_values(obj))
+                self.assertEqual(obj.__dict__["attr"], 1)
+
     def test_class_hierarchy(self):
         def check_subs(types, base):
             for tp in types:

Lib/test/test_io/test_memoryio.py

@@ -10,6 +10,7 @@
 import io
 import _pyio as pyio
 import pickle
+import struct
 import sys
 import weakref
 
@@ -1037,11 +1038,13 @@ def test_setstate(self):
     @support.cpython_only
     def test_sizeof(self):
         if support.Py_GIL_DISABLED:
-            basesize = support.calcobjsize('P2n2Pni')
+            basesize = support.calcobjsize('P2nni')
         else:
-            basesize = support.calcobjsize('P2n2Pn')
+            basesize = support.calcobjsize('P2nn')
         check = self.check_sizeof
         self.assertEqual(object.__sizeof__(io.BytesIO()), basesize)
+        # sys.getsizeof() includes the managed dict and weakref pre-header
+        basesize += 2 * struct.calcsize('P')
         check(io.BytesIO(), basesize )
         n = 1000  # use a variable to prevent constant folding
         check(io.BytesIO(b'a' * n), basesize + sys.getsizeof(b'a' * n))

Misc/NEWS.d/next/Library/2026-11-03-08-21-06.gh-issue-161184.PIhcD9.rst

@@ -0,0 +1,6 @@
+Speed up attribute access on instances of :mod:`io` classes and their
+subclasses by storing instance attributes and weak references in the object
+pre-header. The ``__dictoffset__`` and ``__weakrefoffset__`` attributes
+of these classes now report the managed offsets, and C extension types that
+subclass an :mod:`io` class can no longer declare their own ``__dictoffset__``
+or ``__weaklistoffset__`` member.

Modules/_io/bufferedio.c

@@ -9,11 +9,11 @@
 
 #include "Python.h"
 #include "pycore_call.h"                // _PyObject_CallNoArgs()
+#include "pycore_critical_section.h"    // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED()
 #include "pycore_fileutils.h"           // _PyFile_Flush
 #include "pycore_object.h"              // _PyObject_GC_UNTRACK()
 #include "pycore_pyerrors.h"            // _Py_FatalErrorFormat()
 #include "pycore_pylifecycle.h"         // _Py_IsInterpreterFinalizing()
-#include "pycore_weakref.h"             // FT_CLEAR_WEAKREFS()
 
 #include "_iomodule.h"
 
@@ -260,8 +260,6 @@ typedef struct {
     Py_ssize_t buffer_size;
     Py_ssize_t buffer_mask;
 
-    PyObject *dict;
-    PyObject *weakreflist;
 } buffered;
 
 #define buffered_CAST(op)   ((buffered *)(op))
@@ -417,7 +415,7 @@ buffered_clear(PyObject *op)
     buffered *self = buffered_CAST(op);
     self->ok = 0;
     Py_CLEAR(self->raw);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     return 0;
 }
 
@@ -431,7 +429,7 @@ buffered_dealloc(PyObject *op)
         return;
     _PyObject_GC_UNTRACK(self);
     self->ok = 0;
-    FT_CLEAR_WEAKREFS(op, self->weakreflist);
+    PyObject_ClearWeakRefs(op);
     if (self->buffer) {
         PyMem_Free(self->buffer);
         self->buffer = NULL;
@@ -467,8 +465,7 @@ buffered_traverse(PyObject *op, visitproc visit, void *arg)
     buffered *self = buffered_CAST(op);
     Py_VISIT(Py_TYPE(self));
     Py_VISIT(self->raw);
-    Py_VISIT(self->dict);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 /* Because this can call arbitrary code, it shouldn't be called when
@@ -2250,8 +2247,6 @@ typedef struct {
     PyObject_HEAD
     buffered *reader;
     buffered *writer;
-    PyObject *dict;
-    PyObject *weakreflist;
 } rwpair;
 
 #define rwpair_CAST(op) ((rwpair *)(op))
@@ -2309,10 +2304,9 @@ bufferedrwpair_traverse(PyObject *op, visitproc visit, void *arg)
 {
     rwpair *self = rwpair_CAST(op);
     Py_VISIT(Py_TYPE(self));
-    Py_VISIT(self->dict);
     Py_VISIT(self->reader);
     Py_VISIT(self->writer);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 static int
@@ -2321,7 +2315,7 @@ bufferedrwpair_clear(PyObject *op)
     rwpair *self = rwpair_CAST(op);
     Py_CLEAR(self->reader);
     Py_CLEAR(self->writer);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     return 0;
 }
 
@@ -2331,7 +2325,7 @@ bufferedrwpair_dealloc(PyObject *op)
     rwpair *self = rwpair_CAST(op);
     PyTypeObject *tp = Py_TYPE(self);
     _PyObject_GC_UNTRACK(self);
-    FT_CLEAR_WEAKREFS(op, self->weakreflist);
+    PyObject_ClearWeakRefs(op);
     (void)bufferedrwpair_clear(op);
     tp->tp_free(self);
     Py_DECREF(tp);
@@ -2638,8 +2632,6 @@ static PyMethodDef bufferedreader_methods[] = {
 static PyMemberDef bufferedreader_members[] = {
     {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY},
     {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0},
-    {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY},
     {NULL}
 };
 
@@ -2697,8 +2689,6 @@ static PyMethodDef bufferedwriter_methods[] = {
 static PyMemberDef bufferedwriter_members[] = {
     {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY},
     {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0},
-    {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY},
     {NULL}
 };
 
@@ -2751,8 +2741,6 @@ static PyMethodDef bufferedrwpair_methods[] = {
 };
 
 static PyMemberDef bufferedrwpair_members[] = {
-    {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rwpair, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(rwpair, dict), Py_READONLY},
     {NULL}
 };
 
@@ -2814,8 +2802,6 @@ static PyMethodDef bufferedrandom_methods[] = {
 static PyMemberDef bufferedrandom_members[] = {
     {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY},
     {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0},
-    {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY},
     {NULL}
 };
 

Modules/_io/bytesio.c

@@ -5,7 +5,6 @@
 #include "pycore_object.h"
 #include "pycore_pyatomic_ft_wrappers.h"
 #include "pycore_sysmodule.h"         // _PySys_GetSizeOf()
-#include "pycore_weakref.h"           // FT_CLEAR_WEAKREFS()
 
 #include <stddef.h>                   // offsetof()
 #include "_iomodule.h"
@@ -21,8 +20,6 @@ typedef struct {
     PyObject *buf;
     Py_ssize_t pos;
     Py_ssize_t string_size;
-    PyObject *dict;
-    PyObject *weakreflist;
     Py_ssize_t exports;
 #ifdef Py_GIL_DISABLED
     int buf_shared;
@@ -974,11 +971,16 @@ _io_BytesIO___getstate___impl(bytesio *self)
 
     if (initvalue == NULL)
         return NULL;
-    if (self->dict == NULL) {
-        dict = Py_NewRef(Py_None);
+    dict = PyObject_GenericGetDict((PyObject *)self, NULL);
+    if (dict == NULL) {
+        Py_DECREF(initvalue);
+        return NULL;
+    }
+    if (PyDict_GET_SIZE(dict) == 0) {
+        Py_SETREF(dict, Py_NewRef(Py_None));
     }
     else {
-        dict = PyDict_Copy(self->dict);
+        Py_SETREF(dict, PyDict_Copy(dict));
         if (dict == NULL) {
             Py_DECREF(initvalue);
             return NULL;
@@ -1061,16 +1063,16 @@ _io_BytesIO___setstate___impl(bytesio *self, PyObject *state)
                          Py_TYPE(dict)->tp_name);
             return NULL;
         }
-        if (self->dict) {
-            /* Alternatively, we could replace the internal dictionary
-               completely. However, it seems more practical to just update it. */
-            if (PyDict_Update(self->dict, dict) < 0)
-                return NULL;
+        /* Alternatively, we could replace the internal dictionary
+           completely. However, it seems more practical to just update it. */
+        PyObject *self_dict = PyObject_GenericGetDict((PyObject *)self, NULL);
+        if (self_dict == NULL) {
+            return NULL;
         }
-        else {
-            /* The LOAD_ATTR specializations read the dict slot lock-free
-               with an acquire load, so pair it with a release store. */
-            FT_ATOMIC_STORE_PTR_RELEASE(self->dict, Py_NewRef(dict));
+        int rc = PyDict_Update(self_dict, dict);
+        Py_DECREF(self_dict);
+        if (rc < 0) {
+            return NULL;
         }
     }
 
@@ -1089,8 +1091,8 @@ bytesio_dealloc(PyObject *op)
         PyErr_Print();
     }
     Py_CLEAR(self->buf);
-    Py_CLEAR(self->dict);
-    FT_CLEAR_WEAKREFS(op, self->weakreflist);
+    PyObject_ClearManagedDict(op);
+    PyObject_ClearWeakRefs(op);
     tp->tp_free(self);
     Py_DECREF(tp);
 }
@@ -1186,16 +1188,15 @@ bytesio_traverse(PyObject *op, visitproc visit, void *arg)
 {
     bytesio *self = bytesio_CAST(op);
     Py_VISIT(Py_TYPE(self));
-    Py_VISIT(self->dict);
     Py_VISIT(self->buf);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 static int
 bytesio_clear(PyObject *op)
 {
     bytesio *self = bytesio_CAST(op);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) {
         Py_CLEAR(self->buf);
     }
@@ -1239,8 +1240,6 @@ static struct PyMethodDef bytesio_methods[] = {
 };
 
 static PyMemberDef bytesio_members[] = {
-    {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(bytesio, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(bytesio, dict), Py_READONLY},
     {NULL}
 };
 

Modules/_io/fileio.c

@@ -4,7 +4,6 @@
 #include "pycore_fileutils.h"     // _Py_BEGIN_SUPPRESS_IPH
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
 #include "pycore_pyerrors.h"      // _PyErr_ChainExceptions1()
-#include "pycore_weakref.h"       // FT_CLEAR_WEAKREFS()
 
 #include <stdbool.h>              // bool
 #ifdef HAVE_UNISTD_H
@@ -80,8 +79,6 @@ typedef struct {
        modified outside of the fileio object / Python (ex. gh-90102, GH-121941,
        gh-109523). */
     struct _Py_stat_struct *stat_atopen;
-    PyObject *weakreflist;
-    PyObject *dict;
 } fileio;
 
 #define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type))
@@ -220,7 +217,6 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     self->truncate = 0;
     self->stat_atopen = NULL;
     self->closefd = 1;
-    self->weakreflist = NULL;
     return (PyObject *) self;
 }
 
@@ -558,15 +554,13 @@ fileio_traverse(PyObject *op, visitproc visit, void *arg)
 {
     fileio *self = PyFileIO_CAST(op);
     Py_VISIT(Py_TYPE(self));
-    Py_VISIT(self->dict);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 static int
 fileio_clear(PyObject *op)
 {
-    fileio *self = PyFileIO_CAST(op);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     return 0;
 }
 
@@ -584,7 +578,7 @@ fileio_dealloc(PyObject *op)
         PyMem_Free(self->stat_atopen);
         self->stat_atopen = NULL;
     }
-    FT_CLEAR_WEAKREFS(op, self->weakreflist);
+    PyObject_ClearWeakRefs(op);
     (void)fileio_clear(op);
 
     PyTypeObject *tp = Py_TYPE(op);
@@ -1366,8 +1360,6 @@ static PyGetSetDef fileio_getsetlist[] = {
 
 static PyMemberDef fileio_members[] = {
     {"_finalizing", Py_T_BOOL, offsetof(fileio, finalizing), 0},
-    {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(fileio, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(fileio, dict), Py_READONLY},
     {NULL}
 };