Use a managed dict for the _io types

Replace the explicit dict slot of _io._IOBase and its C subclasses with
Py_TPFLAGS_MANAGED_DICT, so instance attributes are stored as inline
values with shared keys instead of a per-instance dict, and attribute
access can be specialized. The flag is inherited by all C and Python
subclasses.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: eendebakpt

Lib/test/test_io/test_general.py

@@ -895,6 +895,41 @@ def test_immutable_types(self):
                 with self.assertRaisesRegex(TypeError, "immutable"):
                     tp.foo = "bar"
 
+    def test_managed_dict(self):
+        # The _io types use Py_TPFLAGS_MANAGED_DICT: there is no explicit
+        # dict slot and instance attributes are stored as inline values.
+        _testinternalcapi = import_helper.import_module("_testinternalcapi")
+        _io = self._io
+        no_dict = {_io.IncrementalNewlineDecoder, _io._BytesIOBuffer}
+        for tp in self.types:
+            with self.subTest(tp=tp):
+                if tp in no_dict:
+                    self.assertEqual(tp.__dictoffset__, 0)
+                else:
+                    self.assertEqual(tp.__dictoffset__, -1)
+
+        # Use fresh subclasses: each class has its own shared keys, which
+        # may already be exhausted on the base types by earlier tests.
+        class BytesIO(_io.BytesIO): pass
+        class StringIO(_io.StringIO): pass
+        class BufferedReader(_io.BufferedReader): pass
+        class TextIOWrapper(_io.TextIOWrapper): pass
+        class RawIO(_io._RawIOBase):
+            def __init__(self):
+                self.attr = 1
+        instances = [
+            BytesIO(b"data"),
+            StringIO("data"),
+            BufferedReader(_io.BytesIO(b"data")),
+            TextIOWrapper(_io.BytesIO()),
+            RawIO(),
+        ]
+        for obj in instances:
+            with self.subTest(obj=type(obj).__name__):
+                obj.attr = 1
+                self.assertTrue(_testinternalcapi.has_inline_values(obj))
+                self.assertEqual(obj.__dict__["attr"], 1)
+
     def test_class_hierarchy(self):
         def check_subs(types, base):
             for tp in types:

Lib/test/test_io/test_memoryio.py

@@ -10,6 +10,7 @@
 import io
 import _pyio as pyio
 import pickle
+import struct
 import sys
 import weakref
 
@@ -1037,11 +1038,13 @@ def test_setstate(self):
     @support.cpython_only
     def test_sizeof(self):
         if support.Py_GIL_DISABLED:
-            basesize = support.calcobjsize('P2n2Pni')
+            basesize = support.calcobjsize('P2nPni')
         else:
-            basesize = support.calcobjsize('P2n2Pn')
+            basesize = support.calcobjsize('P2nPn')
         check = self.check_sizeof
         self.assertEqual(object.__sizeof__(io.BytesIO()), basesize)
+        # sys.getsizeof() includes the managed dict pre-header
+        basesize += 2 * struct.calcsize('P')
         check(io.BytesIO(), basesize )
         n = 1000  # use a variable to prevent constant folding
         check(io.BytesIO(b'a' * n), basesize + sys.getsizeof(b'a' * n))

Modules/_io/bufferedio.c

@@ -260,7 +260,6 @@ typedef struct {
     Py_ssize_t buffer_size;
     Py_ssize_t buffer_mask;
 
-    PyObject *dict;
     PyObject *weakreflist;
 } buffered;
 
@@ -417,7 +416,7 @@ buffered_clear(PyObject *op)
     buffered *self = buffered_CAST(op);
     self->ok = 0;
     Py_CLEAR(self->raw);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     return 0;
 }
 
@@ -467,8 +466,7 @@ buffered_traverse(PyObject *op, visitproc visit, void *arg)
     buffered *self = buffered_CAST(op);
     Py_VISIT(Py_TYPE(self));
     Py_VISIT(self->raw);
-    Py_VISIT(self->dict);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 /* Because this can call arbitrary code, it shouldn't be called when
@@ -2250,7 +2248,6 @@ typedef struct {
     PyObject_HEAD
     buffered *reader;
     buffered *writer;
-    PyObject *dict;
     PyObject *weakreflist;
 } rwpair;
 
@@ -2309,10 +2306,9 @@ bufferedrwpair_traverse(PyObject *op, visitproc visit, void *arg)
 {
     rwpair *self = rwpair_CAST(op);
     Py_VISIT(Py_TYPE(self));
-    Py_VISIT(self->dict);
     Py_VISIT(self->reader);
     Py_VISIT(self->writer);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 static int
@@ -2321,7 +2317,7 @@ bufferedrwpair_clear(PyObject *op)
     rwpair *self = rwpair_CAST(op);
     Py_CLEAR(self->reader);
     Py_CLEAR(self->writer);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     return 0;
 }
 
@@ -2639,7 +2635,6 @@ static PyMemberDef bufferedreader_members[] = {
     {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY},
     {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0},
     {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY},
     {NULL}
 };
 
@@ -2698,7 +2693,6 @@ static PyMemberDef bufferedwriter_members[] = {
     {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY},
     {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0},
     {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY},
     {NULL}
 };
 
@@ -2752,7 +2746,6 @@ static PyMethodDef bufferedrwpair_methods[] = {
 
 static PyMemberDef bufferedrwpair_members[] = {
     {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rwpair, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(rwpair, dict), Py_READONLY},
     {NULL}
 };
 
@@ -2815,7 +2808,6 @@ static PyMemberDef bufferedrandom_members[] = {
     {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY},
     {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0},
     {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY},
     {NULL}
 };
 

Modules/_io/bytesio.c

@@ -21,7 +21,6 @@ typedef struct {
     PyObject *buf;
     Py_ssize_t pos;
     Py_ssize_t string_size;
-    PyObject *dict;
     PyObject *weakreflist;
     Py_ssize_t exports;
 #ifdef Py_GIL_DISABLED
@@ -974,11 +973,16 @@ _io_BytesIO___getstate___impl(bytesio *self)
 
     if (initvalue == NULL)
         return NULL;
-    if (self->dict == NULL) {
-        dict = Py_NewRef(Py_None);
+    dict = PyObject_GenericGetDict((PyObject *)self, NULL);
+    if (dict == NULL) {
+        Py_DECREF(initvalue);
+        return NULL;
+    }
+    if (PyDict_GET_SIZE(dict) == 0) {
+        Py_SETREF(dict, Py_NewRef(Py_None));
     }
     else {
-        dict = PyDict_Copy(self->dict);
+        Py_SETREF(dict, PyDict_Copy(dict));
         if (dict == NULL) {
             Py_DECREF(initvalue);
             return NULL;
@@ -1061,16 +1065,16 @@ _io_BytesIO___setstate___impl(bytesio *self, PyObject *state)
                          Py_TYPE(dict)->tp_name);
             return NULL;
         }
-        if (self->dict) {
-            /* Alternatively, we could replace the internal dictionary
-               completely. However, it seems more practical to just update it. */
-            if (PyDict_Update(self->dict, dict) < 0)
-                return NULL;
+        /* Alternatively, we could replace the internal dictionary
+           completely. However, it seems more practical to just update it. */
+        PyObject *self_dict = PyObject_GenericGetDict((PyObject *)self, NULL);
+        if (self_dict == NULL) {
+            return NULL;
         }
-        else {
-            /* The LOAD_ATTR specializations read the dict slot lock-free
-               with an acquire load, so pair it with a release store. */
-            FT_ATOMIC_STORE_PTR_RELEASE(self->dict, Py_NewRef(dict));
+        int rc = PyDict_Update(self_dict, dict);
+        Py_DECREF(self_dict);
+        if (rc < 0) {
+            return NULL;
         }
     }
 
@@ -1089,7 +1093,7 @@ bytesio_dealloc(PyObject *op)
         PyErr_Print();
     }
     Py_CLEAR(self->buf);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     FT_CLEAR_WEAKREFS(op, self->weakreflist);
     tp->tp_free(self);
     Py_DECREF(tp);
@@ -1186,16 +1190,15 @@ bytesio_traverse(PyObject *op, visitproc visit, void *arg)
 {
     bytesio *self = bytesio_CAST(op);
     Py_VISIT(Py_TYPE(self));
-    Py_VISIT(self->dict);
     Py_VISIT(self->buf);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 static int
 bytesio_clear(PyObject *op)
 {
     bytesio *self = bytesio_CAST(op);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) {
         Py_CLEAR(self->buf);
     }
@@ -1240,7 +1243,6 @@ static struct PyMethodDef bytesio_methods[] = {
 
 static PyMemberDef bytesio_members[] = {
     {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(bytesio, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(bytesio, dict), Py_READONLY},
     {NULL}
 };
 

Modules/_io/fileio.c

@@ -81,7 +81,6 @@ typedef struct {
        gh-109523). */
     struct _Py_stat_struct *stat_atopen;
     PyObject *weakreflist;
-    PyObject *dict;
 } fileio;
 
 #define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type))
@@ -558,15 +557,13 @@ fileio_traverse(PyObject *op, visitproc visit, void *arg)
 {
     fileio *self = PyFileIO_CAST(op);
     Py_VISIT(Py_TYPE(self));
-    Py_VISIT(self->dict);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 static int
 fileio_clear(PyObject *op)
 {
-    fileio *self = PyFileIO_CAST(op);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     return 0;
 }
 
@@ -1367,7 +1364,6 @@ static PyGetSetDef fileio_getsetlist[] = {
 static PyMemberDef fileio_members[] = {
     {"_finalizing", Py_T_BOOL, offsetof(fileio, finalizing), 0},
     {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(fileio, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(fileio, dict), Py_READONLY},
     {NULL}
 };
 

Modules/_io/iobase.c

@@ -33,7 +33,6 @@ class _io._RawIOBase "PyObject *" "clinic_state()->PyRawIOBase_Type"
 typedef struct {
     PyObject_HEAD
 
-    PyObject *dict;
     PyObject *weakreflist;
 } iobase;
 
@@ -383,15 +382,13 @@ iobase_traverse(PyObject *op, visitproc visit, void *arg)
 {
     iobase *self = iobase_CAST(op);
     Py_VISIT(Py_TYPE(self));
-    Py_VISIT(self->dict);
-    return 0;
+    return PyObject_VisitManagedDict(op, visit, arg);
 }
 
 static int
 iobase_clear(PyObject *op)
 {
-    iobase *self = iobase_CAST(op);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     return 0;
 }
 
@@ -400,8 +397,8 @@ iobase_clear(PyObject *op)
 static void
 iobase_dealloc(PyObject *op)
 {
-    /* NOTE: since IOBaseObject has its own dict, Python-defined attributes
-       are still available here for close() to use.
+    /* NOTE: since the managed dict is only cleared below, Python-defined
+       attributes are still available here for close() to use.
        However, if the derived class declares a __slots__, those slots are
        already gone.
     */
@@ -417,7 +414,7 @@ iobase_dealloc(PyObject *op)
     PyTypeObject *tp = Py_TYPE(self);
     _PyObject_GC_UNTRACK(self);
     FT_CLEAR_WEAKREFS(op, self->weakreflist);
-    Py_CLEAR(self->dict);
+    PyObject_ClearManagedDict(op);
     tp->tp_free(self);
     Py_DECREF(tp);
 }
@@ -909,7 +906,6 @@ static PyGetSetDef iobase_getset[] = {
 
 static struct PyMemberDef iobase_members[] = {
     {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(iobase, weakreflist), Py_READONLY},
-    {"__dictoffset__", Py_T_PYSSIZET, offsetof(iobase, dict), Py_READONLY},
     {NULL},
 };
 
@@ -932,7 +928,7 @@ PyType_Spec _Py_iobase_spec = {
     .name = "_io._IOBase",
     .basicsize = sizeof(iobase),
     .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC |
-              Py_TPFLAGS_IMMUTABLETYPE),
+              Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_DICT),
     .slots = iobase_slots,
 };