gh-157526: Call __instancecheck__ and __subclasscheck__ without creating a bound method

Include/internal/pycore_object.h

@@ -957,7 +957,8 @@ extern int _PyObject_IsInstanceDictEmpty(PyObject *);
 
 // Export for 'math' shared extension
 PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *);
-PyAPI_FUNC(int) _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self);
+PyAPI_FUNC(int) _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method,
+                                              _PyStackRef *self);
 
 // Calls the method named `attr` on `self`, but does not set an exception if
 // the attribute does not exist.

Objects/abstract.c

@@ -9,6 +9,7 @@
 #include "pycore_list.h"          // _PyList_AppendTakeRef()
 #include "pycore_long.h"          // _PyLong_IsNegative()
 #include "pycore_object.h"        // _Py_CheckSlotResult()
+#include "pycore_stackref.h"      // _PyStackRef
 #include "pycore_pybuffer.h"      // _PyBuffer_ReleaseInInterpreterAndRawFree()
 #include "pycore_pyerrors.h"      // _PyErr_Occurred()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
@@ -2635,6 +2636,37 @@ object_isinstance(PyObject *inst, PyObject *cls)
     return retval;
 }
 
+static int
+call_special_method(PyThreadState *tstate, PyObject *cls, PyObject *name,
+                    const char *where, PyObject *arg, PyObject **res)
+{
+    _PyCStackRef cref;
+    _PyThreadState_PushCStackRef(tstate, &cref);
+    _PyStackRef self = PyStackRef_FromPyObjectBorrow(cls);
+    int found = _PyObject_LookupSpecialMethod(name, &cref.ref, &self);
+    if (found > 0) {
+        *res = NULL;
+        if (!_Py_EnterRecursiveCallTstate(tstate, where)) {
+            PyObject *method = PyStackRef_AsPyObjectBorrow(cref.ref);
+            PyObject *args[2] = {PyStackRef_AsPyObjectBorrow(self), arg};
+            if (args[0] != NULL) {
+                /* Unbound method: prepend self. */
+                *res = PyObject_Vectorcall(method, args, 2, NULL);
+            }
+            else {
+                *res = PyObject_Vectorcall(method, args + 1, 1, NULL);
+            }
+            _Py_LeaveRecursiveCallTstate(tstate);
+        }
+        if (*res == NULL) {
+            found = -1;
+        }
+    }
+    PyStackRef_XCLOSE(self);
+    _PyThreadState_PopCStackRef(tstate, &cref);
+    return found;
+}
+
 static int
 object_recursive_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls)
 {
@@ -2672,26 +2704,16 @@ object_recursive_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls
         return r;
     }
 
-    PyObject *checker = _PyObject_LookupSpecial(cls, &_Py_ID(__instancecheck__));
-    if (checker != NULL) {
-        if (_Py_EnterRecursiveCallTstate(tstate, " in __instancecheck__")) {
-            Py_DECREF(checker);
-            return -1;
-        }
-
-        PyObject *res = PyObject_CallOneArg(checker, inst);
-        _Py_LeaveRecursiveCallTstate(tstate);
-        Py_DECREF(checker);
-
-        if (res == NULL) {
-            return -1;
-        }
+    PyObject *res;
+    int found = call_special_method(tstate, cls, &_Py_ID(__instancecheck__),
+                                    " in __instancecheck__", inst, &res);
+    if (found > 0) {
         int ok = PyObject_IsTrue(res);
         Py_DECREF(res);
 
         return ok;
     }
-    else if (_PyErr_Occurred(tstate)) {
+    else if (found < 0) {
         return -1;
     }
 
@@ -2731,8 +2753,6 @@ recursive_issubclass(PyObject *derived, PyObject *cls)
 static int
 object_issubclass(PyThreadState *tstate, PyObject *derived, PyObject *cls)
 {
-    PyObject *checker;
-
     /* We know what type's __subclasscheck__ does. */
     if (PyType_CheckExact(cls)) {
         /* Quick test for an exact match */
@@ -2763,23 +2783,15 @@ object_issubclass(PyThreadState *tstate, PyObject *derived, PyObject *cls)
         return r;
     }
 
-    checker = _PyObject_LookupSpecial(cls, &_Py_ID(__subclasscheck__));
-    if (checker != NULL) {
-        int ok = -1;
-        if (_Py_EnterRecursiveCallTstate(tstate, " in __subclasscheck__")) {
-            Py_DECREF(checker);
-            return ok;
-        }
-        PyObject *res = PyObject_CallOneArg(checker, derived);
-        _Py_LeaveRecursiveCallTstate(tstate);
-        Py_DECREF(checker);
-        if (res != NULL) {
-            ok = PyObject_IsTrue(res);
-            Py_DECREF(res);
-        }
+    PyObject *res;
+    int found = call_special_method(tstate, cls, &_Py_ID(__subclasscheck__),
+                                    " in __subclasscheck__", derived, &res);
+    if (found > 0) {
+        int ok = PyObject_IsTrue(res);
+        Py_DECREF(res);
         return ok;
     }
-    else if (_PyErr_Occurred(tstate)) {
+    else if (found < 0) {
         return -1;
     }
 

Objects/typeobject.c

@@ -2931,17 +2931,22 @@ _PyObject_LookupSpecial(PyObject *self, PyObject *attr)
     return res;
 }
 
-// Lookup the method name `attr` on `self`. On entry, `method_and_self[0]`
-// is null and `method_and_self[1]` is `self`. On exit, `method_and_self[0]`
-// is the method object and `method_and_self[1]` is `self` if the method is
-// not bound.
+// Lookup the method name `attr` on `*self`. On entry, `*method` is null.
+// On exit, `*method` is the method object and `*self` is cleared if the
+// method is bound.
 // Return 1 on success, -1 on error, and 0 if the method is missing.
+//
+// `method` must point to a location that the garbage collector can see,
+// such as the `ref` field of a `_PyCStackRef` or a slot on the interpreter
+// stack. A descriptor may be invoked while `*method` holds the only
+// reference to the method object, and that can trigger a collection.
 int
-_PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self)
+_PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method,
+                              _PyStackRef *self)
 {
-    PyObject *self = PyStackRef_AsPyObjectBorrow(method_and_self[1]);
-    _PyType_LookupStackRefAndVersion(Py_TYPE(self), attr, &method_and_self[0]);
-    PyObject *method_o = PyStackRef_AsPyObjectBorrow(method_and_self[0]);
+    PyObject *self_o = PyStackRef_AsPyObjectBorrow(*self);
+    _PyType_LookupStackRefAndVersion(Py_TYPE(self_o), attr, method);
+    PyObject *method_o = PyStackRef_AsPyObjectBorrow(*method);
     if (method_o == NULL) {
         return 0;
     }
@@ -2953,14 +2958,14 @@ _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self)
 
     descrgetfunc f = Py_TYPE(method_o)->tp_descr_get;
     if (f != NULL) {
-        PyObject *func = f(method_o, self, (PyObject *)(Py_TYPE(self)));
+        PyObject *func = f(method_o, self_o, (PyObject *)(Py_TYPE(self_o)));
         if (func == NULL) {
             return -1;
         }
-        PyStackRef_CLEAR(method_and_self[0]); // clear method
-        method_and_self[0] = PyStackRef_FromPyObjectSteal(func);
+        PyStackRef_CLEAR(*method); // clear method
+        *method = PyStackRef_FromPyObjectSteal(func);
     }
-    PyStackRef_CLEAR(method_and_self[1]); // clear self
+    PyStackRef_CLEAR(*self); // clear self
     return 1;
 }
 

Python/bytecodes.c

@@ -4141,7 +4141,8 @@ dummy_func(
 
         op(_LOAD_SPECIAL, (method_and_self[2] -- method_and_self[2])) {
             PyObject *name = _Py_SpecialMethods[oparg].name;
-            int err = _PyObject_LookupSpecialMethod(name, method_and_self);
+            int err = _PyObject_LookupSpecialMethod(name, &method_and_self[0],
+                                                    &method_and_self[1]);
             if (err <= 0) {
                 if (err == 0) {
                     PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]);