diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index a9db8860e91c06c..477b3c20e53f6e6 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -12,6 +12,11 @@ extern "C" { #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_stats.h" +/* Flags that determine the C calling convention. */ +#define _Py_METH_CALL_FLAGS \ + (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | \ + METH_KEYWORDS | METH_METHOD) + /* Suggested size (number of positional arguments) for arrays of PyObject* allocated on a C stack to avoid allocating memory on the heap memory. Such array is used to pass positional arguments to call functions of the diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 36efab518781410..56db091f3ae8e3a 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -3307,6 +3307,186 @@ def testfunc(n): self.assertIn("_CALL_BUILTIN_FAST_WITH_KEYWORDS", uops) self.assertNotIn("_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", uops) + def test_call_builtin_o_extra_flags(self): + # Extra method flags must not prevent callable guard elimination. + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + namespace = { + "METH_CLASS_O": _testcapi.MethClass.meth_o, + "METH_STATIC_O": _testcapi.MethStatic.meth_o, + "METH_COEXIST_O": {}.__contains__, + } + + @reset_code + def testfunc(n): + for _ in range(n): + class_result = METH_CLASS_O(1) + static_result = METH_STATIC_O(1) + coexist_result = METH_COEXIST_O(1) + return class_result, static_result, coexist_result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, ((_testcapi.MethClass, 1), (None, 1), False)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_BUILTIN_O"), 3) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_O", uops) + + def test_call_builtin_fast_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + obj = _testcapi.MethInstance() + namespace = { + "METH_CLASS_FASTCALL": _testcapi.MethClass.meth_fastcall, + "METH_STATIC_FASTCALL": _testcapi.MethStatic.meth_fastcall, + "METH_COEXIST_FASTCALL": obj.meth_fastcall_coexist, + } + + @reset_code + def testfunc(n): + for _ in range(n): + class_result = METH_CLASS_FASTCALL(1, 2) + static_result = METH_STATIC_FASTCALL(1, 2) + coexist_result = METH_COEXIST_FASTCALL(1, 2) + return class_result, static_result, coexist_result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, ( + (_testcapi.MethClass, (1, 2)), + (None, (1, 2)), + (obj, (1, 2)), + )) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_BUILTIN_FAST"), 3) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_FAST", uops) + + def test_call_builtin_fast_with_keywords_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + obj = _testcapi.MethInstance() + namespace = { + "METH_CLASS_FASTCALL_KEYWORDS": ( + _testcapi.MethClass.meth_fastcall_keywords), + "METH_STATIC_FASTCALL_KEYWORDS": ( + _testcapi.MethStatic.meth_fastcall_keywords), + "METH_COEXIST_FASTCALL_KEYWORDS": ( + obj.meth_fastcall_keywords_coexist), + } + + @reset_code + def testfunc(n): + # Use positional arguments to exercise CALL, not CALL_KW. + for _ in range(n): + class_result = METH_CLASS_FASTCALL_KEYWORDS(1, 2) + static_result = METH_STATIC_FASTCALL_KEYWORDS(1, 2) + coexist_result = METH_COEXIST_FASTCALL_KEYWORDS(1, 2) + return class_result, static_result, coexist_result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, ( + (_testcapi.MethClass, (1, 2), {}), + (None, (1, 2), {}), + (obj, (1, 2), {}), + )) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_BUILTIN_FAST_WITH_KEYWORDS"), 3) + self.assertNotIn("_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", uops) + + def test_call_method_descriptor_o_extra_flags(self): + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + @reset_code + def testfunc(n): + d = {1: None} + for _ in range(n): + result = d.__contains__(1) + return result + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIs(res, True) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_METHOD_DESCRIPTOR_O_INLINE"), 1) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_O", uops) + + def test_call_method_descriptor_noargs_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + namespace = { + "METH_COEXIST_NOARGS_OBJECT": _testcapi.DocStringNoSignatureTest(), + } + + @reset_code + def testfunc(n): + for _ in range(n): + result = METH_COEXIST_NOARGS_OBJECT.meth_noargs_coexist() + return result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNone(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual( + uops.count("_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE"), 1) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS", uops) + + def test_call_method_descriptor_fast_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + obj = _testcapi.MethInstance() + namespace = {"METH_COEXIST_FAST_OBJECT": obj} + + @reset_code + def testfunc(n): + for _ in range(n): + result = METH_COEXIST_FAST_OBJECT.meth_fastcall_coexist(1, 2) + return result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, (obj, (1, 2))) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_CALL_METHOD_DESCRIPTOR_FAST_INLINE"), 1) + self.assertNotIn("_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST", uops) + + def test_call_method_descriptor_fast_with_keywords_extra_flags(self): + _testcapi = import_helper.import_module("_testcapi") + self.addCleanup(_testinternalcapi.clear_executor_deletion_list) + + obj = _testcapi.MethInstance() + namespace = {"METH_COEXIST_FAST_OBJECT": obj} + + @reset_code + def testfunc(n): + # Use positional arguments to exercise CALL, not CALL_KW. + for _ in range(n): + result = ( + METH_COEXIST_FAST_OBJECT.meth_fastcall_keywords_coexist( + 1, 2)) + return result + + testfunc = types.FunctionType(testfunc.__code__, namespace) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, (obj, (1, 2), {})) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual( + uops.count("_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE"), 1) + self.assertNotIn( + "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", uops) + def test_call_method_descriptor_o(self): def testfunc(n): x = 0 diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 7946550ec0db637..60879e2774e7077 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -2171,6 +2171,89 @@ class MyList(list): pass self.assert_no_opcode(my_list_append, "CALL_LIST_APPEND") self.assert_no_opcode(my_list_append, "CALL") + @cpython_only + @requires_specialization + def test_call_c_function_extra_flags(self): + # METH_CLASS, METH_STATIC and METH_COEXIST do not change the C + # calling convention, so the specialized instructions must not + # miss because of them. + _testcapi = import_module("_testcapi") + + def call_1(func, arg): + return func(arg) + + def call_2(func, arg1, arg2): + return func(arg1, arg2) + + def call_3(func, arg1, arg2, arg3): + return func(arg1, arg2, arg3) + + def call_method(obj, arg): + return obj.__contains__(arg) + + def call_method_noargs(obj): + return obj.meth_noargs_coexist() + + def call_method_fast(obj, arg1, arg2): + return obj.meth_fastcall_coexist(arg1, arg2) + + def call_method_fast_with_keywords(obj, arg1, arg2): + return obj.meth_fastcall_keywords_coexist(arg1, arg2) + + def call_site(f): + [call] = [instr for instr in dis.get_instructions(f, adaptive=True) + if instr.baseopname == "CALL"] + cache = {name: data for name, _, data in call.cache_info} + return call.opname, cache["counter"] + + def label(obj): + return getattr(obj, "__qualname__", type(obj).__name__) + + coexist = _testcapi.MethInstance() + cases = [ + # dict.__contains__ has METH_O | METH_COEXIST + (call_1, {}.__contains__, ("key",), "CALL_BUILTIN_O"), + (call_method, {}, ("key",), "CALL_METHOD_DESCRIPTOR_O"), + # meth_noargs_coexist has METH_NOARGS | METH_COEXIST + (call_method_noargs, _testcapi.DocStringNoSignatureTest(), (), + "CALL_METHOD_DESCRIPTOR_NOARGS"), + # METH_FASTCALL, with or without METH_KEYWORDS, and METH_COEXIST + (call_method_fast, coexist, (1, 2), + "CALL_METHOD_DESCRIPTOR_FAST"), + (call_method_fast_with_keywords, coexist, (1, 2), + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS"), + (call_3, _testcapi.MethInstance.meth_fastcall_coexist, + (coexist, 1, 2), "CALL_METHOD_DESCRIPTOR_FAST"), + (call_3, _testcapi.MethInstance.meth_fastcall_keywords_coexist, + (coexist, 1, 2), "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS"), + (call_2, coexist.meth_fastcall_coexist, (1, 2), + "CALL_BUILTIN_FAST"), + (call_2, coexist.meth_fastcall_keywords_coexist, (1, 2), + "CALL_BUILTIN_FAST_WITH_KEYWORDS"), + ] + for owner in (_testcapi.MethClass, _testcapi.MethStatic): + cases += [ + (call_1, owner.meth_o, (1,), "CALL_BUILTIN_O"), + (call_2, owner.meth_fastcall, (1, 2), "CALL_BUILTIN_FAST"), + (call_2, owner.meth_fastcall_keywords, (1, 2), + "CALL_BUILTIN_FAST_WITH_KEYWORDS"), + ] + + for f, func, args, opname in cases: + with self.subTest(call=f.__name__, func=label(func)): + reset_code(f) + expected = f(func, *args) + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f(func, *args) + self.assertEqual(call_site(f)[0], opname) + + # A hit leaves the counter of the call site unchanged. + # A miss decrements it. + before = call_site(f) + for _ in range(10): + self.assertEqual(f(func, *args), expected) + self.assertEqual(call_site(f), before) + @cpython_only @requires_specialization def test_load_attr_module_with_getattr(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-20-08-59-48.gh-issue-157833.3b0bac.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-20-08-59-48.gh-issue-157833.3b0bac.rst new file mode 100644 index 000000000000000..cf38270dc447a88 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-20-08-59-48.gh-issue-157833.3b0bac.rst @@ -0,0 +1,2 @@ +Fix call specialization for C functions and methods with ``METH_CLASS``, +``METH_STATIC``, or ``METH_COEXIST`` flags, improving their call performance. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 577bb14df134059..19c02e6774e3d48 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3360,6 +3360,11 @@ static PyMethodDef meth_instance_methods[] = { {"meth_noargs", meth_noargs, METH_NOARGS}, {"meth_fastcall", _PyCFunction_CAST(meth_fastcall), METH_FASTCALL}, {"meth_fastcall_keywords", _PyCFunction_CAST(meth_fastcall_keywords), METH_FASTCALL|METH_KEYWORDS}, + {"meth_fastcall_coexist", _PyCFunction_CAST(meth_fastcall), + METH_FASTCALL|METH_COEXIST}, + {"meth_fastcall_keywords_coexist", + _PyCFunction_CAST(meth_fastcall_keywords), + METH_FASTCALL|METH_KEYWORDS|METH_COEXIST}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 7a75e80298fcd82..79aa51ffdfbfef7 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -2488,7 +2488,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2581,7 +2582,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2674,7 +2676,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4041,7 +4044,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_FASTCALL) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4160,7 +4164,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4283,7 +4288,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_NOARGS) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4406,7 +4412,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_O) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 8ceef6489881679..a4f0987f0c1e0a0 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -936,8 +936,7 @@ PyDescr_NewMethod(PyTypeObject *type, PyMethodDef *method) { /* Figure out correct vectorcall function to use */ vectorcallfunc vectorcall; - switch (method->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | - METH_O | METH_KEYWORDS | METH_METHOD)) + switch (method->ml_flags & _Py_METH_CALL_FLAGS) { case METH_VARARGS: vectorcall = method_vectorcall_VARARGS; diff --git a/Objects/methodobject.c b/Objects/methodobject.c index e6e469ca270ac96..9b37b1da95bb501 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -48,8 +48,7 @@ PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *c { /* Figure out correct vectorcall function to use */ vectorcallfunc vectorcall; - switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | - METH_O | METH_KEYWORDS | METH_METHOD)) + switch (ml->ml_flags & _Py_METH_CALL_FLAGS) { case METH_VARARGS: case METH_VARARGS | METH_KEYWORDS: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fb0cdf4d65e060d..75b0b66745b3807 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -10,6 +10,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" +#include "pycore_call.h" // _Py_METH_CALL_FLAGS #include "pycore_cell.h" // PyCell_GetRef() #include "pycore_ceval.h" // _PyEval_LazyImportName(), _PyEval_LazyImportFrom() #include "pycore_code.h" @@ -4871,7 +4872,8 @@ dummy_func( op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O); + EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_O); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; @@ -4915,7 +4917,8 @@ dummy_func( op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); + EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_FASTCALL); } op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -4953,7 +4956,8 @@ dummy_func( op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); - EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); + EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)); } op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -5087,7 +5091,8 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - EXIT_IF(method->d_method->ml_flags != METH_O); + EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_O); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; @@ -5162,7 +5167,8 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - EXIT_IF(method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); + EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + (METH_FASTCALL | METH_KEYWORDS)); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { @@ -5236,7 +5242,8 @@ dummy_func( PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - EXIT_IF(method->d_method->ml_flags != METH_NOARGS); + EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_NOARGS); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; @@ -5305,7 +5312,8 @@ dummy_func( PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); - EXIT_IF(method->d_method->ml_flags != METH_FASTCALL); + EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_FASTCALL); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9aad9e003765cf8..50aaf63dccfba6c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -18365,7 +18365,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_O) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -18439,7 +18440,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -18505,7 +18507,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -19152,7 +19155,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (method->d_method->ml_flags != METH_O) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_O) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -19357,7 +19361,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + (METH_FASTCALL | METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -19501,7 +19506,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (method->d_method->ml_flags != METH_NOARGS) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_NOARGS) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -19630,7 +19636,8 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (method->d_method->ml_flags != METH_FASTCALL) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 77c18b3d61fefc7..aa3a838868bf7cf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2488,7 +2488,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2581,7 +2582,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2674,7 +2676,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + if ((PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4041,7 +4044,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_FASTCALL) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4160,7 +4164,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4283,7 +4288,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_NOARGS) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4406,7 +4412,8 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (method->d_method->ml_flags != METH_O) { + if ((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != + METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e726dc0e6fd1114..9b70874b35a009e 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -13,6 +13,7 @@ * */ #include "Python.h" #include "opcode.h" +#include "pycore_call.h" // _Py_METH_CALL_FLAGS #include "pycore_dict.h" #include "pycore_interp.h" #include "pycore_opcode_metadata.h" diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 5246e50633461bd..fc507027615206c 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_call.h" // _Py_METH_CALL_FLAGS #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" @@ -1711,7 +1712,9 @@ dummy_func(void) { if (sym_is_not_null(self_or_null)) { total_args++; } - if (total_args == 1 && PyCFunction_GET_FLAGS(callable_o) == METH_O) { + if (total_args == 1 && + (PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) == METH_O) { ADD_OP(_NOP, 0, 0); } } @@ -1723,7 +1726,8 @@ dummy_func(void) { op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == + METH_FASTCALL) { ADD_OP(_NOP, 0, 0); } } @@ -1735,7 +1739,8 @@ dummy_func(void) { op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == + (METH_FASTCALL | METH_KEYWORDS)) { ADD_OP(_NOP, 0, 0); } } @@ -1786,9 +1791,10 @@ dummy_func(void) { else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args == 2 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_O && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == METH_O && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -1813,9 +1819,11 @@ dummy_func(void) { else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == (METH_FASTCALL|METH_KEYWORDS) && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + (METH_FASTCALL | METH_KEYWORDS) && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -1840,9 +1848,11 @@ dummy_func(void) { else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args == 1 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_NOARGS && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + METH_NOARGS && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -1918,9 +1928,11 @@ dummy_func(void) { else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_FASTCALL && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + METH_FASTCALL && self_type == d_type) { ADD_OP(_NOP, 0, 0); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 21f275f27cafe04..3c4cc6e7c82dde0 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -4500,7 +4500,9 @@ if (sym_is_not_null(self_or_null)) { total_args++; } - if (total_args == 1 && PyCFunction_GET_FLAGS(callable_o) == METH_O) { + if (total_args == 1 && + (PyCFunction_GET_FLAGS(callable_o) & + _Py_METH_CALL_FLAGS) == METH_O) { ADD_OP(_NOP, 0, 0); } } @@ -4546,7 +4548,8 @@ callable = stack_pointer[-2 - oparg]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == METH_FASTCALL) { + if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == + METH_FASTCALL) { ADD_OP(_NOP, 0, 0); } } @@ -4569,7 +4572,8 @@ callable = stack_pointer[-2 - oparg]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { - if (PyCFunction_GET_FLAGS(callable_o) == (METH_FASTCALL | METH_KEYWORDS)) { + if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == + (METH_FASTCALL | METH_KEYWORDS)) { ADD_OP(_NOP, 0, 0); } } @@ -4755,9 +4759,10 @@ else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args == 2 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_O && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == METH_O && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -4855,9 +4860,11 @@ else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == (METH_FASTCALL|METH_KEYWORDS) && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + (METH_FASTCALL | METH_KEYWORDS) && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -4910,9 +4917,11 @@ else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args == 1 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_NOARGS && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + METH_NOARGS && self_type == d_type) { ADD_OP(_NOP, 0, 0); } @@ -4998,9 +5007,11 @@ else { self_type = sym_get_type(args[0]); } - PyTypeObject *d_type = ((PyMethodDescrObject *)callable_o)->d_common.d_type; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + PyTypeObject *d_type = method->d_common.d_type; if (total_args != 0 && - ((PyMethodDescrObject *)callable_o)->d_method->ml_flags == METH_FASTCALL && + (method->d_method->ml_flags & _Py_METH_CALL_FLAGS) == + METH_FASTCALL && self_type == d_type) { ADD_OP(_NOP, 0, 0); } diff --git a/Python/specialize.c b/Python/specialize.c index 05cb76ff015ff40..f658df3bc5d2228 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3,6 +3,7 @@ #include "opcode.h" #include "pycore_bytesobject.h" // _PyBytes_Concat +#include "pycore_call.h" // _Py_METH_CALL_FLAGS #include "pycore_code.h" #include "pycore_critical_section.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type @@ -1710,9 +1711,7 @@ static int specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, _Py_CODEUNIT *instr, int nargs) { - switch (descr->d_method->ml_flags & - (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { + switch (descr->d_method->ml_flags & _Py_METH_CALL_FLAGS) { case METH_NOARGS: { if (nargs != 1) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); @@ -1832,9 +1831,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER); return 1; } - switch (PyCFunction_GET_FLAGS(callable) & - (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { + switch (PyCFunction_GET_FLAGS(callable) & _Py_METH_CALL_FLAGS) { case METH_O: { if (nargs != 1) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);