Merge branch 'main' into gh-100240-freelist

Author: colesbury

.github/workflows/reusable-tsan.yml

@@ -36,11 +36,11 @@ jobs:
         # Install clang-18
         wget https://apt.llvm.org/llvm.sh
         chmod +x llvm.sh
-        sudo ./llvm.sh 18
-        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100
-        sudo update-alternatives --set clang /usr/bin/clang-18
-        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100
-        sudo update-alternatives --set clang++ /usr/bin/clang++-18
+        sudo ./llvm.sh 17  # gh-121946: llvm-18 package is temporarily broken
+        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100
+        sudo update-alternatives --set clang /usr/bin/clang-17
+        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100
+        sudo update-alternatives --set clang++ /usr/bin/clang++-17
         # Reduce ASLR to avoid TSAN crashing
         sudo sysctl -w vm.mmap_rnd_bits=28
     - name: TSAN Option Setup

Doc/library/functions.rst

@@ -1509,7 +1509,7 @@ are always available.  They are listed here in alphabetical order.
    (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`,
    and :mod:`shutil`.
 
-   .. audit-event:: open file,mode,flags open
+   .. audit-event:: open path,mode,flags open
 
    The ``mode`` and ``flags`` arguments may have been modified or inferred from
    the original call.

Doc/library/unittest.mock.rst

@@ -860,6 +860,20 @@ object::
     3
     >>> p.assert_called_once_with()
 
+.. caution::
+
+    If an :exc:`AttributeError` is raised by :class:`PropertyMock`,
+    it will be interpreted as a missing descriptor and
+    :meth:`~object.__getattr__` will be called on the parent mock::
+
+        >>> m = MagicMock()
+        >>> no_attribute = PropertyMock(side_effect=AttributeError)
+        >>> type(m).my_property = no_attribute
+        >>> m.my_property
+        <MagicMock name='mock.my_property' id='140165240345424'>
+
+    See :meth:`~object.__getattr__` for details.
+
 
 .. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs)
 

Include/cpython/pystate.h

@@ -68,8 +68,6 @@ struct _ts {
        pycore_ceval.h. */
     uintptr_t eval_breaker;
 
-    PyObject *asyncio_running_loop; // Strong reference
-
     struct {
         /* Has been initialized to a safe state.
 

Include/internal/pycore_frame.h

@@ -68,7 +68,7 @@ typedef struct _PyInterpreterFrame {
     PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */
     PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */
     _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */
-    int stacktop;  /* Offset of TOS from localsplus  */
+    _PyStackRef *stackpointer;
     uint16_t return_offset;  /* Only relevant during a function call */
     char owner;
     /* Locals and stack */
@@ -88,20 +88,20 @@ static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) {
 }
 
 static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) {
-    assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
-    assert(!PyStackRef_IsNull(f->localsplus[f->stacktop-1]));
-    return f->localsplus[f->stacktop-1];
+    assert(f->stackpointer >  f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus);
+    assert(!PyStackRef_IsNull(f->stackpointer[-1]));
+    return f->stackpointer[-1];
 }
 
 static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) {
-    assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
-    f->stacktop--;
-    return f->localsplus[f->stacktop];
+    assert(f->stackpointer >  f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus);
+    f->stackpointer--;
+    return *f->stackpointer;
 }
 
 static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, _PyStackRef value) {
-    f->localsplus[f->stacktop] = value;
-    f->stacktop++;
+    *f->stackpointer = value;
+    f->stackpointer++;
 }
 
 #define FRAME_SPECIALS_SIZE ((int)((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *)))
@@ -117,9 +117,12 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code)
 
 static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest)
 {
-    assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus);
     *dest = *src;
-    for (int i = 1; i < src->stacktop; i++) {
+    assert(src->stackpointer != NULL);
+    int stacktop = (int)(src->stackpointer - src->localsplus);
+    assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus);
+    dest->stackpointer = dest->localsplus + stacktop;
+    for (int i = 1; i < stacktop; i++) {
         dest->localsplus[i] = src->localsplus[i];
     }
     // Don't leave a dangling pointer to the old frame when creating generators
@@ -141,7 +144,7 @@ _PyFrame_Initialize(
     frame->f_builtins = func->func_builtins;
     frame->f_globals = func->func_globals;
     frame->f_locals = locals;
-    frame->stacktop = code->co_nlocalsplus;
+    frame->stackpointer = frame->localsplus + code->co_nlocalsplus;
     frame->frame_obj = NULL;
     frame->instr_ptr = _PyCode_CODE(code);
     frame->return_offset = 0;
@@ -161,22 +164,23 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
     return frame->localsplus;
 }
 
-/* Fetches the stack pointer, and sets stacktop to -1.
-   Having stacktop <= 0 ensures that invalid
-   values are not visible to the cycle GC.
-   We choose -1 rather than 0 to assist debugging. */
+/* Fetches the stack pointer, and sets stackpointer to NULL.
+   Having stackpointer == NULL ensures that invalid
+   values are not visible to the cycle GC. */
 static inline _PyStackRef*
 _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 {
-    _PyStackRef *sp = frame->localsplus + frame->stacktop;
-    frame->stacktop = -1;
+    assert(frame->stackpointer != NULL);
+    _PyStackRef *sp = frame->stackpointer;
+    frame->stackpointer = NULL;
     return sp;
 }
 
 static inline void
 _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
 {
-    frame->stacktop = (int)(stack_pointer - frame->localsplus);
+    assert(frame->stackpointer == NULL);
+    frame->stackpointer = stack_pointer;
 }
 
 /* Determine whether a frame is incomplete.
@@ -304,7 +308,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     frame->f_globals = NULL;
 #endif
     frame->f_locals = NULL;
-    frame->stacktop = code->co_nlocalsplus + stackdepth;
+    assert(stackdepth <= code->co_stacksize);
+    frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth;
     frame->frame_obj = NULL;
     frame->instr_ptr = _PyCode_CODE(code);
     frame->owner = FRAME_OWNED_BY_THREAD;

Include/internal/pycore_object.h

@@ -15,6 +15,30 @@ extern "C" {
 #include "pycore_pyatomic_ft_wrappers.h"  // FT_ATOMIC_STORE_PTR_RELAXED
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 
+
+#define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1)
+
+// gh-121528, gh-118997: Similar to _Py_IsImmortal() but be more loose when
+// comparing the reference count to stay compatible with C extensions built
+// with the stable ABI 3.11 or older. Such extensions implement INCREF/DECREF
+// as refcnt++ and refcnt-- without taking in account immortal objects. For
+// example, the reference count of an immortal object can change from
+// _Py_IMMORTAL_REFCNT to _Py_IMMORTAL_REFCNT+1 (INCREF) or
+// _Py_IMMORTAL_REFCNT-1 (DECREF).
+//
+// This function should only be used in assertions. Otherwise, _Py_IsImmortal()
+// must be used instead.
+static inline int _Py_IsImmortalLoose(PyObject *op)
+{
+#if defined(Py_GIL_DISABLED)
+    return _Py_IsImmortal(op);
+#else
+    return (op->ob_refcnt >= _Py_IMMORTAL_REFCNT_LOOSE);
+#endif
+}
+#define _Py_IsImmortalLoose(op) _Py_IsImmortalLoose(_PyObject_CAST(op))
+
+
 /* Check if an object is consistent. For example, ensure that the reference
    counter is greater than or equal to 1, and ensure that ob_type is not NULL.
 
@@ -134,7 +158,7 @@ PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op);
 static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt)
 {
     if (op) {
-        assert(_Py_IsImmortal(op));
+        assert(_Py_IsImmortalLoose(op));
 #ifdef Py_GIL_DISABLED
         op->ob_tid = _Py_UNOWNED_TID;
         op->ob_ref_local = 0;
@@ -266,7 +290,7 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
 {
     assert(op != NULL);
     Py_SET_TYPE(op, typeobj);
-    assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
+    assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortalLoose(typeobj));
     Py_INCREF(typeobj);
     _Py_NewReference(op);
 }

Include/internal/pycore_tstate.h

@@ -21,6 +21,8 @@ typedef struct _PyThreadStateImpl {
     // semi-public fields are in PyThreadState.
     PyThreadState base;
 
+    PyObject *asyncio_running_loop; // Strong reference
+
     struct _qsbr_thread_state *qsbr;  // only used by free-threaded build
     struct llist_node mem_free_queue; // delayed free queue
 

Lib/email/_header_value_parser.py

@@ -2988,6 +2988,7 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset,
             excess = len(encoded_word) - remaining_space
         lines[-1] += encoded_word
         to_encode = to_encode[len(to_encode_word):]
+        leading_whitespace = ''
 
         if to_encode:
             lines.append(' ')

Lib/test/crashers/bogus_code_obj.py

@@ -12,8 +12,8 @@
 
 """
 
-import types
+def f():
+    pass
 
-co = types.CodeType(0, 0, 0, 0, 0, 0, b'\x04\x00\x71\x00',
-                    (), (), (), '', '', 1, b'')
-exec(co)
+f.__code__ = f.__code__.replace(co_code=b"")
+f()

Lib/test/test_email/test_generator.py

@@ -294,6 +294,19 @@ def test_defaults_handle_spaces_between_encoded_words_when_folded(self):
         g.flatten(msg)
         self.assertEqual(s.getvalue(), expected)
 
+    def test_defaults_handle_spaces_when_encoded_words_is_folded_in_middle(self):
+        source = ('A very long long long long long long long long long long long long '
+                  'long long long long long long long long long long long súmmäry')
+        expected = ('Subject: A very long long long long long long long long long long long long\n'
+                    ' long long long long long long long long long long long =?utf-8?q?s=C3=BAmm?=\n'
+                    ' =?utf-8?q?=C3=A4ry?=\n\n').encode('ascii')
+        msg = EmailMessage()
+        msg['Subject'] = source
+        s = io.BytesIO()
+        g = BytesGenerator(s)
+        g.flatten(msg)
+        self.assertEqual(s.getvalue(), expected)
+
     def test_defaults_handle_spaces_at_start_of_subject(self):
         source = " Уведомление"
         expected = b"Subject:  =?utf-8?b?0KPQstC10LTQvtC80LvQtdC90LjQtQ==?=\n\n"