Free objects with qsbr if shared

Author: DinoV

Include/internal/pycore_pymem.h

@@ -119,6 +119,13 @@ extern int _PyMem_DebugEnabled(void);
 // Enqueue a pointer to be freed possibly after some delay.
 extern void _PyMem_FreeDelayed(void *ptr);
 
+// Enqueue an object to be freed possibly after some delay
+extern void _PyObject_FreeDelayed(void *ptr);
+
+// Free all outstanding delayed freed objects.  The world must be
+// stopped before calling this.
+extern void _PyMem_ProcessAllDelayed(PyInterpreterState *interp);
+
 // Periodically process delayed free requests.
 extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
 

Objects/obmalloc.c

@@ -957,7 +957,7 @@ _PyMem_Strdup(const char *str)
 
 // A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
 struct _mem_work_item {
-    void *ptr;
+    void *ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
     uint64_t qsbr_goal;
 };
 
@@ -971,16 +971,26 @@ struct _mem_work_chunk {
     struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
 };
 
-void
-_PyMem_FreeDelayed(void *ptr)
+void free_work_item(void *ptr)
+{
+    if (((uintptr_t)ptr) & 0x01) {
+        PyObject_Free(((char *)ptr) - 1);
+    }
+    else {
+        PyMem_Free(ptr);
+    }
+}
+
+static void
+free_delayed(void *ptr)
 {
 #ifndef Py_GIL_DISABLED
-    PyMem_Free(ptr);
+    free_work_item(ptr);
 #else
     if (_PyRuntime.stoptheworld.world_stopped) {
         // Free immediately if the world is stopped, including during
         // interpreter shutdown.
-        PyMem_Free(ptr);
+        free_work_item(ptr);
         return;
     }
 
@@ -1007,7 +1017,7 @@ _PyMem_FreeDelayed(void *ptr)
     if (buf == NULL) {
         // failed to allocate a buffer, free immediately
         _PyEval_StopTheWorld(tstate->base.interp);
-        PyMem_Free(ptr);
+        free_work_item(ptr);
         _PyEval_StartTheWorld(tstate->base.interp);
         return;
     }
@@ -1024,6 +1034,20 @@ _PyMem_FreeDelayed(void *ptr)
 #endif
 }
 
+void
+_PyMem_FreeDelayed(void *ptr)
+{
+    assert(!((uintptr_t)ptr & 0x01));
+    free_delayed(ptr);
+}
+
+
+void _PyObject_FreeDelayed(void *ptr)
+{
+    assert(!((uintptr_t)ptr & 0x01));
+    free_delayed((void *)(((uintptr_t)ptr)|0x01));
+}
+
 static struct _mem_work_chunk *
 work_queue_first(struct llist_node *head)
 {
@@ -1043,7 +1067,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr,
                 return;
             }
 
-            PyMem_Free(item->ptr);
+            free_work_item(item->ptr);
             buf->rd_idx++;
         }
 
@@ -1119,6 +1143,23 @@ _PyMem_AbandonDelayed(PyThreadState *tstate)
     assert(llist_empty(queue));  // the thread's queue is now empty
 }
 
+void
+_PyMem_ProcessAllDelayed(PyInterpreterState *interp)
+{
+    PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
+    while (tstate != NULL) {
+        _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
+
+        // Process thread-local work
+        process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true);
+
+        tstate = PyThreadState_Next(tstate);
+    }
+
+    // Process shared interpreter work
+    process_interp_queue(&interp->mem_free_queue, ((_PyThreadStateImpl *)_PyThreadState_GET())->qsbr);
+}
+
 void
 _PyMem_FiniDelayed(PyInterpreterState *interp)
 {
@@ -1130,7 +1171,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
             // Free the remaining items immediately. There should be no other
             // threads accessing the memory at this point during shutdown.
             struct _mem_work_item *item = &buf->array[buf->rd_idx];
-            PyMem_Free(item->ptr);
+            free_work_item(item->ptr);
             buf->rd_idx++;
         }
 

Python/gc_free_threading.c

@@ -1025,6 +1025,11 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state)
     merge_all_queued_objects(interp, state);
     process_delayed_frees(interp);
 
+    // Free all outstanding objects.  During finalization we could collect
+    // types before the underlying objects so its important that we not
+    // traverse any delay-freed objects.
+    _PyMem_ProcessAllDelayed(interp);
+
     // Find unreachable objects
     int err = deduce_unreachable_heap(interp, state);
     if (err < 0) {
@@ -1704,8 +1709,13 @@ PyObject_GC_Del(void *op)
     }
 
     record_deallocation(_PyThreadState_GET());
-
-    PyObject_Free(((char *)op)-presize);
+    PyObject *self = (PyObject *)op;
+    if (_PyObject_GC_IS_SHARED(self)) {
+        _PyObject_FreeDelayed(((char *)op)-presize);
+    }
+    else {
+        PyObject_Free(((char *)op)-presize);
+    }
 }
 
 int