PORTREVISION=	1

From: Victor Stinner <vstinner@python.org>

Date: Mon, 9 Mar 2020 23:37:49 +0100

Subject: [PATCH] bpo-19466: Py_Finalize() clears daemon threads earlier

 (GH-18848)

 Clear the frames of daemon threads earlier during the Python shutdown to

 call objects destructors. So "unclosed file" resource warnings are now

 emitted for daemon threads in a more reliable way.

 Cleanup _PyThreadState_DeleteExcept() code: rename "garbage" to

 "list".

BPO-19466 obtained from:

https://github.com/python/cpython/commit/9ad58acbe8b90b4d0f2d2e139e38bb5aa32b7fb6

From: Victor Stinner <vstinner@python.org>

Date: Sat, 1 Feb 2020 02:30:25 +0100

Subject: [PATCH] bpo-39511: PyThreadState_Clear() calls on_delete (GH-18296)

PyThreadState.on_delete is a callback used to notify Python when a

thread completes. _thread._set_sentinel() function creates a lock

which is released when the thread completes. It sets on_delete

callback to the internal release_sentinel() function. This lock is

known as Threading._tstate_lock in the threading module.

The release_sentinel() function uses the Python C API. The problem is

that on_delete is called late in the Python finalization, when the C

API is no longer fully working.

The PyThreadState_Clear() function now calls the

PyThreadState.on_delete callback. Previously, that happened in

PyThreadState_Delete().

The release_sentinel() function is now called when the C API is still

fully working.

BPO-39511 obtained from:

https://github.com/python/cpython/commit/4d96b4635aeff1b8ad41d41422ce808ce0b971c8

--- Lib/test/test_threading.py.orig	2021-04-02 10:32:10 UTC

+++ Lib/test/test_threading.py

@@ -762,6 +762,51 @@ class ThreadTests(BaseTestCase):

                 # Daemon threads must never add it to _shutdown_locks.

                 self.assertNotIn(tstate_lock, threading._shutdown_locks)

+    def test_locals_at_exit(self):

+        # bpo-19466: thread locals must not be deleted before destructors

+        # are called

+        rc, out, err = assert_python_ok("-c", """if 1:

+            import threading

+

+            class Atexit:

+                def __del__(self):

+                    print("thread_dict.atexit = %r" % thread_dict.atexit)

+

+            thread_dict = threading.local()

+            thread_dict.atexit = "value"

+

+            atexit = Atexit()

+        """)

+        self.assertEqual(out.rstrip(), b"thread_dict.atexit = 'value'")

+

+    def test_warnings_at_exit(self):

+        # bpo-19466: try to call most destructors at Python shutdown before

+        # destroying Python thread states

+        filename = __file__

+        rc, out, err = assert_python_ok("-Wd", "-c", """if 1:

+            import time

+            import threading

+            from test import support

+

+            def open_sleep():

+                # a warning will be emitted when the open file will be

+                # destroyed (without being explicitly closed) while the daemon

+                # thread is destroyed

+                fileobj = open(%a, 'rb')

+                start_event.set()

+                time.sleep(support.LONG_TIMEOUT)

+

+            start_event = threading.Event()

+

+            thread = threading.Thread(target=open_sleep, daemon=True)

+            thread.start()

+

+            # wait until the thread started

+            start_event.wait()

+        """ % filename)

+        self.assertRegex(err.rstrip(),

+                         b"^sys:1: ResourceWarning: unclosed file ")

+

 class ThreadJoinOnShutdown(BaseTestCase):

--- Python/pylifecycle.c.orig	2021-04-02 10:32:10 UTC

+++ Python/pylifecycle.c

@@ -1196,6 +1196,16 @@ Py_FinalizeEx(void)

     runtime->initialized = 0;

     runtime->core_initialized = 0;

+    /* Destroy the state of all threads of the interpreter, except of the

+       current thread. In practice, only daemon threads should still be alive,

+       except if wait_for_thread_shutdown() has been cancelled by CTRL+C.

+       Clear frames of other threads to call objects destructors. Destructors

+       will be called in the current Python thread. Since

+       _PyRuntimeState_SetFinalizing() has been called, no other Python thread

+       can take the GIL at this point: if they try, they will exit

+       immediately. */

+    _PyThreadState_DeleteExcept(runtime, tstate);

+

     /* Flush sys.stdout and sys.stderr */

     if (flush_std_files() < 0) {

         status = -1;

--- Python/pystate.c.orig	2021-04-02 10:32:10 UTC

+++ Python/pystate.c

@@ -802,6 +802,10 @@ PyThreadState_Clear(PyThreadState *tstate)

     Py_CLEAR(tstate->async_gen_finalizer);

     Py_CLEAR(tstate->context);

+

+    if (tstate->on_delete != NULL) {

+        tstate->on_delete(tstate->on_delete_data);

+    }

 }

@@ -824,9 +828,7 @@ tstate_delete_common(_PyRuntimeState *runtime, PyThrea

     if (tstate->next)

         tstate->next->prev = tstate->prev;

     HEAD_UNLOCK(runtime);

-    if (tstate->on_delete != NULL) {

-        tstate->on_delete(tstate->on_delete_data);

-    }

+

     PyMem_RawFree(tstate);

 }

@@ -890,25 +892,30 @@ void

 _PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate)

 {

     PyInterpreterState *interp = tstate->interp;

-    PyThreadState *p, *next, *garbage;

+

     HEAD_LOCK(runtime);

     /* Remove all thread states, except tstate, from the linked list of

        thread states.  This will allow calling PyThreadState_Clear()

        without holding the lock. */

-    garbage = interp->tstate_head;

-    if (garbage == tstate)

-        garbage = tstate->next;

-    if (tstate->prev)

+    PyThreadState *list = interp->tstate_head;

+    if (list == tstate) {

+        list = tstate->next;

+    }

+    if (tstate->prev) {

         tstate->prev->next = tstate->next;

-    if (tstate->next)

+    }

+    if (tstate->next) {

         tstate->next->prev = tstate->prev;

+    }

     tstate->prev = tstate->next = NULL;

     interp->tstate_head = tstate;

     HEAD_UNLOCK(runtime);

+

     /* Clear and deallocate all stale thread states.  Even if this

        executes Python code, we should be safe since it executes

        in the current thread, not one of the stale threads. */

-    for (p = garbage; p; p = next) {

+    PyThreadState *p, *next;

+    for (p = list; p; p = next) {

         next = p->next;

         PyThreadState_Clear(p);

         PyMem_RawFree(p);