merge 3.4-slp (Stackless python#81, python#85)

Author: Anselm Kruis

Stackless/changelog.txt

@@ -34,9 +34,8 @@ What's New in Stackless 3.X.X?
 - https://bitbucket.org/stackless-dev/stackless/issues/81
   Fix (crash) bugs during thread and interpreter shutdown.
   If a thread dies, Stackless now really kills tasklets with C-state. During
-  interpreter shutdown, Stackless also kills daemon threads, if they execute
-  Python code or switch tasklets. This change avoids crashes during a later
-  shutdown stage.
+  interpreter shutdown, Stackless also kills daemon threads, if they switch
+  tasklets. This change avoids crashes during a later shutdown stage.
 
 - https://bitbucket.org/stackless-dev/stackless/issues/92
   If you run Stackless with the option '-v' (or set the environment variable

Stackless/core/stacklesseval.c

@@ -516,9 +516,6 @@ void slp_kill_tasks_with_stacks(PyThreadState *target_ts)
                      */
                     count++;
                     kill_pending_current_main_and_watchdogs(ts);
-                    /* It helps to inactivate threads reliably */
-                    if (PyExc_TaskletExit)
-                        PyThreadState_SetAsyncExc(ts->thread_id, PyExc_TaskletExit);
                 }
             }
             /* We must not release the GIL while we might hold the HEAD-lock.

Stackless/test/test_interpreter_shutdown.py

@@ -2,7 +2,7 @@
 import sys
 import _thread as thread
 import stackless
-import codecs
+import builtins
 import os
 import time
 import collections
@@ -63,13 +63,14 @@ def __init__(self, out, checks, tasklets):
         self.checks = checks
         self.tasklets = tasklets
 
-        # register the detector in the codecs search path
-        # it will be cleared after clearing the thread states
-        # this way the detector runs very late
-        def dummy_codecs_search_function(name):
-            return None
-        codecs.register(dummy_codecs_search_function)
-        dummy_codecs_search_function.x = self
+        # In Py_Finalize() the PyImport_Cleanup() runs shortly after 
+        # slp_kill_tasks_with_stacks(NULL).
+        # As very first action of PyImport_Cleanup() the Python
+        # interpreter sets builtins._ to None.
+        # Therefore we can use this assignment to trigger the execution
+        # of this detector. At this point the interpreter is still almost fully
+        # intact.
+        builtins._ = self
 
     def __del__(self):
         if self.debug:

Stackless/unittests/test_defects.py

@@ -4,7 +4,11 @@
 import gc
 import sys
 import types
-import threading
+try:
+    import threading
+    withThreads = True
+except:
+    withThreads = False
 
 from stackless import _test_nostacklesscall as apply
 from support import StacklessTestCase, captured_stderr, require_one_thread
@@ -81,6 +85,7 @@ def TaskletFunc(self):
         hex(id(self.TaskletWithDelAndCollect(TaskletFunc)(self)))
         stackless.run()  # crash here
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_tasklet_dealloc_in_thread_shutdown(self):
         # Test for https://bitbucket.org/stackless-dev/stackless/issues/89
         def other_thread_main():
@@ -298,6 +303,7 @@ def func():
             """])
         self.assertEqual(rc, 42)
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_interthread_kill(self):
         # test for issue #87 https://bitbucket.org/stackless-dev/stackless/issues/87/
         import subprocess
@@ -358,6 +364,7 @@ def other_thread_main(self):
             """])
         self.assertEqual(rc, 42)
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_tasklet_end_with_wrong_recursion_level(self):
         # test for issue #91 https://bitbucket.org/stackless-dev/stackless/issues/91/
         """A test for issue #91, wrong recursion level after tasklet re-binding

Stackless/unittests/test_miscell.py

@@ -5,21 +5,21 @@
 import stackless
 import sys
 import traceback
-import contextlib
 import weakref
 import types
-import _thread as thread
+import contextlib
 import time
 import os
 from stackless import _test_nostacklesscall as apply
-
-from support import StacklessTestCase, AsTaskletTestCase, require_one_thread
 try:
+    import _thread as thread
     import threading
     withThreads = True
 except:
     withThreads = False
 
+from support import StacklessTestCase, AsTaskletTestCase, require_one_thread
+
 
 def is_soft():
     softswitch = stackless.enable_softswitch(0)
@@ -504,6 +504,7 @@ def task():
         self.assertFalse(t.alive)
         self.assertEqual(t.thread_id, stackless.current.thread_id)
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     @require_one_thread
     def test_kill_thread_without_main_tasklet(self):
         # this test depends on a race condition.
@@ -607,15 +608,19 @@ def other_thread_main():
             tlet.remove()
             tlet.kill()
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_kill_without_thread_state_nl0(self):
         return self._test_kill_without_thread_state(0, False)
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_kill_without_thread_state_nl1(self):
         return self._test_kill_without_thread_state(1, False)
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_kill_without_thread_state_blocked_nl0(self):
         return self._test_kill_without_thread_state(0, True)
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_kill_without_thread_state_blocked_nl1(self):
         return self._test_kill_without_thread_state(1, True)
 
@@ -1011,6 +1016,7 @@ def test_bind_args_not_runnable(self):
         self.assertFalse(t.scheduled)
         t.run()
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_unbind_main(self):
         self.skipUnlessSoftswitching()
 
@@ -1034,6 +1040,7 @@ def other_thread():
         t.join()
         self.assertTrue(done[0])
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_rebind_main(self):
         # rebind the main tasklet of a thread. This is highly discouraged,
         # because it will deadlock, if the thread is a non daemon threading.Thread.
@@ -1086,6 +1093,7 @@ def test():
         self.assertEqual(tlet.recursion_depth, 0)
         self.assertEqual(self.recursion_depth_in_test, 1)
 
+    @unittest.skipUnless(withThreads, "requires thread support")
     def test_unbind_fail_cstate_no_thread(self):
         # https://bitbucket.org/stackless-dev/stackless/issues/92
         loop = True

Stackless/unittests/test_shutdown.py

@@ -34,19 +34,24 @@
 
 from __future__ import print_function, absolute_import, division
 import sys
-import _thread as thread
 import stackless
-import codecs
+import builtins
 import os
 import time
 import collections
 import unittest
-import threading
 import subprocess
 from stackless import _test_nostacklesscall as apply
+try:
+    import _thread as thread
+    import threading
+    withThreads = True
+except:
+    withThreads = False
 from support import StacklessTestCase
 
 
+@unittest.skipUnless(withThreads, "requires thread support")
 class TestThreadShutdown(StacklessTestCase):
 
     def _test_thread_shutdown(self, func, expect_kill):
@@ -125,6 +130,8 @@ class TestInterpreterShutdown(unittest.TestCase):
 
     def _test_shutdown_with_thread(self, case, is_hard, allowed_errors=(), debug=False):
         # test for issue #81 https://bitbucket.org/stackless-dev/stackless/issues/81/
+        if not withThreads:
+            self.skipTest("requires thread support")
         script = __file__
         if script.endswith("pyc") or script.endswith("pyo"):
             script = script[:-1]
@@ -213,13 +220,14 @@ def __init__(self, out, checks, tasklets):
         self.checks = checks
         self.tasklets = tasklets
 
-        # register the detector in the codecs search path
-        # it will be cleared after clearing the thread states
-        # this way the detector runs very late
-        def dummy_codecs_search_function(name):
-            return None
-        codecs.register(dummy_codecs_search_function)
-        dummy_codecs_search_function.x = self
+        # In Py_Finalize() the PyImport_Cleanup() runs shortly after 
+        # slp_kill_tasks_with_stacks(NULL).
+        # As very first action of PyImport_Cleanup() the Python
+        # interpreter sets builtins._ to None.
+        # Therefore we can use this assignment to trigger the execution
+        # of this detector. At this point the interpreter is still almost fully
+        # intact.
+        builtins._ = self
 
     def __del__(self):
         if self.debug:
@@ -398,7 +406,7 @@ def interpreter_shutdown_test():
     test = Test()
     detector = Detector(test.out, test.checks, test.tasklets)
     assert sys.getrefcount(detector) - sys.getrefcount(object()) == 2
-    detector = None  # the last ref is now in the codecs search path
+    detector = None  # the last ref is now in bultins._
     thread.start_new_thread(test.other_thread_main, ())
     ready.acquire()  # Be sure the other thread is ready.
     # print("at end")

Stackless/unittests/test_thread.py

@@ -16,6 +16,8 @@
     withThreads = True
 except:
     withThreads = False
+    class threading(object):
+        Thread = object
 
 
 class SkipMixin(object):
@@ -278,6 +280,7 @@ def test_rebind_nontrivial(self):
             end()
 
 
+@unittest.skipUnless(withThreads, "requires thread support")
 class RemoteTaskletTests(SkipMixin, StacklessTestCase):
     ThreadClass = LingeringThread
 

Stackless/unittests/test_watchdog.py

@@ -440,6 +440,7 @@ def tearDown(self):
         except AssertionError:
             self.watchdog_list[0] = None
             self.watchdog_list[1:] = []
+            raise
         super(TestNewWatchdog, self).tearDown()
 
     def test_run_from_worker(self):
@@ -559,7 +560,7 @@ def runner_func():
 
         def task():
             while True:
-                for i in range(200):
+                for i in range(500):
                     i = i
                 stackless.schedule()
 
@@ -607,7 +608,6 @@ def task():
         stackless.run()
         self.assertEqual(self.done, 2)
 
-    @unittest.skip("issue #85, assertion violation")
     def test_watchdog_priority_soft(self):
         """Verify that outermost "real" watchdog gets awoken"""
         self._test_watchdog_priority(True)