syncdaemon: force termination for unhandled exception in any thread

Signed-off-by: Csaba Henk <csaba@gluster.com>
Signed-off-by: Anand Avati <avati@gluster.com>

BUG: 2736 (gsyncd hangs if crash occurs in the non-main thread)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2736

5 files changed, 27 insertions, 13 deletions

diff --git a/xlators/features/marker/utils/syncdaemon/gsyncd.py b/xlators/features/marker/utils/syncdaemon/gsyncd.py
index 140a8283d6c..46819aa6562 100644
--- a/xlators/features/marker/utils/syncdaemon/gsyncd.py
+++ b/xlators/features/marker/utils/syncdaemon/gsyncd.py
@@ -159,9 +159,7 @@ def finalize(*a):
     sys.stderr.flush()
 
 def main():
-    # ??? "finally" clause does not take effect with SIGTERM...
-    # but this handler neither does
-    # signal.signal(signal.SIGTERM, finalize)
+    signal.signal(signal.SIGTERM, lambda *a: (finalize(*a), os._exit(1)))
     GLogger.setup()
     exval = 0
     try:
diff --git a/xlators/features/marker/utils/syncdaemon/master.py b/xlators/features/marker/utils/syncdaemon/master.py
index 02cf1be5b05..35dc4ee06aa 100644
--- a/xlators/features/marker/utils/syncdaemon/master.py
+++ b/xlators/features/marker/utils/syncdaemon/master.py
@@ -6,10 +6,10 @@ import signal
 import logging
 import errno
 from errno import ENOENT, ENODATA
-from threading import Thread, currentThread, Condition, Lock
+from threading import currentThread, Condition, Lock
 
 from gconf import gconf
-from syncdutils import FreeObject
+from syncdutils import FreeObject, Thread
 
 URXTIME = (-1, 0)
 
@@ -108,7 +108,6 @@ class GMaster(object):
                     self.slave.server.keep_alive(vi)
                     time.sleep(gap)
             t = Thread(target=keep_alive)
-            t.setDaemon(True)
             t.start()
         while not self.terminate:
             self.crawl()
@@ -342,7 +341,6 @@ class Syncer(object):
         self.pb = PostBox()
         for i in range(int(gconf.sync_jobs)):
             t = Thread(target=self.syncjob)
-            t.setDaemon = True
             t.start()
 
     def syncjob(self):
diff --git a/xlators/features/marker/utils/syncdaemon/repce.py b/xlators/features/marker/utils/syncdaemon/repce.py
index 8b7e0ae94e2..47691301e29 100644
--- a/xlators/features/marker/utils/syncdaemon/repce.py
+++ b/xlators/features/marker/utils/syncdaemon/repce.py
@@ -3,7 +3,7 @@ import sys
 import select
 import time
 import logging
-from threading import Thread, Condition
+from threading import Condition
 try:
     import thread
 except ImportError:
@@ -20,6 +20,8 @@ except ImportError:
     # py 3
     import pickle
 
+from syncdutils import Thread
+
 pickle_proto = -1
 repce_version = 1.0
 
@@ -51,7 +53,6 @@ class RepceServer(object):
     def service_loop(self):
         for i in range(self.wnum):
             t = Thread(target=self.worker)
-            t.setDaemon(True)
             t.start()
         try:
             while True:
@@ -109,7 +110,6 @@ class RepceClient(object):
         self.inf, self.out = ioparse(i, o)
         self.jtab = {}
         t = Thread(target = self.listen)
-        t.setDaemon(True)
         t.start()
 
     def listen(self):
diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/xlators/features/marker/utils/syncdaemon/resource.py
index 9bf1e441dc6..ce855fb715d 100644
--- a/xlators/features/marker/utils/syncdaemon/resource.py
+++ b/xlators/features/marker/utils/syncdaemon/resource.py
@@ -10,7 +10,6 @@ import select
 import socket
 import logging
 import tempfile
-import threading
 from ctypes import *
 from ctypes.util import find_library
 from errno import EEXIST, ENOENT, ENODATA, ENOTDIR, ELOOP, EISDIR
@@ -235,8 +234,7 @@ class SlaveLocal(object):
 
     def service_loop(self):
         repce = RepceServer(self.server, sys.stdin, sys.stdout, int(gconf.sync_jobs))
-        t = threading.Thread(target=repce.service_loop)
-        t.setDaemon(True)
+        t = syncdutils.Thread(target=repce.service_loop)
         t.start()
         logging.info("slave listening")
         if gconf.timeout and int(gconf.timeout) > 0:
diff --git a/xlators/features/marker/utils/syncdaemon/syncdutils.py b/xlators/features/marker/utils/syncdaemon/syncdutils.py
index d3ab2f34507..56bc515d49f 100644
--- a/xlators/features/marker/utils/syncdaemon/syncdutils.py
+++ b/xlators/features/marker/utils/syncdaemon/syncdutils.py
@@ -1,5 +1,8 @@
 import os
 import fcntl
+from threading import Thread as baseThread
+from signal import SIGTERM
+
 try:
     # py 3
     from urllib import parse as urllib
@@ -49,3 +52,20 @@ class FreeObject(object):
     def __init__(self, **kw):
         for k,v in kw.iteritems():
             setattr(self, k, v)
+
+class Thread(baseThread):
+
+    def __init__(self, *a, **kw):
+        tf = kw.get('target')
+        if tf:
+            def twrap(*aa):
+                try:
+                    tf(*aa)
+                except:
+                    try:
+                        raise
+                    finally:
+                        os.kill(os.getpid(), SIGTERM)
+            kw['target'] = twrap
+        baseThread.__init__(self, *a, **kw)
+        self.setDaemon(True)