Shut down everything if any server exits before ^C/timer. Fixup log messages.

start.py

@@ -8,18 +8,37 @@ Keeps servers alive until ^C or 100K seconds elapse. Exits non-zero if
 any servers fail to start, or die before timer/^C.
 """
 
+import os
+import signal
 import sys
 import time
 
 sys.path.append('./test')
 import startservers
 
+
+MAX_RUNTIME = 100000
+
+
+class Alarm(Exception):
+    pass
+
+
 if not startservers.start():
     sys.exit(1)
 try:
-    time.sleep(100000)
+    time.sleep(1)
-except KeyboardInterrupt:
+    print("All servers are running. To stop, hit ^C or wait %d seconds." % MAX_RUNTIME)
-    pass
+
-if not startservers.check():
+    def handler(*args):
+        raise Alarm
+    signal.signal(signal.SIGALRM, handler)
+    signal.alarm(MAX_RUNTIME)
+    os.wait()
+
+    # If we reach here, a child died early. Log what died:
+    startservers.check()
     sys.exit(1)
-print "stopping servers."
+except KeyboardInterrupt, Alarm:
+    signal.alarm(0)
+    print "\nstopping servers."

test/startservers.py

@@ -15,16 +15,15 @@ tempdir = tempfile.mkdtemp()
 
 def run(path):
     binary = os.path.join(tempdir, os.path.basename(path))
-    cmd = 'GORACE="halt_on_error=1" go build -race -o %s ./%s' % (binary, path)
+
-    print(cmd)
+    buildcmd = 'GORACE="halt_on_error=1" go build -race -o %s ./%s' % (binary, path)
-    subprocess.check_call(cmd, shell=True)
+    print(buildcmd)
-    def _ignore_sigint():
+    subprocess.check_call(buildcmd, shell=True)
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
+
-    p = subprocess.Popen(
+    srvcmd = [binary, '--config', config]
-        [binary, '--config', config],
+    p = subprocess.Popen(srvcmd)
-        preexec_fn=_ignore_sigint)
+    p.cmd = srvcmd
-    p.cmd = cmd
+    print('started %s with pid %d' % (p.cmd, p.pid))
-    print('started %s with pid %d' % (binary, p.pid))
     return p
 
 
@@ -48,7 +47,10 @@ def start():
         except Exception as e:
             print(e)
             return False
-    return check()
+        if not check():
+            # Don't keep building stuff if a server has already died.
+            return False
+    return True
 
 
 def check():
@@ -64,11 +66,10 @@ def check():
             stillok.append(p)
         else:
             busted.append(p)
-    print "\n%d servers are running." % len(stillok)
     if busted:
-        print "\n\nThese processes didn't start up successfully (check above for their output):"
+        print "\n\nThese processes exited early (check above for their output):"
         for p in busted:
-            print "\t'%s' exited %d" % (p.cmd, p.returncode)
+            print "\t'%s' with pid %d exited %d" % (p.cmd, p.pid, p.returncode)
     processes = stillok
     return not busted
 
@@ -76,5 +77,6 @@ def check():
 @atexit.register
 def stop():
     for p in processes:
-        p.kill()
+        if p.poll() is None:
+            p.kill()
     shutil.rmtree(tempdir)