Add monit-based monitoring of build manager

Should catch when the build manager freezes and restart it

conf/init/service/buildmanager/run

@@ -2,7 +2,11 @@
 
 echo 'Starting internal build manager'
 
+# Run monit to ensure the build manager is restarted if/when it locks up.
+monit
+
+# Run the build manager.
 cd /
-TROLLIUSDEBUG=1 venv/bin/python -m buildman.builder 2>&1
+exec TROLLIUSDEBUG=1 venv/bin/python -m buildman.builder 2>&1
 
 echo 'Internal build manager exited'

conf/monitrc

@@ -0,0 +1,17 @@
+set daemon 10 with start delay 30
+
+set httpd port 2812 and
+    use address localhost  # only accept connection from localhost
+    allow localhost        # allow localhost to connect to the server and
+    allow admin:monit      # require user 'admin' with password 'monit'
+
+check host buildmanager with address localhost
+    start program = "/usr/bin/sv start /etc/service/buildmanager" with timeout 10 seconds
+    stop program  = "/usr/bin/sv kill /etc/service/buildmanager"
+    if failed port 8787 protocol http
+      request "/"
+      status = 405
+      with timeout 3 seconds
+      for 3 cycles
+      then restart
+    if 10 restarts within 10 cycles then timeout