Add supervisord integration example

README.md

@@ -67,3 +67,8 @@ of systemd, you can work around this by logging directly to journald, for exampl
 by using [go-systemd/journal](https://godoc.org/github.com/coreos/go-systemd/journal)
 and looking for the [$JOURNAL_STREAM](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#$JOURNAL_STREAM)
 environment variable.
+
+## Integration with `supervisord`
+
+See the [supervisord example](examples/supervisord) for a process proxy and
+configuration that keep supervisord attached across tableflip upgrades.

examples/supervisord/README.md

@@ -0,0 +1,52 @@
+# Running tableflip under supervisord
+
+A successful tableflip upgrade replaces the process that supervisord started. The replacement keeps serving, but
+supervisord sees its original child exit. If automatic restarts are enabled, it then starts a second copy of the
+service.
+
+The `tableflip-supervisor` proxy in this directory remains attached to supervisord while the service PID changes. It
+reads the PID file written by `tableflip.Upgrader.Ready`, forwards signals to the current service process, and exits if
+that process stops without handing off to a replacement.
+
+Supervisor also ships a program named [`pidproxy`](https://supervisord.org/subprocess.html#pidproxy-program). Its
+[main loop exits](https://github.com/Supervisor/supervisor/blob/main/supervisor/pidproxy.py) when the command it
+started exits. Because a tableflip upgrade deliberately exits that process, the standard proxy does not remain
+attached after the first upgrade.
+
+## Build the proxy
+
+From the repository root:
+
+```sh
+go build -o /usr/local/bin/tableflip-supervisor ./examples/supervisord
+```
+
+Your service must pass the same PID-file path to tableflip:
+
+```go
+upg, err := tableflip.New(tableflip.Options{
+	PIDFile: "/run/myservice.pid",
+})
+```
+
+## Configure supervisord
+
+Copy [`supervisord.conf`](supervisord.conf) into your Supervisor configuration and replace the example paths and user.
+The proxy syntax is:
+
+```text
+tableflip-supervisor PID_FILE -- COMMAND [ARG...]
+```
+
+The PID-file directory must be writable only by the account that runs the service. The proxy trusts the PID stored in
+that file when it forwards signals.
+
+After deploying a new binary, request a tableflip upgrade through the proxy:
+
+```sh
+supervisorctl signal HUP myservice
+```
+
+Do not use `supervisorctl restart` for a tableflip upgrade; it stops the running process before starting the new one.
+Regular `supervisorctl stop` and `start` commands continue to work. If the service does not stop before
+`stopwaitsecs`, `killasgroup=true` lets supervisord terminate the proxy and any remaining tableflip processes.

examples/supervisord/main.go

@@ -0,0 +1,168 @@
+//go:build !windows
+// +build !windows
+
+// tableflip-supervisor keeps supervisord attached to a service across
+// tableflip upgrades and forwards signals to the process in its PID file.
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+)
+
+const defaultPollInterval = 100 * time.Millisecond
+
+func main() {
+	log.SetFlags(0)
+
+	pidFile, command, err := parseArgs(os.Args[1:])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals,
+		syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGQUIT,
+		syscall.SIGTERM,
+		syscall.SIGUSR1,
+		syscall.SIGUSR2,
+	)
+	defer signal.Stop(signals)
+
+	if err := runProxy(pidFile, command, signals, defaultPollInterval); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func parseArgs(args []string) (string, []string, error) {
+	if len(args) < 3 || args[1] != "--" {
+		return "", nil, errors.New("usage: tableflip-supervisor PID_FILE -- COMMAND [ARG...]")
+	}
+	return args[0], args[2:], nil
+}
+
+func runProxy(pidFile string, command []string, signals <-chan os.Signal, pollInterval time.Duration) error {
+	if err := os.Remove(pidFile); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("remove stale PID file: %v", err)
+	}
+
+	cmd := exec.Command(command[0], command[1:]...)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("start %s: %v", command[0], err)
+	}
+
+	directPID := cmd.Process.Pid
+	activePID := directPID
+	ready := false
+	stopping := false
+	directExit := make(chan error, 1)
+	go func() {
+		directExit <- cmd.Wait()
+	}()
+
+	ticker := time.NewTicker(pollInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case sig := <-signals:
+			pid, err := livePID(pidFile)
+			if err == nil {
+				activePID = pid
+				ready = true
+			}
+			if err := signalProcess(activePID, sig); err != nil {
+				return fmt.Errorf("forward %s to PID %d: %v", sig, activePID, err)
+			}
+			if isStopSignal(sig) {
+				stopping = true
+			}
+
+		case err := <-directExit:
+			directExit = nil
+			pid, pidErr := livePID(pidFile)
+			if pidErr == nil && pid != directPID {
+				activePID = pid
+				ready = true
+				continue
+			}
+			if stopping {
+				return nil
+			}
+			if err == nil {
+				return errors.New("managed command exited before handing off to a replacement")
+			}
+			return fmt.Errorf("managed command exited: %v", err)
+
+		case <-ticker.C:
+			pid, err := livePID(pidFile)
+			if err == nil {
+				activePID = pid
+				ready = true
+				continue
+			}
+
+			if ready && !processAlive(activePID) {
+				if stopping {
+					return nil
+				}
+				return fmt.Errorf("managed process %d exited without handing off to a replacement", activePID)
+			}
+		}
+	}
+}
+
+func readPID(path string) (int, error) {
+	contents, err := ioutil.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+
+	pid, err := strconv.Atoi(strings.TrimSpace(string(contents)))
+	if err != nil || pid <= 0 {
+		return 0, fmt.Errorf("invalid PID in %s", path)
+	}
+	return pid, nil
+}
+
+func livePID(path string) (int, error) {
+	pid, err := readPID(path)
+	if err != nil {
+		return 0, err
+	}
+	if !processAlive(pid) {
+		return 0, fmt.Errorf("process %d is not running", pid)
+	}
+	return pid, nil
+}
+
+func processAlive(pid int) bool {
+	err := syscall.Kill(pid, 0)
+	return err == nil || err == syscall.EPERM
+}
+
+func signalProcess(pid int, sig os.Signal) error {
+	syscallSignal, ok := sig.(syscall.Signal)
+	if !ok {
+		return fmt.Errorf("unsupported signal %v", sig)
+	}
+	return syscall.Kill(pid, syscallSignal)
+}
+
+func isStopSignal(sig os.Signal) bool {
+	return sig == syscall.SIGINT || sig == syscall.SIGQUIT || sig == syscall.SIGTERM
+}

examples/supervisord/main_test.go

@@ -0,0 +1,157 @@
+//go:build !windows
+// +build !windows
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"syscall"
+	"testing"
+	"time"
+)
+
+func TestParseArgs(t *testing.T) {
+	pidFile, command, err := parseArgs([]string{"/run/service.pid", "--", "/usr/bin/service", "-flag"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pidFile != "/run/service.pid" {
+		t.Fatalf("unexpected PID file %q", pidFile)
+	}
+	if len(command) != 2 || command[0] != "/usr/bin/service" || command[1] != "-flag" {
+		t.Fatalf("unexpected command %q", command)
+	}
+
+	if _, _, err := parseArgs([]string{"service.pid", "/usr/bin/service"}); err == nil {
+		t.Fatal("expected invalid arguments to fail")
+	}
+}
+
+func TestProxyTracksReplacementProcess(t *testing.T) {
+	if os.Getenv("TABLEFLIP_PROXY_HELPER") != "" {
+		runHelperProcess()
+		return
+	}
+
+	tempDir, err := ioutil.TempDir("", "tableflip-supervisor-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempDir)
+	pidFile := filepath.Join(tempDir, "service.pid")
+	if err := os.Setenv("TABLEFLIP_PROXY_HELPER", "parent"); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Unsetenv("TABLEFLIP_PROXY_HELPER")
+	if err := os.Setenv("TABLEFLIP_PROXY_PID_FILE", pidFile); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Unsetenv("TABLEFLIP_PROXY_PID_FILE")
+
+	signals := make(chan os.Signal, 1)
+	result := make(chan error, 1)
+	go func() {
+		result <- runProxy(pidFile, []string{os.Args[0], "-test.run=TestProxyTracksReplacementProcess"}, signals, time.Millisecond)
+	}()
+
+	firstPID := waitForPID(t, pidFile, 0)
+	secondPID := 0
+	defer func() {
+		syscall.Kill(firstPID, syscall.SIGKILL)
+		if secondPID != 0 {
+			syscall.Kill(secondPID, syscall.SIGKILL)
+		}
+	}()
+	signals <- syscall.SIGHUP
+	secondPID = waitForPID(t, pidFile, firstPID)
+	if firstPID == secondPID {
+		t.Fatal("upgrade did not replace the managed process")
+	}
+
+	select {
+	case err := <-result:
+		t.Fatalf("proxy exited during a successful handoff: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	signals <- syscall.SIGTERM
+	select {
+	case err := <-result:
+		if err != nil {
+			t.Fatalf("proxy returned an error while stopping: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		syscall.Kill(secondPID, syscall.SIGKILL)
+		t.Fatal("proxy did not exit after the managed process stopped")
+	}
+}
+
+func waitForPID(t *testing.T, path string, previous int) int {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		pid, err := readPID(path)
+		if err == nil && pid != previous {
+			return pid
+		}
+		time.Sleep(time.Millisecond)
+	}
+	t.Fatalf("PID file %s was not updated", path)
+	return 0
+}
+
+func runHelperProcess() {
+	pidFile := os.Getenv("TABLEFLIP_PROXY_PID_FILE")
+	if err := ioutil.WriteFile(pidFile, []byte(strconv.Itoa(os.Getpid())), 0600); err != nil {
+		os.Exit(2)
+	}
+
+	signals := make(chan os.Signal, 1)
+	role := os.Getenv("TABLEFLIP_PROXY_HELPER")
+	if role == "parent" {
+		signal.Notify(signals, syscall.SIGHUP)
+		<-signals
+
+		cmd := exec.Command(os.Args[0], "-test.run=TestProxyTracksReplacementProcess")
+		cmd.Env = replaceEnv(os.Environ(), "TABLEFLIP_PROXY_HELPER", "replacement")
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Start(); err != nil {
+			os.Exit(3)
+		}
+		waitForReplacementPID(pidFile, os.Getpid())
+		os.Exit(0)
+	}
+
+	signal.Notify(signals, syscall.SIGTERM)
+	<-signals
+	os.Exit(0)
+}
+
+func waitForReplacementPID(path string, previous int) {
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		pid, err := readPID(path)
+		if err == nil && pid != previous {
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+	os.Exit(4)
+}
+
+func replaceEnv(environ []string, name, value string) []string {
+	prefix := name + "="
+	for i, entry := range environ {
+		if len(entry) >= len(prefix) && entry[:len(prefix)] == prefix {
+			environ[i] = prefix + value
+			return environ
+		}
+	}
+	return append(environ, prefix+value)
+}