diff --git a/README.md b/README.md index 4c1433f..3c5f953 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,8 @@ of systemd, you can work around this by logging directly to journald, for exampl by using [go-systemd/journal](https://godoc.org/github.com/coreos/go-systemd/journal) and looking for the [$JOURNAL_STREAM](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#$JOURNAL_STREAM) environment variable. + +## Integration with `supervisord` + +See the [supervisord example](examples/supervisord) for a process proxy and +configuration that keep supervisord attached across tableflip upgrades. diff --git a/examples/supervisord/README.md b/examples/supervisord/README.md new file mode 100644 index 0000000..1a92688 --- /dev/null +++ b/examples/supervisord/README.md @@ -0,0 +1,52 @@ +# Running tableflip under supervisord + +A successful tableflip upgrade replaces the process that supervisord started. The replacement keeps serving, but +supervisord sees its original child exit. If automatic restarts are enabled, it then starts a second copy of the +service. + +The `tableflip-supervisor` proxy in this directory remains attached to supervisord while the service PID changes. It +reads the PID file written by `tableflip.Upgrader.Ready`, forwards signals to the current service process, and exits if +that process stops without handing off to a replacement. + +Supervisor also ships a program named [`pidproxy`](https://supervisord.org/subprocess.html#pidproxy-program). Its +[main loop exits](https://github.com/Supervisor/supervisor/blob/main/supervisor/pidproxy.py) when the command it +started exits. Because a tableflip upgrade deliberately exits that process, the standard proxy does not remain +attached after the first upgrade. + +## Build the proxy + +From the repository root: + +```sh +go build -o /usr/local/bin/tableflip-supervisor ./examples/supervisord +``` + +Your service must pass the same PID-file path to tableflip: + +```go +upg, err := tableflip.New(tableflip.Options{ + PIDFile: "/run/myservice.pid", +}) +``` + +## Configure supervisord + +Copy [`supervisord.conf`](supervisord.conf) into your Supervisor configuration and replace the example paths and user. +The proxy syntax is: + +```text +tableflip-supervisor PID_FILE -- COMMAND [ARG...] +``` + +The PID-file directory must be writable only by the account that runs the service. The proxy trusts the PID stored in +that file when it forwards signals. + +After deploying a new binary, request a tableflip upgrade through the proxy: + +```sh +supervisorctl signal HUP myservice +``` + +Do not use `supervisorctl restart` for a tableflip upgrade; it stops the running process before starting the new one. +Regular `supervisorctl stop` and `start` commands continue to work. If the service does not stop before +`stopwaitsecs`, `killasgroup=true` lets supervisord terminate the proxy and any remaining tableflip processes. diff --git a/examples/supervisord/main.go b/examples/supervisord/main.go new file mode 100644 index 0000000..764d990 --- /dev/null +++ b/examples/supervisord/main.go @@ -0,0 +1,168 @@ +//go:build !windows +// +build !windows + +// tableflip-supervisor keeps supervisord attached to a service across +// tableflip upgrades and forwards signals to the process in its PID file. +package main + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "os/signal" + "strconv" + "strings" + "syscall" + "time" +) + +const defaultPollInterval = 100 * time.Millisecond + +func main() { + log.SetFlags(0) + + pidFile, command, err := parseArgs(os.Args[1:]) + if err != nil { + log.Fatal(err) + } + + signals := make(chan os.Signal, 1) + signal.Notify(signals, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGQUIT, + syscall.SIGTERM, + syscall.SIGUSR1, + syscall.SIGUSR2, + ) + defer signal.Stop(signals) + + if err := runProxy(pidFile, command, signals, defaultPollInterval); err != nil { + log.Fatal(err) + } +} + +func parseArgs(args []string) (string, []string, error) { + if len(args) < 3 || args[1] != "--" { + return "", nil, errors.New("usage: tableflip-supervisor PID_FILE -- COMMAND [ARG...]") + } + return args[0], args[2:], nil +} + +func runProxy(pidFile string, command []string, signals <-chan os.Signal, pollInterval time.Duration) error { + if err := os.Remove(pidFile); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove stale PID file: %v", err) + } + + cmd := exec.Command(command[0], command[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + return fmt.Errorf("start %s: %v", command[0], err) + } + + directPID := cmd.Process.Pid + activePID := directPID + ready := false + stopping := false + directExit := make(chan error, 1) + go func() { + directExit <- cmd.Wait() + }() + + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + select { + case sig := <-signals: + pid, err := livePID(pidFile) + if err == nil { + activePID = pid + ready = true + } + if err := signalProcess(activePID, sig); err != nil { + return fmt.Errorf("forward %s to PID %d: %v", sig, activePID, err) + } + if isStopSignal(sig) { + stopping = true + } + + case err := <-directExit: + directExit = nil + pid, pidErr := livePID(pidFile) + if pidErr == nil && pid != directPID { + activePID = pid + ready = true + continue + } + if stopping { + return nil + } + if err == nil { + return errors.New("managed command exited before handing off to a replacement") + } + return fmt.Errorf("managed command exited: %v", err) + + case <-ticker.C: + pid, err := livePID(pidFile) + if err == nil { + activePID = pid + ready = true + continue + } + + if ready && !processAlive(activePID) { + if stopping { + return nil + } + return fmt.Errorf("managed process %d exited without handing off to a replacement", activePID) + } + } + } +} + +func readPID(path string) (int, error) { + contents, err := ioutil.ReadFile(path) + if err != nil { + return 0, err + } + + pid, err := strconv.Atoi(strings.TrimSpace(string(contents))) + if err != nil || pid <= 0 { + return 0, fmt.Errorf("invalid PID in %s", path) + } + return pid, nil +} + +func livePID(path string) (int, error) { + pid, err := readPID(path) + if err != nil { + return 0, err + } + if !processAlive(pid) { + return 0, fmt.Errorf("process %d is not running", pid) + } + return pid, nil +} + +func processAlive(pid int) bool { + err := syscall.Kill(pid, 0) + return err == nil || err == syscall.EPERM +} + +func signalProcess(pid int, sig os.Signal) error { + syscallSignal, ok := sig.(syscall.Signal) + if !ok { + return fmt.Errorf("unsupported signal %v", sig) + } + return syscall.Kill(pid, syscallSignal) +} + +func isStopSignal(sig os.Signal) bool { + return sig == syscall.SIGINT || sig == syscall.SIGQUIT || sig == syscall.SIGTERM +} diff --git a/examples/supervisord/main_test.go b/examples/supervisord/main_test.go new file mode 100644 index 0000000..7880e50 --- /dev/null +++ b/examples/supervisord/main_test.go @@ -0,0 +1,157 @@ +//go:build !windows +// +build !windows + +package main + +import ( + "io/ioutil" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "syscall" + "testing" + "time" +) + +func TestParseArgs(t *testing.T) { + pidFile, command, err := parseArgs([]string{"/run/service.pid", "--", "/usr/bin/service", "-flag"}) + if err != nil { + t.Fatal(err) + } + if pidFile != "/run/service.pid" { + t.Fatalf("unexpected PID file %q", pidFile) + } + if len(command) != 2 || command[0] != "/usr/bin/service" || command[1] != "-flag" { + t.Fatalf("unexpected command %q", command) + } + + if _, _, err := parseArgs([]string{"service.pid", "/usr/bin/service"}); err == nil { + t.Fatal("expected invalid arguments to fail") + } +} + +func TestProxyTracksReplacementProcess(t *testing.T) { + if os.Getenv("TABLEFLIP_PROXY_HELPER") != "" { + runHelperProcess() + return + } + + tempDir, err := ioutil.TempDir("", "tableflip-supervisor-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + pidFile := filepath.Join(tempDir, "service.pid") + if err := os.Setenv("TABLEFLIP_PROXY_HELPER", "parent"); err != nil { + t.Fatal(err) + } + defer os.Unsetenv("TABLEFLIP_PROXY_HELPER") + if err := os.Setenv("TABLEFLIP_PROXY_PID_FILE", pidFile); err != nil { + t.Fatal(err) + } + defer os.Unsetenv("TABLEFLIP_PROXY_PID_FILE") + + signals := make(chan os.Signal, 1) + result := make(chan error, 1) + go func() { + result <- runProxy(pidFile, []string{os.Args[0], "-test.run=TestProxyTracksReplacementProcess"}, signals, time.Millisecond) + }() + + firstPID := waitForPID(t, pidFile, 0) + secondPID := 0 + defer func() { + syscall.Kill(firstPID, syscall.SIGKILL) + if secondPID != 0 { + syscall.Kill(secondPID, syscall.SIGKILL) + } + }() + signals <- syscall.SIGHUP + secondPID = waitForPID(t, pidFile, firstPID) + if firstPID == secondPID { + t.Fatal("upgrade did not replace the managed process") + } + + select { + case err := <-result: + t.Fatalf("proxy exited during a successful handoff: %v", err) + case <-time.After(50 * time.Millisecond): + } + + signals <- syscall.SIGTERM + select { + case err := <-result: + if err != nil { + t.Fatalf("proxy returned an error while stopping: %v", err) + } + case <-time.After(5 * time.Second): + syscall.Kill(secondPID, syscall.SIGKILL) + t.Fatal("proxy did not exit after the managed process stopped") + } +} + +func waitForPID(t *testing.T, path string, previous int) int { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + pid, err := readPID(path) + if err == nil && pid != previous { + return pid + } + time.Sleep(time.Millisecond) + } + t.Fatalf("PID file %s was not updated", path) + return 0 +} + +func runHelperProcess() { + pidFile := os.Getenv("TABLEFLIP_PROXY_PID_FILE") + if err := ioutil.WriteFile(pidFile, []byte(strconv.Itoa(os.Getpid())), 0600); err != nil { + os.Exit(2) + } + + signals := make(chan os.Signal, 1) + role := os.Getenv("TABLEFLIP_PROXY_HELPER") + if role == "parent" { + signal.Notify(signals, syscall.SIGHUP) + <-signals + + cmd := exec.Command(os.Args[0], "-test.run=TestProxyTracksReplacementProcess") + cmd.Env = replaceEnv(os.Environ(), "TABLEFLIP_PROXY_HELPER", "replacement") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + os.Exit(3) + } + waitForReplacementPID(pidFile, os.Getpid()) + os.Exit(0) + } + + signal.Notify(signals, syscall.SIGTERM) + <-signals + os.Exit(0) +} + +func waitForReplacementPID(path string, previous int) { + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + pid, err := readPID(path) + if err == nil && pid != previous { + return + } + time.Sleep(time.Millisecond) + } + os.Exit(4) +} + +func replaceEnv(environ []string, name, value string) []string { + prefix := name + "=" + for i, entry := range environ { + if len(entry) >= len(prefix) && entry[:len(prefix)] == prefix { + environ[i] = prefix + value + return environ + } + } + return append(environ, prefix+value) +} diff --git a/examples/supervisord/supervisord.conf b/examples/supervisord/supervisord.conf new file mode 100644 index 0000000..d6e1755 --- /dev/null +++ b/examples/supervisord/supervisord.conf @@ -0,0 +1,13 @@ +[program:myservice] +command=/usr/local/bin/tableflip-supervisor /run/myservice.pid -- /usr/local/bin/myservice -pid-file /run/myservice.pid +user=myservice +directory=/var/lib/myservice +autostart=true +autorestart=true +startsecs=5 +startretries=3 +stopsignal=TERM +stopwaitsecs=30 +killasgroup=true +redirect_stderr=true +stdout_logfile=/var/log/myservice.log