forked from ebhomengo/niki
add supervisor
This commit is contained in:
parent
74c1d223d1
commit
748dee60e2
|
|
@ -0,0 +1,350 @@
|
||||||
|
package supervisor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"git.gocasts.ir/ebhomengo/niki/logger"
|
||||||
|
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProcessFunc is a long-running process which listens on finishSignal
|
||||||
|
// It notifies the supervisor by terminate channel when it terminates
|
||||||
|
type ProcessFunc func(finishSignal context.Context, processName string, terminateChannel chan<- string) error
|
||||||
|
|
||||||
|
var noopProcessFunc = func(finishSignal context.Context, processName string, terminateChannel chan<- string) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Supervisor is responsible to manage long-running processes
|
||||||
|
// Supervisor is not for concurrent use and should be used as the main goroutine of application
|
||||||
|
type Supervisor struct {
|
||||||
|
logger *slog.Logger
|
||||||
|
lock *sync.Mutex
|
||||||
|
processes map[string]Process
|
||||||
|
shutdownSignal chan os.Signal
|
||||||
|
ctx context.Context
|
||||||
|
ctxCancel context.CancelFunc
|
||||||
|
shutdownTimeout time.Duration
|
||||||
|
// terminateChannel should be used to notify supervisor when a process terminates
|
||||||
|
terminateChannel chan string
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(shutdownTimeout time.Duration, l *slog.Logger) *Supervisor {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
if l == nil {
|
||||||
|
l = logger.L()
|
||||||
|
}
|
||||||
|
|
||||||
|
if shutdownTimeout == 0 {
|
||||||
|
shutdownTimeout = DefaultGracefulShutdownTimeout
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Supervisor{
|
||||||
|
lock: &sync.Mutex{},
|
||||||
|
logger: l.WithGroup(LogNSSupervisor),
|
||||||
|
processes: make(map[string]Process),
|
||||||
|
shutdownSignal: make(chan os.Signal, 1),
|
||||||
|
ctx: ctx,
|
||||||
|
ctxCancel: cancel,
|
||||||
|
shutdownTimeout: shutdownTimeout,
|
||||||
|
// TODO : how to set terminateChannel buffer?
|
||||||
|
terminateChannel: make(chan string, 10),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type Process struct {
|
||||||
|
name string
|
||||||
|
handler ProcessFunc
|
||||||
|
options ProcessOption
|
||||||
|
state ProcessState
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProcessState struct {
|
||||||
|
// RecoveredNum count number of time the process recovered
|
||||||
|
RecoveredNum int
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProcessOption struct {
|
||||||
|
Recover bool
|
||||||
|
RecoverInterval time.Duration
|
||||||
|
RecoverCount int
|
||||||
|
RetryCount int
|
||||||
|
RetryInterval time.Duration
|
||||||
|
IsFatal bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
ProcessRetryCount = 3
|
||||||
|
ProcessRetryInterval = 3 * time.Second
|
||||||
|
ProcessRecoverCount = 10
|
||||||
|
ProcessRecoverInterval = 2 * time.Second
|
||||||
|
DefaultGracefulShutdownTimeout = 5 * time.Second
|
||||||
|
LogNSSupervisor = "supervisor"
|
||||||
|
)
|
||||||
|
|
||||||
|
var defaultOptions = ProcessOption{
|
||||||
|
Recover: true,
|
||||||
|
RetryInterval: ProcessRetryInterval,
|
||||||
|
RecoverInterval: ProcessRecoverInterval,
|
||||||
|
RecoverCount: ProcessRecoverCount,
|
||||||
|
RetryCount: ProcessRetryCount,
|
||||||
|
IsFatal: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register registers a new process to supervisor
|
||||||
|
func (s *Supervisor) Register(name string, process ProcessFunc, options *ProcessOption) {
|
||||||
|
// TODO : don't allow any registration after Start is called using a mutex
|
||||||
|
|
||||||
|
s.warnIfNameAlreadyInUse(name)
|
||||||
|
|
||||||
|
// TODO : validate name
|
||||||
|
p := Process{
|
||||||
|
name: name,
|
||||||
|
handler: process,
|
||||||
|
options: defaultOptions,
|
||||||
|
state: ProcessState{RecoveredNum: 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
if options != nil {
|
||||||
|
p.options = *options
|
||||||
|
}
|
||||||
|
|
||||||
|
s.lock.Lock()
|
||||||
|
s.processes[name] = p
|
||||||
|
s.lock.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start spawns a new goroutine for each process
|
||||||
|
// Spawned goroutine is responsible to handle the panics and restart the process
|
||||||
|
func (s *Supervisor) Start() {
|
||||||
|
// TODO : is it viable to use a goroutine pool such as Ants ?
|
||||||
|
for name := range s.processes {
|
||||||
|
go s.executeProcessWithRetryPolicy(name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) executeProcessWithRetryPolicy(name string) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
s.logger.Error("recover from panic", slog.String("process_name", name), slog.Any("panic", r))
|
||||||
|
|
||||||
|
if s.isRecoverable(name) {
|
||||||
|
s.incRecover(name)
|
||||||
|
s.waitFoRecover(name)
|
||||||
|
s.logger.Info("restart the process", slog.String("process_name", name))
|
||||||
|
|
||||||
|
// spawn new goroutine to avoid heap/stack memory leak when the recover count is big
|
||||||
|
go s.executeProcessWithRetryPolicy(name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("don't try any more to restart the process", slog.String("process_name", name))
|
||||||
|
s.removeProcess(name)
|
||||||
|
|
||||||
|
if s.isFatal(name) {
|
||||||
|
s.logger.Error("can't recover important process. exit..", slog.String("process_name", name))
|
||||||
|
s.shutdownSignal <- os.Interrupt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for i := 1; i <= s.retryCount(name); i++ {
|
||||||
|
s.logger.Info("execute process", slog.String("process_name", name))
|
||||||
|
f := s.handler(name)
|
||||||
|
err := f(s.ctx, name, s.terminateChannel)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("failed to execute process", slog.String("process_name", name),
|
||||||
|
slog.Int("attempt", i), slog.String("error", err.Error()))
|
||||||
|
|
||||||
|
s.waitFoRetry(name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// don't expect handler return if it hasn't any error because it's long-running process
|
||||||
|
// it should return when receives shutdown signal
|
||||||
|
s.logger.Info("process terminates with no error", slog.String("process_name", name))
|
||||||
|
|
||||||
|
if s.isFatal(name) {
|
||||||
|
s.logger.Error("can't recover important process. exit..", slog.String("process_name", name))
|
||||||
|
s.shutdownSignal <- os.Interrupt
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("don't try any more to execute process", slog.String("process_name", name))
|
||||||
|
s.removeProcess(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitOnShutdownSignal wait to receive shutdown signal.
|
||||||
|
// WaitOnShutdownSignal should not be called in other goroutines except main goroutine of application
|
||||||
|
func (s *Supervisor) WaitOnShutdownSignal() {
|
||||||
|
// TODO : is it necessary to add os.Interrupt to supervisor config?
|
||||||
|
signal.Notify(s.shutdownSignal, os.Interrupt)
|
||||||
|
<-s.shutdownSignal
|
||||||
|
|
||||||
|
s.gracefulShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) gracefulShutdown() {
|
||||||
|
s.logger.Info("shutdown all processes gracefully")
|
||||||
|
|
||||||
|
s.logger.Info("notify all processes (goroutines) to finish their jobs", slog.Duration("shutdown_timeout", s.shutdownTimeout))
|
||||||
|
s.ctxCancel()
|
||||||
|
|
||||||
|
forceExitCtx, forceExitCancel := context.WithTimeout(context.Background(), s.shutdownTimeout)
|
||||||
|
defer forceExitCancel()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case name := <-s.terminateChannel:
|
||||||
|
s.logger.Info("process terminates gracefully", slog.String("process_name", name))
|
||||||
|
s.removeProcess(name)
|
||||||
|
|
||||||
|
case <-forceExitCtx.Done():
|
||||||
|
s.logger.Info("supervisor terminates its job.", slog.Int("number_of_unfinished_processes", len(s.processes)))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) removeProcess(name string) {
|
||||||
|
s.lock.Lock()
|
||||||
|
delete(s.processes, name)
|
||||||
|
s.lock.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) isRecoverable(name string) bool {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
|
||||||
|
v, ok := s.processes[name]
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("process doesn't exist", slog.String("process_name", name))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.options.Recover && v.state.RecoveredNum < v.options.RecoverCount {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) isFatal(name string) bool {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
|
||||||
|
v, ok := s.processes[name]
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("process doesn't exist", slog.String("process_name", name))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return v.options.IsFatal
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) incRecover(name string) {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
|
||||||
|
v, ok := s.processes[name]
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("process doesn't exist", slog.String("process_name", name))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
v.state.RecoveredNum++
|
||||||
|
s.processes[name] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) retryCount(name string) int {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
|
||||||
|
v, ok := s.processes[name]
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("process doesn't exist", slog.String("process_name", name))
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
return v.options.RetryCount
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) retryInterval(name string) time.Duration {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
|
||||||
|
v, ok := s.processes[name]
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("process doesn't exist", slog.String("process_name", name))
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
return v.options.RetryInterval
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) waitFoRecover(name string) {
|
||||||
|
s.lock.Lock()
|
||||||
|
|
||||||
|
v, ok := s.processes[name]
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("process doesn't exist", slog.String("process_name", name))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
t := v.options.RecoverInterval
|
||||||
|
|
||||||
|
// free lock before sleep
|
||||||
|
s.lock.Unlock()
|
||||||
|
|
||||||
|
time.Sleep(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) waitFoRetry(name string) {
|
||||||
|
s.lock.Lock()
|
||||||
|
|
||||||
|
v, ok := s.processes[name]
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("process doesn't exist", slog.String("process_name", name))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
t := v.options.RetryInterval
|
||||||
|
|
||||||
|
// free lock before sleep
|
||||||
|
s.lock.Unlock()
|
||||||
|
|
||||||
|
s.logger.Info("wait to retry execute process after sleep interval",
|
||||||
|
slog.String("process_name", name), slog.Duration("interval",
|
||||||
|
t))
|
||||||
|
|
||||||
|
time.Sleep(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) handler(name string) ProcessFunc {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
|
||||||
|
v, ok := s.processes[name]
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("process doesn't exist", slog.String("process_name", name))
|
||||||
|
return noopProcessFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
return v.handler
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) warnIfNameAlreadyInUse(name string) {
|
||||||
|
s.lock.Lock()
|
||||||
|
defer s.lock.Unlock()
|
||||||
|
|
||||||
|
if _, ok := s.processes[name]; ok {
|
||||||
|
s.logger.Warn("process name already in use", slog.String("process_name", name))
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue