fix dead lock

runtime/runtime.go

@@ -21,18 +21,22 @@ type (
 		executors map[string]cron.EntryID
 
 		ctx          context.Context
-		eventMu      sync.Mutex
+		cancelFunc   context.CancelFunc
+		eventMu      sync.RWMutex
 		eventBuffer  []helper.ContainerEvent
 		eventTrigger chan struct{}
 	}
 )
 
 func NewEventLoop(ctx context.Context, docker *helper.DockerHelper) (*EventLoop, error) {
+	loopCtx, cancelFunc := context.WithCancel(contextutil.WithThreadName(context.Background(), "event_loop"))
 	loop := &EventLoop{
-		docker:    docker,
-		cr:        cron.New(),
-		ctx:       contextutil.WithThreadName(context.Background(), "event_loop"),
-		executors: make(map[string]cron.EntryID),
+		docker:       docker,
+		cr:           cron.New(),
+		ctx:          loopCtx,
+		cancelFunc:   cancelFunc,
+		executors:    make(map[string]cron.EntryID),
+		eventTrigger: make(chan struct{}),
 	}
 
 	if err := loop.firstRun(ctx); err != nil {
@@ -46,6 +50,7 @@ func NewEventLoop(ctx context.Context, docker *helper.DockerHelper) (*EventLoop,
 }
 
 func (el *EventLoop) Close() error {
+	el.cancelFunc()
 	ctx := el.cr.Stop()
 	<-ctx.Done()
 	return nil
@@ -146,15 +151,33 @@ func (el *EventLoop) unregister(container helper.Container) {
 
 func (el *EventLoop) startEventLoop() {
 	for {
-		if len(el.eventBuffer) == 0 {
-			<-el.eventTrigger
+		el.eventMu.RLock()
+		eventBufferLen := len(el.eventBuffer)
+		el.eventMu.RUnlock()
+
+		if eventBufferLen == 0 {
+			select {
+			case <-el.eventTrigger:
+				{
+					slog.Debug("event loop waken up", "thread", contextutil.ThreadName(el.ctx))
+				}
+			case <-el.ctx.Done():
+				{
+					return
+				}
+			}
 		}
 
+		var ev helper.ContainerEvent
 		el.eventMu.Lock()
-		ev := el.eventBuffer[0]
-		el.eventBuffer = append([]helper.ContainerEvent{}, el.eventBuffer[1:]...)
+		if len(el.eventBuffer) == 0 {
+			el.eventMu.Unlock()
+			continue
+		}
+		ev, el.eventBuffer = el.eventBuffer[0], el.eventBuffer[1:]
 		el.eventMu.Unlock()
 
+		slog.Debug("processing docker event", "thread", contextutil.ThreadName(el.ctx))
 		el.process(ev)
 	}
 }