gracefully handle too many open files instead of crashing

When we hit "too many open files" (syscall.EMFILE), unwatch the last 256 files
(or 20% of the currently watched files, whichever is smaller) so that the
commands after that may use their own file descriptors and not have all file
descriptors sucked up by the watcher. In the future we may decide to gracefully
fallback to polling, but this error really only happens when somebody does
something crazy like watching their entire home directory and I'm not sure I
want to empower that.

Author: bokwoon95

wgo_cmd.go

@@ -17,6 +17,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 	"unicode/utf8"
 
@@ -408,8 +409,8 @@ func (wgoCmd *WgoCmd) Run() error {
 	defer watcher.Close()
 	for _, root := range wgoCmd.Roots {
 		if wgoCmd.PollDuration > 0 {
-			wgoCmd.Logger.Println("POLL", filepath.ToSlash(root))
 			go wgoCmd.pollDirectory(wgoCmd.ctx, root, events)
+			wgoCmd.Logger.Println("POLL", filepath.ToSlash(root))
 		} else {
 			wgoCmd.addDirsRecursively(watcher, root)
 		}
@@ -629,8 +630,14 @@ func (wgoCmd *WgoCmd) addDirsRecursively(watcher *fsnotify.Watcher, dir string)
 		normalizedDir := filepath.ToSlash(path)
 		_, isRoot := roots[path]
 		if isRoot {
+			err := watcher.Add(path)
+			if err != nil {
+				if errors.Is(err, syscall.EMFILE) {
+					return fs.SkipAll
+				}
+				return fs.SkipDir
+			}
 			wgoCmd.Logger.Println("WATCH", normalizedDir)
-			watcher.Add(path)
 			return nil
 		}
 		for _, root := range wgoCmd.Roots {
@@ -646,8 +653,23 @@ func (wgoCmd *WgoCmd) addDirsRecursively(watcher *fsnotify.Watcher, dir string)
 		}
 		for _, r := range wgoCmd.DirRegexps {
 			if r.MatchString(normalizedDir) {
+				err := watcher.Add(path)
+				if err != nil {
+					if errors.Is(err, syscall.EMFILE) {
+						watchList := watcher.WatchList()
+						unwatchCount := 256
+						if unwatchCount > len(watchList)/2 {
+							unwatchCount = int(0.2 * float64(len(watchList)))
+						}
+						for i := len(watchList) - 1; i >= unwatchCount; i-- {
+							watcher.Remove(watchList[i])
+						}
+						wgoCmd.Logger.Printf("ERROR too many open files (%d open files), not watching any more\n", len(watchList))
+						return fs.SkipAll
+					}
+					return fs.SkipDir
+				}
 				wgoCmd.Logger.Println("WATCH", normalizedDir)
-				watcher.Add(path)
 				return nil
 			}
 		}
@@ -659,8 +681,23 @@ func (wgoCmd *WgoCmd) addDirsRecursively(watcher *fsnotify.Watcher, dir string)
 		if strings.HasPrefix(name, ".") {
 			return filepath.SkipDir
 		}
+		err = watcher.Add(path)
+		if err != nil {
+			if errors.Is(err, syscall.EMFILE) {
+				watchList := watcher.WatchList()
+				unwatchCount := 256
+				if unwatchCount > len(watchList)/2 {
+					unwatchCount = int(0.2 * float64(len(watchList)))
+				}
+				for i := len(watchList) - 1; i >= unwatchCount; i-- {
+					watcher.Remove(watchList[i])
+				}
+				wgoCmd.Logger.Printf("ERROR too many open files (%d open files), not watching any more\n", len(watchList))
+				return fs.SkipAll
+			}
+			return fs.SkipDir
+		}
 		wgoCmd.Logger.Println("WATCH", normalizedDir)
-		watcher.Add(path)
 		return nil
 	})
 }