diff --git a/cmd/watchdog/main.go b/cmd/watchdog/main.go index 8f3db2e..9799e61 100755 --- a/cmd/watchdog/main.go +++ b/cmd/watchdog/main.go @@ -7,11 +7,18 @@ import ( "time" "github.com/go-ping/ping" - _ "github.com/go-ping/ping" "github.com/tursom/esxi-monitor/vmomi" ) +type ( + watchdog struct { + addr string + vm *vmomi.VirtualMachine + life int + } +) + func main() { cfg, err := parseConfig("config.yaml") if err != nil { @@ -34,7 +41,7 @@ func main() { } // ip -> vm instance - watches := make(map[string]*vmomi.VirtualMachine) + watches := make([]*watchdog, 0) for _, vm := range vms { target, contains := watchTargets[vm.Name()] if !contains { @@ -42,51 +49,76 @@ func main() { } fmt.Printf("watching virtual machine: addr=%s, name=%s\n", target.Addr, vm.Name()) - watches[target.Addr] = vm + watches = append(watches, &watchdog{ + addr: target.Addr, + vm: vm, + life: 3, + }) } doWatch(watches) } // watches map ip -> vm instance -func doWatch(watches map[string]*vmomi.VirtualMachine) { +func doWatch(watches []*watchdog) { t := time.NewTicker(time.Minute) for { <-t.C - for ip, vm := range watches { - pinger, err := ping.NewPinger(ip) + fmt.Println("in ping tick") + + for _, w := range watches { + pinger, err := ping.NewPinger(w.addr) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "ping to target failed: %s\n", err) continue } + pinger.Timeout = time.Second * 15 pinger.Count = 4 + fmt.Printf("start ping to %s\n", w.addr) err = pinger.Run() if err != nil { _, _ = fmt.Fprintf(os.Stderr, "ping to target failed: %s\n", err) continue } + fmt.Printf("finished ping to %s\n", w.addr) statistics := pinger.Statistics() - if statistics.PacketsRecv == 0 { - fmt.Printf("restarting vm %s(%s)", ip, vm.Name()) + if statistics.PacketsRecv != 0 { + w.life = 3 + continue + } - task, err := vm.Reset() - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "restart vm failed: %s\n", err) - continue - } + w.life-- + fmt.Printf("ping vm %s(%s) failed, left %d life", w.addr, w.vm.Name(), w.life) - ctx, _ := context.WithTimeout(context.Background(), time.Minute) - err = task.Wait(ctx) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "restart vm failed: %s\n", err) - continue - } + if w.life > 0 { + continue + } + + fmt.Printf("restarting vm %s(%s)", w.addr, w.vm.Name()) + + if err := reset(w.vm); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "restart vm failed: %s\n", err) } } } } + +func reset(vm *vmomi.VirtualMachine) error { + task, err := vm.Reset() + if err != nil { + return err + } + + ctx, _ := context.WithTimeout(context.Background(), time.Minute) + err = task.Wait(ctx) + if err != nil { + return err + } + + return nil +} diff --git a/cmd/watchdog/watchdog b/cmd/watchdog/watchdog new file mode 100755 index 0000000..2472358 Binary files /dev/null and b/cmd/watchdog/watchdog differ diff --git a/watchdog b/watchdog new file mode 120000 index 0000000..d50d139 --- /dev/null +++ b/watchdog @@ -0,0 +1 @@ +cmd/watchdog/watchdog \ No newline at end of file