diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 9f7666907..9b97ef00d 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1699,6 +1699,22 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque } } + getAndWriteSysErrors := func() { + if query.Get(string(madmin.HealthDataTypeSysErrors)) == "true" { + localSysErrors := madmin.GetSysErrors(deadlinedCtx, globalLocalNodeName) + anonymizeAddr(&localSysErrors) + healthInfo.Sys.SysErrs = append(healthInfo.Sys.SysErrs, localSysErrors) + partialWrite(healthInfo) + + peerSysErrs := globalNotificationSys.GetSysErrors(deadlinedCtx) + for _, se := range peerSysErrs { + anonymizeAddr(&se) + healthInfo.Sys.SysErrs = append(healthInfo.Sys.SysErrs, se) + } + partialWrite(healthInfo) + } + } + anonymizeCmdLine := func(cmdLine string) string { if !globalIsDistErasure { // FS mode - single server - hard code to `server1` @@ -1872,6 +1888,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque getAndWriteMinioConfig() getAndWriteDrivePerfInfo() getAndWriteNetPerfInfo() + getAndWriteSysErrors() if query.Get("minioinfo") == "true" { infoMessage := getServerInfo(ctx, r) diff --git a/cmd/notification.go b/cmd/notification.go index 6743f024c..0956e3af9 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -1066,6 +1066,36 @@ func (sys *NotificationSys) GetOSInfo(ctx context.Context) []madmin.OSInfo { return reply } +// GetSysErrors - Memory information +func (sys *NotificationSys) GetSysErrors(ctx context.Context) []madmin.SysErrors { + reply := make([]madmin.SysErrors, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetSysErrors(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + addr := sys.peerClients[index].host.String() + reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr) + ctx := logger.SetReqInfo(GlobalContext, reqInfo) + logger.LogIf(ctx, err) + reply[index].Addr = addr + reply[index].Error = err.Error() + } + } + return reply +} + // GetMemInfo - Memory information func (sys *NotificationSys) GetMemInfo(ctx context.Context) []madmin.MemInfo { reply := make([]madmin.MemInfo, len(sys.peerClients)) diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go index ff794a319..60f19fca8 100644 --- a/cmd/peer-rest-client.go +++ b/cmd/peer-rest-client.go @@ -400,6 +400,17 @@ func (client *peerRESTClient) GetOSInfo(ctx context.Context) (info madmin.OSInfo return info, err } +// GetSysErrors - fetch memory information for a remote node. +func (client *peerRESTClient) GetSysErrors(ctx context.Context) (info madmin.SysErrors, err error) { + respBody, err := client.callWithContext(ctx, peerRESTMethodSysErrors, nil, nil, -1) + if err != nil { + return + } + defer http.DrainBody(respBody) + err = gob.NewDecoder(respBody).Decode(&info) + return info, err +} + // GetMemInfo - fetch memory information for a remote node. func (client *peerRESTClient) GetMemInfo(ctx context.Context) (info madmin.MemInfo, err error) { respBody, err := client.callWithContext(ctx, peerRESTMethodMemInfo, nil, nil, -1) diff --git a/cmd/peer-rest-common.go b/cmd/peer-rest-common.go index 7315ae2cb..c428adda2 100644 --- a/cmd/peer-rest-common.go +++ b/cmd/peer-rest-common.go @@ -34,6 +34,7 @@ const ( peerRESTMethodOsInfo = "/osinfo" peerRESTMethodMemInfo = "/meminfo" peerRESTMethodProcInfo = "/procinfo" + peerRESTMethodSysErrors = "/syserrors" peerRESTMethodDispatchNetInfo = "/dispatchnetinfo" peerRESTMethodDeleteBucketMetadata = "/deletebucketmetadata" peerRESTMethodLoadBucketMetadata = "/loadbucketmetadata" diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index 0bdc0a29d..32300b261 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -510,6 +510,22 @@ func (s *peerRESTServer) GetMemInfoHandler(w http.ResponseWriter, r *http.Reques logger.LogIf(ctx, gob.NewEncoder(w).Encode(info)) } +// GetSysErrorsHandler - returns memory information. +func (s *peerRESTServer) GetSysErrorsHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + info := madmin.GetSysErrors(ctx, r.Host) + + defer w.(http.Flusher).Flush() + logger.LogIf(ctx, gob.NewEncoder(w).Encode(info)) +} + // DeleteBucketMetadataHandler - Delete in memory bucket metadata func (s *peerRESTServer) DeleteBucketMetadataHandler(w http.ResponseWriter, r *http.Request) { if !s.IsValid(w, r) { @@ -1285,6 +1301,7 @@ func registerPeerRESTHandlers(router *mux.Router) { subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(httpTraceHdrs(server.GetProcInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(httpTraceHdrs(server.GetMemInfoHandler)) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(httpTraceHdrs(server.GetSysErrorsHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))