simplify CPU monitoring logic

pull/334/head
Jesse Duffield 2 years ago
parent b5384d6cdb
commit d27ce19f6c

@ -4,7 +4,6 @@ go 1.18
require (
github.com/OpenPeeDeeP/xdg v0.2.1-0.20190312153938-4ba9e1eb294c
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/boz/go-throttle v0.0.0-20160922054636-fdc4eab740c1
github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
github.com/docker/docker v0.7.3-0.20190307005417-54dddadc7d5d

@ -6,8 +6,6 @@ github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+q
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/OpenPeeDeeP/xdg v0.2.1-0.20190312153938-4ba9e1eb294c h1:YDsGA6tou+tAxVe0Dre29iSbQ8TrWdWfwOisKArJT5E=
github.com/OpenPeeDeeP/xdg v0.2.1-0.20190312153938-4ba9e1eb294c/go.mod h1:tMoSueLQlMf0TCldjrJLNIjAc5qAOIcHt5REi88/Ygo=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/boz/go-throttle v0.0.0-20160922054636-fdc4eab740c1 h1:1fx+RA5lk1ZkzPAUP7DEgZnVHYxEcHO77vQO/V8z/2Q=
github.com/boz/go-throttle v0.0.0-20160922054636-fdc4eab740c1/go.mod h1:z0nyIb42Zs97wyX1V+8MbEFhHeTw1OgFQfR6q57ZuHc=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=

@ -6,6 +6,7 @@ import (
"os/exec"
"strconv"
"strings"
"sync"
"time"
"github.com/docker/docker/api/types/container"
@ -38,12 +39,13 @@ type Container struct {
OSCommand *OSCommand
Config *config.AppConfig
Log *logrus.Entry
CLIStats ContainerCliStat // for realtime we use the CLI, for long-term we use the client
StatHistory []RecordedStats
StatHistory []*RecordedStats
Details Details
MonitoringStats bool
DockerCommand LimitedDockerCommand
Tr *i18n.TranslationSet
StatsMutex sync.Mutex
}
// Details is a struct containing what we get back from `docker inspect` on a container
@ -230,19 +232,6 @@ type Details struct {
} `json:"NetworkSettings"`
}
// ContainerCliStat is a stat object returned by the CLI docker stat command
type ContainerCliStat struct {
BlockIO string `json:"BlockIO"`
CPUPerc string `json:"CPUPerc"`
Container string `json:"Container"`
ID string `json:"ID"`
MemPerc string `json:"MemPerc"`
MemUsage string `json:"MemUsage"`
Name string `json:"Name"`
NetIO string `json:"NetIO"`
PIDs string `json:"PIDs"`
}
// GetDisplayStrings returns the dispaly string of Container
func (c *Container) GetDisplayStrings(isFocused bool) []string {
image := strings.TrimPrefix(c.Container.Image, "sha256:")
@ -285,17 +274,13 @@ func (c *Container) getHealthStatus() string {
// GetDisplayCPUPerc colors the cpu percentage based on how extreme it is
func (c *Container) GetDisplayCPUPerc() string {
stats := c.CLIStats
if stats.CPUPerc == "" {
stats, ok := c.getLastStats()
if !ok {
return ""
}
percentage, err := strconv.ParseFloat(strings.TrimSuffix(stats.CPUPerc, "%"), 32)
if err != nil {
// probably complaining about not being able to convert '--'
return ""
}
percentage := stats.DerivedStats.CPUPercentage
formattedPercentage := fmt.Sprintf("%.2f%%", stats.DerivedStats.CPUPercentage)
var clr color.Attribute
if percentage > 90 {
@ -306,7 +291,7 @@ func (c *Container) GetDisplayCPUPerc() string {
clr = color.FgWhite
}
return utils.ColoredString(stats.CPUPerc, clr)
return utils.ColoredString(formattedPercentage, clr)
}
// ProducingLogs tells us whether we should bother checking a container's logs
@ -400,20 +385,6 @@ func (c *Container) Top() (container.ContainerTopOKBody, error) {
return c.Client.ContainerTop(context.Background(), c.ID, []string{})
}
// EraseOldHistory removes any history before the user-specified max duration
func (c *Container) EraseOldHistory() {
if c.Config.UserConfig.Stats.MaxDuration == 0 {
return
}
for i, stat := range c.StatHistory {
if time.Since(stat.RecordedAt) < c.Config.UserConfig.Stats.MaxDuration {
c.StatHistory = c.StatHistory[i:]
return
}
}
}
// ViewLogs attaches to a subprocess viewing the container's logs
func (c *Container) ViewLogs() (*exec.Cmd, error) {
templateString := c.OSCommand.Config.UserConfig.CommandTemplates.ViewContainerLogs

@ -149,9 +149,8 @@ type ContainerStats struct {
func (s *ContainerStats) CalculateContainerCPUPercentage() float64 {
cpuUsageDelta := s.CPUStats.CPUUsage.TotalUsage - s.PrecpuStats.CPUUsage.TotalUsage
cpuTotalUsageDelta := s.CPUStats.SystemCPUUsage - s.PrecpuStats.SystemCPUUsage
numberOfCores := len(s.CPUStats.CPUUsage.PercpuUsage)
value := float64(cpuUsageDelta*100) * float64(numberOfCores) / float64(cpuTotalUsageDelta)
value := float64(cpuUsageDelta*100) / float64(cpuTotalUsageDelta)
if math.IsNaN(value) {
return 0
}
@ -169,11 +168,10 @@ func (s *ContainerStats) CalculateContainerMemoryUsage() float64 {
// RenderStats returns a string containing the rendered stats of the container
func (c *Container) RenderStats(viewWidth int) (string, error) {
history := c.StatHistory
if len(history) == 0 {
stats, ok := c.getLastStats()
if !ok {
return "", nil
}
currentStats := history[len(history)-1]
graphSpecs := c.Config.UserConfig.Stats.Graphs
graphs := make([]string, len(graphSpecs))
@ -185,11 +183,11 @@ func (c *Container) RenderStats(viewWidth int) (string, error) {
graphs[i] = utils.ColoredString(graph, utils.GetColorAttribute(spec.Color))
}
pidsCount := fmt.Sprintf("PIDs: %d", currentStats.ClientStats.PidsStats.Current)
dataReceived := fmt.Sprintf("Traffic received: %s", utils.FormatDecimalBytes(currentStats.ClientStats.Networks.Eth0.RxBytes))
dataSent := fmt.Sprintf("Traffic sent: %s", utils.FormatDecimalBytes(currentStats.ClientStats.Networks.Eth0.TxBytes))
pidsCount := fmt.Sprintf("PIDs: %d", stats.ClientStats.PidsStats.Current)
dataReceived := fmt.Sprintf("Traffic received: %s", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.RxBytes))
dataSent := fmt.Sprintf("Traffic sent: %s", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.TxBytes))
originalJSON, err := json.MarshalIndent(currentStats, "", " ")
originalJSON, err := json.MarshalIndent(stats, "", " ")
if err != nil {
return "", err
}
@ -205,8 +203,43 @@ func (c *Container) RenderStats(viewWidth int) (string, error) {
return contents, nil
}
func (c *Container) appendStats(stats *RecordedStats) {
c.StatsMutex.Lock()
defer c.StatsMutex.Unlock()
c.StatHistory = append(c.StatHistory, stats)
c.eraseOldHistory()
}
// eraseOldHistory removes any history before the user-specified max duration
func (c *Container) eraseOldHistory() {
if c.Config.UserConfig.Stats.MaxDuration == 0 {
return
}
for i, stat := range c.StatHistory {
if time.Since(stat.RecordedAt) < c.Config.UserConfig.Stats.MaxDuration {
c.StatHistory = c.StatHistory[i:]
return
}
}
}
func (c *Container) getLastStats() (*RecordedStats, bool) {
c.StatsMutex.Lock()
defer c.StatsMutex.Unlock()
history := c.StatHistory
if len(history) == 0 {
return nil, false
}
return history[len(history)-1], true
}
// PlotGraph returns the plotted graph based on the graph spec and the stat history
func (c *Container) PlotGraph(spec config.GraphConfig, width int) (string, error) {
c.StatsMutex.Lock()
defer c.StatsMutex.Unlock()
data := make([]float64, len(c.StatHistory))
max := spec.Max

@ -0,0 +1,17 @@
package commands
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestCalculateContainerCPUPercentage(t *testing.T) {
container := &ContainerStats{}
container.CPUStats.CPUUsage.TotalUsage = 10
container.CPUStats.SystemCPUUsage = 10
container.PrecpuStats.CPUUsage.TotalUsage = 5
container.PrecpuStats.SystemCPUUsage = 2
assert.EqualValues(t, 62.5, container.CalculateContainerCPUPercentage())
}

@ -13,7 +13,6 @@ import (
"sync"
"time"
"github.com/acarl005/stripansi"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/imdario/mergo"
@ -121,65 +120,20 @@ func (c *DockerCommand) Close() error {
return utils.CloseMany(c.Closers)
}
// MonitorContainerStats is a function
func (c *DockerCommand) MonitorContainerStats() {
// TODO: pass in a stop channel to these so we don't restart every time we come back from a subprocess
go c.MonitorCLIContainerStats()
go c.MonitorClientContainerStats()
}
// MonitorCLIContainerStats monitors a stream of container stats and updates the containers as each new stats object is received
func (c *DockerCommand) MonitorCLIContainerStats() {
onError := func(err error) {
c.ErrorChan <- err
time.Sleep(2 * time.Second)
}
for {
command := `docker stats --all --no-trunc --format '{{json .}}'`
cmd := c.OSCommand.RunCustomCommand(command)
r, err := cmd.StdoutPipe()
if err != nil {
onError(err)
continue
}
_ = cmd.Start()
scanner := bufio.NewScanner(r)
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
var stats ContainerCliStat
// need to strip ANSI codes because uses escape sequences to clear the screen with each refresh
cleanString := stripansi.Strip(scanner.Text())
if err := json.Unmarshal([]byte(cleanString), &stats); err != nil {
onError(err)
continue
}
c.ContainerMutex.Lock()
for _, container := range c.Containers {
if container.ID == stats.ID {
container.CLIStats = stats
}
}
c.ContainerMutex.Unlock()
}
_ = cmd.Wait()
}
}
// MonitorClientContainerStats is a function
func (c *DockerCommand) MonitorClientContainerStats() {
func (c *DockerCommand) MonitorContainerStats(ctx context.Context) {
// periodically loop through running containers and see if we need to create a monitor goroutine for any
// every second we check if we need to spawn a new goroutine
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for range ticker.C {
for _, container := range c.Containers {
if !container.MonitoringStats {
go c.createClientStatMonitor(container)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
for _, container := range c.Containers {
if !container.MonitoringStats {
go c.createClientStatMonitor(container)
}
}
}
}
@ -189,7 +143,10 @@ func (c *DockerCommand) createClientStatMonitor(container *Container) {
container.MonitoringStats = true
stream, err := c.Client.ContainerStats(context.Background(), container.ID, true)
if err != nil {
c.ErrorChan <- err
// not creating error panel because if we've disconnected from docker we'll
// have already created an error panel
c.Log.Error(err)
container.MonitoringStats = false
return
}
@ -201,7 +158,7 @@ func (c *DockerCommand) createClientStatMonitor(container *Container) {
var stats ContainerStats
_ = json.Unmarshal(data, &stats)
recordedStats := RecordedStats{
recordedStats := &RecordedStats{
ClientStats: stats,
DerivedStats: DerivedStats{
CPUPercentage: stats.CalculateContainerCPUPercentage(),
@ -210,10 +167,7 @@ func (c *DockerCommand) createClientStatMonitor(container *Container) {
RecordedAt: time.Now(),
}
c.ContainerMutex.Lock()
container.StatHistory = append(container.StatHistory, recordedStats)
container.EraseOldHistory()
c.ContainerMutex.Unlock()
container.appendStats(recordedStats)
}
container.MonitoringStats = false

@ -235,10 +235,11 @@ func (gui *Gui) Run() error {
throttledRefresh := throttle.ThrottleFunc(time.Millisecond*50, true, gui.refresh)
defer throttledRefresh.Stop()
finish := make(chan struct{})
defer func() { close(finish) }()
ctx, finish := context.WithCancel(context.Background())
defer finish()
go gui.listenForEvents(finish, throttledRefresh.Trigger)
go gui.listenForEvents(ctx, throttledRefresh.Trigger)
go gui.DockerCommand.MonitorContainerStats(ctx)
go func() {
gui.waitForIntro.Wait()
@ -247,11 +248,9 @@ func (gui *Gui) Run() error {
gui.goEvery(time.Millisecond*30, gui.reRenderMain)
gui.goEvery(time.Millisecond*1000, gui.DockerCommand.UpdateContainerDetails)
gui.goEvery(time.Millisecond*1000, gui.checkForContextChange)
gui.goEvery(time.Millisecond*2000, gui.rerenderContainersAndServices)
gui.goEvery(time.Millisecond*1000, gui.rerenderContainersAndServices)
}()
gui.DockerCommand.MonitorContainerStats()
go func() {
for err := range gui.ErrorChan {
if err == nil {
@ -301,7 +300,7 @@ func (gui *Gui) refresh() {
}()
}
func (gui *Gui) listenForEvents(finish chan struct{}, refresh func()) {
func (gui *Gui) listenForEvents(ctx context.Context, refresh func()) {
errorCount := 0
onError := func(err error) {
@ -336,7 +335,7 @@ outer:
for {
select {
case <-finish:
case <-ctx.Done():
return
case message := <-messageChan:
// We could be more granular about what events should trigger which refreshes.

@ -1,21 +0,0 @@
MIT License
Copyright (c) 2018 Andrew Carlson
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

@ -1,30 +0,0 @@
Strip ANSI
==========
This Go package removes ANSI escape codes from strings.
Ideally, we would prevent these from appearing in any text we want to process.
However, sometimes this can't be helped, and we need to be able to deal with that noise.
This will use a regexp to remove those unwanted escape codes.
## Install
```sh
$ go get -u github.com/acarl005/stripansi
```
## Usage
```go
import (
"fmt"
"github.com/acarl005/stripansi"
)
func main() {
msg := "\x1b[38;5;140m foo\x1b[0m bar"
cleanMsg := stripansi.Strip(msg)
fmt.Println(cleanMsg) // " foo bar"
}
```

@ -1,13 +0,0 @@
package stripansi
import (
"regexp"
)
const ansi = "[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))"
var re = regexp.MustCompile(ansi)
func Strip(str string) string {
return re.ReplaceAllString(str, "")
}

@ -9,7 +9,6 @@ github.com/Microsoft/go-winio/pkg/guid
github.com/OpenPeeDeeP/xdg
# github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
## explicit
github.com/acarl005/stripansi
# github.com/boz/go-throttle v0.0.0-20160922054636-fdc4eab740c1
## explicit
github.com/boz/go-throttle

Loading…
Cancel
Save