fsHandler.go 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151
  1. // Copyright 2015 Google Inc. All Rights Reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // Handler for Docker containers.
  15. package common
  16. import (
  17. "fmt"
  18. "sync"
  19. "time"
  20. "github.com/google/cadvisor/fs"
  21. "k8s.io/klog"
  22. )
  23. type FsHandler interface {
  24. Start()
  25. Usage() FsUsage
  26. Stop()
  27. }
  28. type FsUsage struct {
  29. BaseUsageBytes uint64
  30. TotalUsageBytes uint64
  31. InodeUsage uint64
  32. }
  33. type realFsHandler struct {
  34. sync.RWMutex
  35. lastUpdate time.Time
  36. usage FsUsage
  37. period time.Duration
  38. minPeriod time.Duration
  39. rootfs string
  40. extraDir string
  41. fsInfo fs.FsInfo
  42. // Tells the container to stop.
  43. stopChan chan struct{}
  44. }
  45. const (
  46. maxBackoffFactor = 20
  47. )
  48. const DefaultPeriod = time.Minute
  49. var _ FsHandler = &realFsHandler{}
  50. func NewFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInfo) FsHandler {
  51. return &realFsHandler{
  52. lastUpdate: time.Time{},
  53. usage: FsUsage{},
  54. period: period,
  55. minPeriod: period,
  56. rootfs: rootfs,
  57. extraDir: extraDir,
  58. fsInfo: fsInfo,
  59. stopChan: make(chan struct{}, 1),
  60. }
  61. }
  62. func (fh *realFsHandler) update() error {
  63. var (
  64. rootUsage, extraUsage fs.UsageInfo
  65. rootErr, extraErr error
  66. )
  67. // TODO(vishh): Add support for external mounts.
  68. if fh.rootfs != "" {
  69. rootUsage, rootErr = fh.fsInfo.GetDirUsage(fh.rootfs)
  70. }
  71. if fh.extraDir != "" {
  72. extraUsage, extraErr = fh.fsInfo.GetDirUsage(fh.extraDir)
  73. }
  74. // Wait to handle errors until after all operartions are run.
  75. // An error in one will not cause an early return, skipping others
  76. fh.Lock()
  77. defer fh.Unlock()
  78. fh.lastUpdate = time.Now()
  79. if fh.rootfs != "" && rootErr == nil {
  80. fh.usage.InodeUsage = rootUsage.Inodes
  81. fh.usage.TotalUsageBytes = rootUsage.Bytes + extraUsage.Bytes
  82. }
  83. if fh.extraDir != "" && extraErr == nil {
  84. fh.usage.BaseUsageBytes = rootUsage.Bytes
  85. }
  86. // Combine errors into a single error to return
  87. if rootErr != nil || extraErr != nil {
  88. return fmt.Errorf("rootDiskErr: %v, extraDiskErr: %v", rootErr, extraErr)
  89. }
  90. return nil
  91. }
  92. func (fh *realFsHandler) trackUsage() {
  93. fh.update()
  94. longOp := time.Second
  95. for {
  96. select {
  97. case <-fh.stopChan:
  98. return
  99. case <-time.After(fh.period):
  100. start := time.Now()
  101. if err := fh.update(); err != nil {
  102. klog.Errorf("failed to collect filesystem stats - %v", err)
  103. fh.period = fh.period * 2
  104. if fh.period > maxBackoffFactor*fh.minPeriod {
  105. fh.period = maxBackoffFactor * fh.minPeriod
  106. }
  107. } else {
  108. fh.period = fh.minPeriod
  109. }
  110. duration := time.Since(start)
  111. if duration > longOp {
  112. // adapt longOp time so that message doesn't continue to print
  113. // if the long duration is persistent either because of slow
  114. // disk or lots of containers.
  115. longOp = longOp + time.Second
  116. klog.V(2).Infof("fs: disk usage and inodes count on following dirs took %v: %v; will not log again for this container unless duration exceeds %v", duration, []string{fh.rootfs, fh.extraDir}, longOp)
  117. }
  118. }
  119. }
  120. }
  121. func (fh *realFsHandler) Start() {
  122. go fh.trackUsage()
  123. }
  124. func (fh *realFsHandler) Stop() {
  125. close(fh.stopChan)
  126. }
  127. func (fh *realFsHandler) Usage() FsUsage {
  128. fh.RLock()
  129. defer fh.RUnlock()
  130. return fh.usage
  131. }