fs.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763
  1. // Copyright 2014 Google Inc. All Rights Reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // +build linux
  15. // Provides Filesystem Stats
  16. package fs
  17. import (
  18. "bufio"
  19. "fmt"
  20. "io/ioutil"
  21. "os"
  22. "os/exec"
  23. "path"
  24. "path/filepath"
  25. "regexp"
  26. "strconv"
  27. "strings"
  28. "syscall"
  29. "github.com/docker/docker/pkg/mount"
  30. "github.com/google/cadvisor/devicemapper"
  31. "github.com/google/cadvisor/utils"
  32. dockerutil "github.com/google/cadvisor/utils/docker"
  33. zfs "github.com/mistifyio/go-zfs"
  34. "k8s.io/klog"
  35. )
  36. const (
  37. LabelSystemRoot = "root"
  38. LabelDockerImages = "docker-images"
  39. LabelCrioImages = "crio-images"
  40. )
  41. const (
  42. // The block size in bytes.
  43. statBlockSize uint64 = 512
  44. // The maximum number of `disk usage` tasks that can be running at once.
  45. maxConcurrentOps = 20
  46. )
  47. // A pool for restricting the number of consecutive `du` and `find` tasks running.
  48. var pool = make(chan struct{}, maxConcurrentOps)
  49. func init() {
  50. for i := 0; i < maxConcurrentOps; i++ {
  51. releaseToken()
  52. }
  53. }
  54. func claimToken() {
  55. <-pool
  56. }
  57. func releaseToken() {
  58. pool <- struct{}{}
  59. }
  60. type partition struct {
  61. mountpoint string
  62. major uint
  63. minor uint
  64. fsType string
  65. blockSize uint
  66. }
  67. type RealFsInfo struct {
  68. // Map from block device path to partition information.
  69. partitions map[string]partition
  70. // Map from label to block device path.
  71. // Labels are intent-specific tags that are auto-detected.
  72. labels map[string]string
  73. // Map from mountpoint to mount information.
  74. mounts map[string]*mount.Info
  75. // devicemapper client
  76. dmsetup devicemapper.DmsetupClient
  77. // fsUUIDToDeviceName is a map from the filesystem UUID to its device name.
  78. fsUUIDToDeviceName map[string]string
  79. }
  80. func NewFsInfo(context Context) (FsInfo, error) {
  81. mounts, err := mount.GetMounts(nil)
  82. if err != nil {
  83. return nil, err
  84. }
  85. fsUUIDToDeviceName, err := getFsUUIDToDeviceNameMap()
  86. if err != nil {
  87. // UUID is not always available across different OS distributions.
  88. // Do not fail if there is an error.
  89. klog.Warningf("Failed to get disk UUID mapping, getting disk info by uuid will not work: %v", err)
  90. }
  91. // Avoid devicemapper container mounts - these are tracked by the ThinPoolWatcher
  92. excluded := []string{fmt.Sprintf("%s/devicemapper/mnt", context.Docker.Root)}
  93. fsInfo := &RealFsInfo{
  94. partitions: processMounts(mounts, excluded),
  95. labels: make(map[string]string, 0),
  96. mounts: make(map[string]*mount.Info, 0),
  97. dmsetup: devicemapper.NewDmsetupClient(),
  98. fsUUIDToDeviceName: fsUUIDToDeviceName,
  99. }
  100. for _, mount := range mounts {
  101. fsInfo.mounts[mount.Mountpoint] = mount
  102. }
  103. // need to call this before the log line below printing out the partitions, as this function may
  104. // add a "partition" for devicemapper to fsInfo.partitions
  105. fsInfo.addDockerImagesLabel(context, mounts)
  106. fsInfo.addCrioImagesLabel(context, mounts)
  107. klog.V(1).Infof("Filesystem UUIDs: %+v", fsInfo.fsUUIDToDeviceName)
  108. klog.V(1).Infof("Filesystem partitions: %+v", fsInfo.partitions)
  109. fsInfo.addSystemRootLabel(mounts)
  110. return fsInfo, nil
  111. }
  112. // getFsUUIDToDeviceNameMap creates the filesystem uuid to device name map
  113. // using the information in /dev/disk/by-uuid. If the directory does not exist,
  114. // this function will return an empty map.
  115. func getFsUUIDToDeviceNameMap() (map[string]string, error) {
  116. const dir = "/dev/disk/by-uuid"
  117. if _, err := os.Stat(dir); os.IsNotExist(err) {
  118. return make(map[string]string), nil
  119. }
  120. files, err := ioutil.ReadDir(dir)
  121. if err != nil {
  122. return nil, err
  123. }
  124. fsUUIDToDeviceName := make(map[string]string)
  125. for _, file := range files {
  126. path := filepath.Join(dir, file.Name())
  127. target, err := os.Readlink(path)
  128. if err != nil {
  129. klog.Warningf("Failed to resolve symlink for %q", path)
  130. continue
  131. }
  132. device, err := filepath.Abs(filepath.Join(dir, target))
  133. if err != nil {
  134. return nil, fmt.Errorf("failed to resolve the absolute path of %q", filepath.Join(dir, target))
  135. }
  136. fsUUIDToDeviceName[file.Name()] = device
  137. }
  138. return fsUUIDToDeviceName, nil
  139. }
  140. func processMounts(mounts []*mount.Info, excludedMountpointPrefixes []string) map[string]partition {
  141. partitions := make(map[string]partition, 0)
  142. supportedFsType := map[string]bool{
  143. // all ext systems are checked through prefix.
  144. "btrfs": true,
  145. "overlay": true,
  146. "tmpfs": true,
  147. "xfs": true,
  148. "zfs": true,
  149. }
  150. for _, mount := range mounts {
  151. if !strings.HasPrefix(mount.Fstype, "ext") && !supportedFsType[mount.Fstype] {
  152. continue
  153. }
  154. // Avoid bind mounts, exclude tmpfs.
  155. if _, ok := partitions[mount.Source]; ok {
  156. if mount.Fstype != "tmpfs" {
  157. continue
  158. }
  159. }
  160. hasPrefix := false
  161. for _, prefix := range excludedMountpointPrefixes {
  162. if strings.HasPrefix(mount.Mountpoint, prefix) {
  163. hasPrefix = true
  164. break
  165. }
  166. }
  167. if hasPrefix {
  168. continue
  169. }
  170. // using mountpoint to replace device once fstype it tmpfs
  171. if mount.Fstype == "tmpfs" {
  172. mount.Source = mount.Mountpoint
  173. }
  174. // btrfs fix: following workaround fixes wrong btrfs Major and Minor Ids reported in /proc/self/mountinfo.
  175. // instead of using values from /proc/self/mountinfo we use stat to get Ids from btrfs mount point
  176. if mount.Fstype == "btrfs" && mount.Major == 0 && strings.HasPrefix(mount.Source, "/dev/") {
  177. major, minor, err := getBtrfsMajorMinorIds(mount)
  178. if err != nil {
  179. klog.Warningf("%s", err)
  180. } else {
  181. mount.Major = major
  182. mount.Minor = minor
  183. }
  184. }
  185. // overlay fix: Making mount source unique for all overlay mounts, using the mount's major and minor ids.
  186. if mount.Fstype == "overlay" {
  187. mount.Source = fmt.Sprintf("%s_%d-%d", mount.Source, mount.Major, mount.Minor)
  188. }
  189. partitions[mount.Source] = partition{
  190. fsType: mount.Fstype,
  191. mountpoint: mount.Mountpoint,
  192. major: uint(mount.Major),
  193. minor: uint(mount.Minor),
  194. }
  195. }
  196. return partitions
  197. }
  198. // getDockerDeviceMapperInfo returns information about the devicemapper device and "partition" if
  199. // docker is using devicemapper for its storage driver. If a loopback device is being used, don't
  200. // return any information or error, as we want to report based on the actual partition where the
  201. // loopback file resides, inside of the loopback file itself.
  202. func (self *RealFsInfo) getDockerDeviceMapperInfo(context DockerContext) (string, *partition, error) {
  203. if context.Driver != DeviceMapper.String() {
  204. return "", nil, nil
  205. }
  206. dataLoopFile := context.DriverStatus[dockerutil.DriverStatusDataLoopFile]
  207. if len(dataLoopFile) > 0 {
  208. return "", nil, nil
  209. }
  210. dev, major, minor, blockSize, err := dockerDMDevice(context.DriverStatus, self.dmsetup)
  211. if err != nil {
  212. return "", nil, err
  213. }
  214. return dev, &partition{
  215. fsType: DeviceMapper.String(),
  216. major: major,
  217. minor: minor,
  218. blockSize: blockSize,
  219. }, nil
  220. }
  221. // addSystemRootLabel attempts to determine which device contains the mount for /.
  222. func (self *RealFsInfo) addSystemRootLabel(mounts []*mount.Info) {
  223. for _, m := range mounts {
  224. if m.Mountpoint == "/" {
  225. self.partitions[m.Source] = partition{
  226. fsType: m.Fstype,
  227. mountpoint: m.Mountpoint,
  228. major: uint(m.Major),
  229. minor: uint(m.Minor),
  230. }
  231. self.labels[LabelSystemRoot] = m.Source
  232. return
  233. }
  234. }
  235. }
  236. // addDockerImagesLabel attempts to determine which device contains the mount for docker images.
  237. func (self *RealFsInfo) addDockerImagesLabel(context Context, mounts []*mount.Info) {
  238. dockerDev, dockerPartition, err := self.getDockerDeviceMapperInfo(context.Docker)
  239. if err != nil {
  240. klog.Warningf("Could not get Docker devicemapper device: %v", err)
  241. }
  242. if len(dockerDev) > 0 && dockerPartition != nil {
  243. self.partitions[dockerDev] = *dockerPartition
  244. self.labels[LabelDockerImages] = dockerDev
  245. } else {
  246. self.updateContainerImagesPath(LabelDockerImages, mounts, getDockerImagePaths(context))
  247. }
  248. }
  249. func (self *RealFsInfo) addCrioImagesLabel(context Context, mounts []*mount.Info) {
  250. if context.Crio.Root != "" {
  251. crioPath := context.Crio.Root
  252. crioImagePaths := map[string]struct{}{
  253. "/": {},
  254. }
  255. for _, dir := range []string{"overlay", "overlay2"} {
  256. crioImagePaths[path.Join(crioPath, dir+"-images")] = struct{}{}
  257. }
  258. for crioPath != "/" && crioPath != "." {
  259. crioImagePaths[crioPath] = struct{}{}
  260. crioPath = filepath.Dir(crioPath)
  261. }
  262. self.updateContainerImagesPath(LabelCrioImages, mounts, crioImagePaths)
  263. }
  264. }
  265. // Generate a list of possible mount points for docker image management from the docker root directory.
  266. // Right now, we look for each type of supported graph driver directories, but we can do better by parsing
  267. // some of the context from `docker info`.
  268. func getDockerImagePaths(context Context) map[string]struct{} {
  269. dockerImagePaths := map[string]struct{}{
  270. "/": {},
  271. }
  272. // TODO(rjnagal): Detect docker root and graphdriver directories from docker info.
  273. dockerRoot := context.Docker.Root
  274. for _, dir := range []string{"devicemapper", "btrfs", "aufs", "overlay", "overlay2", "zfs"} {
  275. dockerImagePaths[path.Join(dockerRoot, dir)] = struct{}{}
  276. }
  277. for dockerRoot != "/" && dockerRoot != "." {
  278. dockerImagePaths[dockerRoot] = struct{}{}
  279. dockerRoot = filepath.Dir(dockerRoot)
  280. }
  281. return dockerImagePaths
  282. }
  283. // This method compares the mountpoints with possible container image mount points. If a match is found,
  284. // the label is added to the partition.
  285. func (self *RealFsInfo) updateContainerImagesPath(label string, mounts []*mount.Info, containerImagePaths map[string]struct{}) {
  286. var useMount *mount.Info
  287. for _, m := range mounts {
  288. if _, ok := containerImagePaths[m.Mountpoint]; ok {
  289. if useMount == nil || (len(useMount.Mountpoint) < len(m.Mountpoint)) {
  290. useMount = m
  291. }
  292. }
  293. }
  294. if useMount != nil {
  295. self.partitions[useMount.Source] = partition{
  296. fsType: useMount.Fstype,
  297. mountpoint: useMount.Mountpoint,
  298. major: uint(useMount.Major),
  299. minor: uint(useMount.Minor),
  300. }
  301. self.labels[label] = useMount.Source
  302. }
  303. }
  304. func (self *RealFsInfo) GetDeviceForLabel(label string) (string, error) {
  305. dev, ok := self.labels[label]
  306. if !ok {
  307. return "", fmt.Errorf("non-existent label %q", label)
  308. }
  309. return dev, nil
  310. }
  311. func (self *RealFsInfo) GetLabelsForDevice(device string) ([]string, error) {
  312. labels := []string{}
  313. for label, dev := range self.labels {
  314. if dev == device {
  315. labels = append(labels, label)
  316. }
  317. }
  318. return labels, nil
  319. }
  320. func (self *RealFsInfo) GetMountpointForDevice(dev string) (string, error) {
  321. p, ok := self.partitions[dev]
  322. if !ok {
  323. return "", fmt.Errorf("no partition info for device %q", dev)
  324. }
  325. return p.mountpoint, nil
  326. }
  327. func (self *RealFsInfo) GetFsInfoForPath(mountSet map[string]struct{}) ([]Fs, error) {
  328. filesystems := make([]Fs, 0)
  329. deviceSet := make(map[string]struct{})
  330. diskStatsMap, err := getDiskStatsMap("/proc/diskstats")
  331. if err != nil {
  332. return nil, err
  333. }
  334. for device, partition := range self.partitions {
  335. _, hasMount := mountSet[partition.mountpoint]
  336. _, hasDevice := deviceSet[device]
  337. if mountSet == nil || (hasMount && !hasDevice) {
  338. var (
  339. err error
  340. fs Fs
  341. )
  342. switch partition.fsType {
  343. case DeviceMapper.String():
  344. fs.Capacity, fs.Free, fs.Available, err = getDMStats(device, partition.blockSize)
  345. klog.V(5).Infof("got devicemapper fs capacity stats: capacity: %v free: %v available: %v:", fs.Capacity, fs.Free, fs.Available)
  346. fs.Type = DeviceMapper
  347. case ZFS.String():
  348. if _, devzfs := os.Stat("/dev/zfs"); os.IsExist(devzfs) {
  349. fs.Capacity, fs.Free, fs.Available, err = getZfstats(device)
  350. fs.Type = ZFS
  351. break
  352. }
  353. // if /dev/zfs is not present default to VFS
  354. fallthrough
  355. default:
  356. var inodes, inodesFree uint64
  357. if utils.FileExists(partition.mountpoint) {
  358. fs.Capacity, fs.Free, fs.Available, inodes, inodesFree, err = getVfsStats(partition.mountpoint)
  359. fs.Inodes = &inodes
  360. fs.InodesFree = &inodesFree
  361. fs.Type = VFS
  362. } else {
  363. klog.V(4).Infof("unable to determine file system type, partition mountpoint does not exist: %v", partition.mountpoint)
  364. }
  365. }
  366. if err != nil {
  367. klog.V(4).Infof("Stat fs failed. Error: %v", err)
  368. } else {
  369. deviceSet[device] = struct{}{}
  370. fs.DeviceInfo = DeviceInfo{
  371. Device: device,
  372. Major: uint(partition.major),
  373. Minor: uint(partition.minor),
  374. }
  375. fs.DiskStats = diskStatsMap[device]
  376. filesystems = append(filesystems, fs)
  377. }
  378. }
  379. }
  380. return filesystems, nil
  381. }
  382. var partitionRegex = regexp.MustCompile(`^(?:(?:s|v|xv)d[a-z]+\d*|dm-\d+)$`)
  383. func getDiskStatsMap(diskStatsFile string) (map[string]DiskStats, error) {
  384. diskStatsMap := make(map[string]DiskStats)
  385. file, err := os.Open(diskStatsFile)
  386. if err != nil {
  387. if os.IsNotExist(err) {
  388. klog.Warningf("Not collecting filesystem statistics because file %q was not found", diskStatsFile)
  389. return diskStatsMap, nil
  390. }
  391. return nil, err
  392. }
  393. defer file.Close()
  394. scanner := bufio.NewScanner(file)
  395. for scanner.Scan() {
  396. line := scanner.Text()
  397. words := strings.Fields(line)
  398. if !partitionRegex.MatchString(words[2]) {
  399. continue
  400. }
  401. // 8 50 sdd2 40 0 280 223 7 0 22 108 0 330 330
  402. deviceName := path.Join("/dev", words[2])
  403. wordLength := len(words)
  404. offset := 3
  405. var stats = make([]uint64, wordLength-offset)
  406. if len(stats) < 11 {
  407. return nil, fmt.Errorf("could not parse all 11 columns of /proc/diskstats")
  408. }
  409. var error error
  410. for i := offset; i < wordLength; i++ {
  411. stats[i-offset], error = strconv.ParseUint(words[i], 10, 64)
  412. if error != nil {
  413. return nil, error
  414. }
  415. }
  416. diskStats := DiskStats{
  417. ReadsCompleted: stats[0],
  418. ReadsMerged: stats[1],
  419. SectorsRead: stats[2],
  420. ReadTime: stats[3],
  421. WritesCompleted: stats[4],
  422. WritesMerged: stats[5],
  423. SectorsWritten: stats[6],
  424. WriteTime: stats[7],
  425. IoInProgress: stats[8],
  426. IoTime: stats[9],
  427. WeightedIoTime: stats[10],
  428. }
  429. diskStatsMap[deviceName] = diskStats
  430. }
  431. return diskStatsMap, nil
  432. }
  433. func (self *RealFsInfo) GetGlobalFsInfo() ([]Fs, error) {
  434. return self.GetFsInfoForPath(nil)
  435. }
  436. func major(devNumber uint64) uint {
  437. return uint((devNumber >> 8) & 0xfff)
  438. }
  439. func minor(devNumber uint64) uint {
  440. return uint((devNumber & 0xff) | ((devNumber >> 12) & 0xfff00))
  441. }
  442. func (self *RealFsInfo) GetDeviceInfoByFsUUID(uuid string) (*DeviceInfo, error) {
  443. deviceName, found := self.fsUUIDToDeviceName[uuid]
  444. if !found {
  445. return nil, ErrNoSuchDevice
  446. }
  447. p, found := self.partitions[deviceName]
  448. if !found {
  449. return nil, fmt.Errorf("cannot find device %q in partitions", deviceName)
  450. }
  451. return &DeviceInfo{deviceName, p.major, p.minor}, nil
  452. }
  453. func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) {
  454. buf := new(syscall.Stat_t)
  455. err := syscall.Stat(dir, buf)
  456. if err != nil {
  457. return nil, fmt.Errorf("stat failed on %s with error: %s", dir, err)
  458. }
  459. major := major(buf.Dev)
  460. minor := minor(buf.Dev)
  461. for device, partition := range self.partitions {
  462. if partition.major == major && partition.minor == minor {
  463. return &DeviceInfo{device, major, minor}, nil
  464. }
  465. }
  466. mount, found := self.mounts[dir]
  467. // try the parent dir if not found until we reach the root dir
  468. // this is an issue on btrfs systems where the directory is not
  469. // the subvolume
  470. for !found {
  471. pathdir, _ := filepath.Split(dir)
  472. // break when we reach root
  473. if pathdir == "/" {
  474. break
  475. }
  476. // trim "/" from the new parent path otherwise the next possible
  477. // filepath.Split in the loop will not split the string any further
  478. dir = strings.TrimSuffix(pathdir, "/")
  479. mount, found = self.mounts[dir]
  480. }
  481. if found && mount.Fstype == "btrfs" && mount.Major == 0 && strings.HasPrefix(mount.Source, "/dev/") {
  482. major, minor, err := getBtrfsMajorMinorIds(mount)
  483. if err != nil {
  484. klog.Warningf("%s", err)
  485. } else {
  486. return &DeviceInfo{mount.Source, uint(major), uint(minor)}, nil
  487. }
  488. }
  489. return nil, fmt.Errorf("could not find device with major: %d, minor: %d in cached partitions map", major, minor)
  490. }
  491. func GetDirUsage(dir string) (UsageInfo, error) {
  492. var usage UsageInfo
  493. if dir == "" {
  494. return usage, fmt.Errorf("invalid directory")
  495. }
  496. rootInfo, err := os.Stat(dir)
  497. if err != nil {
  498. return usage, fmt.Errorf("could not stat %q to get inode usage: %v", dir, err)
  499. }
  500. rootStat, ok := rootInfo.Sys().(*syscall.Stat_t)
  501. if !ok {
  502. return usage, fmt.Errorf("unsuported fileinfo for getting inode usage of %q", dir)
  503. }
  504. rootDevId := rootStat.Dev
  505. // dedupedInode stores inodes that could be duplicates (nlink > 1)
  506. dedupedInodes := make(map[uint64]struct{})
  507. err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
  508. if os.IsNotExist(err) {
  509. // expected if files appear/vanish
  510. return nil
  511. }
  512. if err != nil {
  513. return fmt.Errorf("unable to count inodes for part of dir %s: %s", dir, err)
  514. }
  515. // according to the docs, Sys can be nil
  516. if info.Sys() == nil {
  517. return fmt.Errorf("fileinfo Sys is nil")
  518. }
  519. s, ok := info.Sys().(*syscall.Stat_t)
  520. if !ok {
  521. return fmt.Errorf("unsupported fileinfo; could not convert to stat_t")
  522. }
  523. if s.Dev != rootDevId {
  524. // don't descend into directories on other devices
  525. return filepath.SkipDir
  526. }
  527. if s.Nlink > 1 {
  528. if _, ok := dedupedInodes[s.Ino]; !ok {
  529. // Dedupe things that could be hardlinks
  530. dedupedInodes[s.Ino] = struct{}{}
  531. usage.Bytes += uint64(s.Blocks) * statBlockSize
  532. usage.Inodes++
  533. }
  534. } else {
  535. usage.Bytes += uint64(s.Blocks) * statBlockSize
  536. usage.Inodes++
  537. }
  538. return nil
  539. })
  540. return usage, nil
  541. }
  542. func (self *RealFsInfo) GetDirUsage(dir string) (UsageInfo, error) {
  543. claimToken()
  544. defer releaseToken()
  545. return GetDirUsage(dir)
  546. }
  547. func getVfsStats(path string) (total uint64, free uint64, avail uint64, inodes uint64, inodesFree uint64, err error) {
  548. var s syscall.Statfs_t
  549. if err = syscall.Statfs(path, &s); err != nil {
  550. return 0, 0, 0, 0, 0, err
  551. }
  552. total = uint64(s.Frsize) * s.Blocks
  553. free = uint64(s.Frsize) * s.Bfree
  554. avail = uint64(s.Frsize) * s.Bavail
  555. inodes = uint64(s.Files)
  556. inodesFree = uint64(s.Ffree)
  557. return total, free, avail, inodes, inodesFree, nil
  558. }
  559. // Devicemapper thin provisioning is detailed at
  560. // https://www.kernel.org/doc/Documentation/device-mapper/thin-provisioning.txt
  561. func dockerDMDevice(driverStatus map[string]string, dmsetup devicemapper.DmsetupClient) (string, uint, uint, uint, error) {
  562. poolName, ok := driverStatus[dockerutil.DriverStatusPoolName]
  563. if !ok || len(poolName) == 0 {
  564. return "", 0, 0, 0, fmt.Errorf("Could not get dm pool name")
  565. }
  566. out, err := dmsetup.Table(poolName)
  567. if err != nil {
  568. return "", 0, 0, 0, err
  569. }
  570. major, minor, dataBlkSize, err := parseDMTable(string(out))
  571. if err != nil {
  572. return "", 0, 0, 0, err
  573. }
  574. return poolName, major, minor, dataBlkSize, nil
  575. }
  576. // parseDMTable parses a single line of `dmsetup table` output and returns the
  577. // major device, minor device, block size, and an error.
  578. func parseDMTable(dmTable string) (uint, uint, uint, error) {
  579. dmTable = strings.Replace(dmTable, ":", " ", -1)
  580. dmFields := strings.Fields(dmTable)
  581. if len(dmFields) < 8 {
  582. return 0, 0, 0, fmt.Errorf("Invalid dmsetup status output: %s", dmTable)
  583. }
  584. major, err := strconv.ParseUint(dmFields[5], 10, 32)
  585. if err != nil {
  586. return 0, 0, 0, err
  587. }
  588. minor, err := strconv.ParseUint(dmFields[6], 10, 32)
  589. if err != nil {
  590. return 0, 0, 0, err
  591. }
  592. dataBlkSize, err := strconv.ParseUint(dmFields[7], 10, 32)
  593. if err != nil {
  594. return 0, 0, 0, err
  595. }
  596. return uint(major), uint(minor), uint(dataBlkSize), nil
  597. }
  598. func getDMStats(poolName string, dataBlkSize uint) (uint64, uint64, uint64, error) {
  599. out, err := exec.Command("dmsetup", "status", poolName).Output()
  600. if err != nil {
  601. return 0, 0, 0, err
  602. }
  603. used, total, err := parseDMStatus(string(out))
  604. if err != nil {
  605. return 0, 0, 0, err
  606. }
  607. used *= 512 * uint64(dataBlkSize)
  608. total *= 512 * uint64(dataBlkSize)
  609. free := total - used
  610. return total, free, free, nil
  611. }
  612. func parseDMStatus(dmStatus string) (uint64, uint64, error) {
  613. dmStatus = strings.Replace(dmStatus, "/", " ", -1)
  614. dmFields := strings.Fields(dmStatus)
  615. if len(dmFields) < 8 {
  616. return 0, 0, fmt.Errorf("Invalid dmsetup status output: %s", dmStatus)
  617. }
  618. used, err := strconv.ParseUint(dmFields[6], 10, 64)
  619. if err != nil {
  620. return 0, 0, err
  621. }
  622. total, err := strconv.ParseUint(dmFields[7], 10, 64)
  623. if err != nil {
  624. return 0, 0, err
  625. }
  626. return used, total, nil
  627. }
  628. // getZfstats returns ZFS mount stats using zfsutils
  629. func getZfstats(poolName string) (uint64, uint64, uint64, error) {
  630. dataset, err := zfs.GetDataset(poolName)
  631. if err != nil {
  632. return 0, 0, 0, err
  633. }
  634. total := dataset.Used + dataset.Avail + dataset.Usedbydataset
  635. return total, dataset.Avail, dataset.Avail, nil
  636. }
  637. // Simple io.Writer implementation that counts how many bytes were written.
  638. type byteCounter struct{ bytesWritten uint64 }
  639. func (b *byteCounter) Write(p []byte) (int, error) {
  640. b.bytesWritten += uint64(len(p))
  641. return len(p), nil
  642. }
  643. // Get major and minor Ids for a mount point using btrfs as filesystem.
  644. func getBtrfsMajorMinorIds(mount *mount.Info) (int, int, error) {
  645. // btrfs fix: following workaround fixes wrong btrfs Major and Minor Ids reported in /proc/self/mountinfo.
  646. // instead of using values from /proc/self/mountinfo we use stat to get Ids from btrfs mount point
  647. buf := new(syscall.Stat_t)
  648. err := syscall.Stat(mount.Source, buf)
  649. if err != nil {
  650. err = fmt.Errorf("stat failed on %s with error: %s", mount.Source, err)
  651. return 0, 0, err
  652. }
  653. klog.V(4).Infof("btrfs mount %#v", mount)
  654. if buf.Mode&syscall.S_IFMT == syscall.S_IFBLK {
  655. err := syscall.Stat(mount.Mountpoint, buf)
  656. if err != nil {
  657. err = fmt.Errorf("stat failed on %s with error: %s", mount.Mountpoint, err)
  658. return 0, 0, err
  659. }
  660. klog.V(4).Infof("btrfs dev major:minor %d:%d\n", int(major(buf.Dev)), int(minor(buf.Dev)))
  661. klog.V(4).Infof("btrfs rdev major:minor %d:%d\n", int(major(buf.Rdev)), int(minor(buf.Rdev)))
  662. return int(major(buf.Dev)), int(minor(buf.Dev)), nil
  663. } else {
  664. return 0, 0, fmt.Errorf("%s is not a block device", mount.Source)
  665. }
  666. }