subpath_nsenter.go 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187
  1. // +build linux
  2. /*
  3. Copyright 2014 The Kubernetes Authors.
  4. Licensed under the Apache License, Version 2.0 (the "License");
  5. you may not use this file except in compliance with the License.
  6. You may obtain a copy of the License at
  7. http://www.apache.org/licenses/LICENSE-2.0
  8. Unless required by applicable law or agreed to in writing, software
  9. distributed under the License is distributed on an "AS IS" BASIS,
  10. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. See the License for the specific language governing permissions and
  12. limitations under the License.
  13. */
  14. package subpath
  15. import (
  16. "fmt"
  17. "os"
  18. "path/filepath"
  19. "syscall"
  20. "golang.org/x/sys/unix"
  21. "k8s.io/klog"
  22. "k8s.io/utils/nsenter"
  23. "k8s.io/kubernetes/pkg/util/mount"
  24. )
  25. type subpathNSE struct {
  26. mounter mount.Interface
  27. ne *nsenter.Nsenter
  28. rootDir string
  29. }
  30. // Compile time-check for all implementers of subpath interface
  31. var _ Interface = &subpathNSE{}
  32. // NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter
  33. // It is only valid on Linux systems
  34. func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
  35. return &subpathNSE{
  36. mounter: mounter,
  37. ne: ne,
  38. rootDir: rootDir,
  39. }
  40. }
  41. func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error {
  42. return doCleanSubPaths(sp.mounter, podDir, volumeName)
  43. }
  44. func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
  45. // Bind-mount the subpath to avoid using symlinks in subpaths.
  46. newHostPath, err = sp.doNsEnterBindSubPath(subPath)
  47. // There is no action when the container starts. Bind-mount will be cleaned
  48. // when container stops by CleanSubPaths.
  49. cleanupAction = nil
  50. return newHostPath, cleanupAction, err
  51. }
  52. func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
  53. fullSubdirPath := filepath.Join(base, subdir)
  54. evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
  55. if err != nil {
  56. return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
  57. }
  58. evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
  59. evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */)
  60. if err != nil {
  61. return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
  62. }
  63. evaluatedBase = filepath.Clean(evaluatedBase)
  64. rootDir := filepath.Clean(sp.rootDir)
  65. if mount.PathWithinBase(evaluatedBase, rootDir) {
  66. // Base is in /var/lib/kubelet. This directory is shared between the
  67. // container with kubelet and the host. We don't need to add '/rootfs'.
  68. // This is useful when /rootfs is mounted as read-only - we can still
  69. // create subpaths for paths in /var/lib/kubelet.
  70. return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
  71. }
  72. // Base is somewhere on the host's filesystem. Add /rootfs and try to make
  73. // the directory there.
  74. // This requires /rootfs to be writable.
  75. kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath)
  76. kubeletBase := sp.ne.KubeletPath(evaluatedBase)
  77. return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
  78. }
  79. func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) {
  80. // Linux, kubelet runs in a container:
  81. // - safely open the subpath
  82. // - bind-mount the subpath to target (this can be unsafe)
  83. // - check that we mounted the right thing by comparing device ID and inode
  84. // of the subpath (via safely opened fd) and the target (that's under our
  85. // control)
  86. // Evaluate all symlinks here once for all subsequent functions.
  87. evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
  88. if err != nil {
  89. return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
  90. }
  91. evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
  92. if err != nil {
  93. return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
  94. }
  95. klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
  96. subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath)
  97. subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath)
  98. // Check the subpath is correct and open it
  99. fd, err := safeOpenSubPath(sp.mounter, subpath)
  100. if err != nil {
  101. return "", err
  102. }
  103. defer syscall.Close(fd)
  104. alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath)
  105. if err != nil {
  106. return "", err
  107. }
  108. if alreadyMounted {
  109. return bindPathTarget, nil
  110. }
  111. success := false
  112. defer func() {
  113. // Cleanup subpath on error
  114. if !success {
  115. klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
  116. if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil {
  117. klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
  118. }
  119. }
  120. }()
  121. // Leap of faith: optimistically expect that nobody has modified previously
  122. // expanded evalSubPath with evil symlinks and bind-mount it.
  123. // Mount is done on the host! don't use kubelet path!
  124. klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
  125. if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
  126. return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
  127. }
  128. // Check that the bind-mount target is the same inode and device as the
  129. // source that we keept open, i.e. we mounted the right thing.
  130. err = checkDeviceInode(fd, bindPathTarget)
  131. if err != nil {
  132. return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
  133. }
  134. success = true
  135. klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
  136. return bindPathTarget, nil
  137. }
  138. // checkDeviceInode checks that opened file and path represent the same file.
  139. func checkDeviceInode(fd int, path string) error {
  140. var srcStat, dstStat unix.Stat_t
  141. err := unix.Fstat(fd, &srcStat)
  142. if err != nil {
  143. return fmt.Errorf("error running fstat on subpath FD: %v", err)
  144. }
  145. err = unix.Stat(path, &dstStat)
  146. if err != nil {
  147. return fmt.Errorf("error running fstat on %s: %v", path, err)
  148. }
  149. if srcStat.Dev != dstStat.Dev {
  150. return fmt.Errorf("different device number")
  151. }
  152. if srcStat.Ino != dstStat.Ino {
  153. return fmt.Errorf("different inode")
  154. }
  155. return nil
  156. }