docker_container_windows.go 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. // +build windows
  2. /*
  3. Copyright 2019 The Kubernetes Authors.
  4. Licensed under the Apache License, Version 2.0 (the "License");
  5. you may not use this file except in compliance with the License.
  6. You may obtain a copy of the License at
  7. http://www.apache.org/licenses/LICENSE-2.0
  8. Unless required by applicable law or agreed to in writing, software
  9. distributed under the License is distributed on an "AS IS" BASIS,
  10. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11. See the License for the specific language governing permissions and
  12. limitations under the License.
  13. */
  14. package dockershim
  15. import (
  16. "crypto/rand"
  17. "encoding/hex"
  18. "fmt"
  19. "regexp"
  20. "golang.org/x/sys/windows/registry"
  21. dockertypes "github.com/docker/docker/api/types"
  22. dockercontainer "github.com/docker/docker/api/types/container"
  23. runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
  24. )
  25. type containerCleanupInfo struct {
  26. gMSARegistryValueName string
  27. }
  28. // applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct.
  29. // The containerCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCleanup
  30. // after either:
  31. // * the container creation has failed
  32. // * the container has been successfully started
  33. // * the container has been removed
  34. // whichever happens first.
  35. func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.CreateContainerRequest, createConfig *dockertypes.ContainerCreateConfig) (*containerCleanupInfo, error) {
  36. cleanupInfo := &containerCleanupInfo{}
  37. if err := applyGMSAConfig(request.GetConfig(), createConfig, cleanupInfo); err != nil {
  38. return nil, err
  39. }
  40. return cleanupInfo, nil
  41. }
  42. // applyGMSAConfig looks at the container's .Windows.SecurityContext.GMSACredentialSpec field; if present,
  43. // it copies its contents to a unique registry value, and sets a SecurityOpt on the config pointing to that registry value.
  44. // We use registry values instead of files since their location cannot change - as opposed to credential spec files,
  45. // whose location could potentially change down the line, or even be unknown (eg if docker is not installed on the
  46. // C: drive)
  47. // When docker supports passing a credential spec's contents directly, we should switch to using that
  48. // as it will avoid cluttering the registry - there is a moby PR out for this:
  49. // https://github.com/moby/moby/pull/38777
  50. func applyGMSAConfig(config *runtimeapi.ContainerConfig, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCleanupInfo) error {
  51. var credSpec string
  52. if config.Windows != nil && config.Windows.SecurityContext != nil {
  53. credSpec = config.Windows.SecurityContext.CredentialSpec
  54. }
  55. if credSpec == "" {
  56. return nil
  57. }
  58. valueName, err := copyGMSACredSpecToRegistryValue(credSpec)
  59. if err != nil {
  60. return err
  61. }
  62. if createConfig.HostConfig == nil {
  63. createConfig.HostConfig = &dockercontainer.HostConfig{}
  64. }
  65. createConfig.HostConfig.SecurityOpt = append(createConfig.HostConfig.SecurityOpt, "credentialspec=registry://"+valueName)
  66. cleanupInfo.gMSARegistryValueName = valueName
  67. return nil
  68. }
  69. const (
  70. // same as https://github.com/moby/moby/blob/93d994e29c9cc8d81f1b0477e28d705fa7e2cd72/daemon/oci_windows.go#L23
  71. credentialSpecRegistryLocation = `SOFTWARE\Microsoft\Windows NT\CurrentVersion\Virtualization\Containers\CredentialSpecs`
  72. // the prefix for the registry values we write GMSA cred specs to
  73. gMSARegistryValueNamePrefix = "k8s-cred-spec-"
  74. // the number of random bytes to generate suffixes for registry value names
  75. gMSARegistryValueNameSuffixRandomBytes = 40
  76. )
  77. // registryKey is an interface wrapper around `registry.Key`,
  78. // listing only the methods we care about here.
  79. // It's mainly useful to easily allow mocking the registry in tests.
  80. type registryKey interface {
  81. SetStringValue(name, value string) error
  82. DeleteValue(name string) error
  83. ReadValueNames(n int) ([]string, error)
  84. Close() error
  85. }
  86. var registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) {
  87. return registry.CreateKey(baseKey, path, access)
  88. }
  89. // randomReader is only meant to ever be overridden for testing purposes,
  90. // same idea as for `registryKey` above
  91. var randomReader = rand.Reader
  92. // gMSARegistryValueNamesRegex is the regex used to detect gMSA cred spec
  93. // registry values in `removeAllGMSARegistryValues` below.
  94. var gMSARegistryValueNamesRegex = regexp.MustCompile(fmt.Sprintf("^%s[0-9a-f]{%d}$", gMSARegistryValueNamePrefix, 2*gMSARegistryValueNameSuffixRandomBytes))
  95. // copyGMSACredSpecToRegistryKey copies the credential specs to a unique registry value, and returns its name.
  96. func copyGMSACredSpecToRegistryValue(credSpec string) (string, error) {
  97. valueName, err := gMSARegistryValueName()
  98. if err != nil {
  99. return "", err
  100. }
  101. // write to the registry
  102. key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
  103. if err != nil {
  104. return "", fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err)
  105. }
  106. defer key.Close()
  107. if err = key.SetStringValue(valueName, credSpec); err != nil {
  108. return "", fmt.Errorf("unable to write into registry value %q/%q: %v", credentialSpecRegistryLocation, valueName, err)
  109. }
  110. return valueName, nil
  111. }
  112. // gMSARegistryValueName computes the name of the registry value where to store the GMSA cred spec contents.
  113. // The value's name is a purely random suffix appended to `gMSARegistryValueNamePrefix`.
  114. func gMSARegistryValueName() (string, error) {
  115. randomSuffix, err := randomString(gMSARegistryValueNameSuffixRandomBytes)
  116. if err != nil {
  117. return "", fmt.Errorf("error when generating gMSA registry value name: %v", err)
  118. }
  119. return gMSARegistryValueNamePrefix + randomSuffix, nil
  120. }
  121. // randomString returns a random hex string.
  122. func randomString(length int) (string, error) {
  123. randBytes := make([]byte, length)
  124. if n, err := randomReader.Read(randBytes); err != nil || n != length {
  125. if err == nil {
  126. err = fmt.Errorf("only got %v random bytes, expected %v", n, length)
  127. }
  128. return "", fmt.Errorf("unable to generate random string: %v", err)
  129. }
  130. return hex.EncodeToString(randBytes), nil
  131. }
  132. // performPlatformSpecificContainerCleanup is responsible for doing any platform-specific cleanup
  133. // after either:
  134. // * the container creation has failed
  135. // * the container has been successfully started
  136. // * the container has been removed
  137. // whichever happens first.
  138. // Any errors it returns are simply logged, but do not prevent the container from being started or
  139. // removed.
  140. func (ds *dockerService) performPlatformSpecificContainerCleanup(cleanupInfo *containerCleanupInfo) (errors []error) {
  141. if err := removeGMSARegistryValue(cleanupInfo); err != nil {
  142. errors = append(errors, err)
  143. }
  144. return
  145. }
  146. func removeGMSARegistryValue(cleanupInfo *containerCleanupInfo) error {
  147. if cleanupInfo == nil || cleanupInfo.gMSARegistryValueName == "" {
  148. return nil
  149. }
  150. key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
  151. if err != nil {
  152. return fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err)
  153. }
  154. defer key.Close()
  155. if err = key.DeleteValue(cleanupInfo.gMSARegistryValueName); err != nil {
  156. return fmt.Errorf("unable to remove registry value %q/%q: %v", credentialSpecRegistryLocation, cleanupInfo.gMSARegistryValueName, err)
  157. }
  158. return nil
  159. }
  160. // platformSpecificContainerInitCleanup is called when dockershim
  161. // is starting, and is meant to clean up any cruft left by previous runs
  162. // creating containers.
  163. // Errors are simply logged, but don't prevent dockershim from starting.
  164. func (ds *dockerService) platformSpecificContainerInitCleanup() (errors []error) {
  165. return removeAllGMSARegistryValues()
  166. }
  167. func removeAllGMSARegistryValues() (errors []error) {
  168. key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
  169. if err != nil {
  170. return []error{fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err)}
  171. }
  172. defer key.Close()
  173. valueNames, err := key.ReadValueNames(0)
  174. if err != nil {
  175. return []error{fmt.Errorf("unable to list values under registry key %q: %v", credentialSpecRegistryLocation, err)}
  176. }
  177. for _, valueName := range valueNames {
  178. if gMSARegistryValueNamesRegex.MatchString(valueName) {
  179. if err = key.DeleteValue(valueName); err != nil {
  180. errors = append(errors, fmt.Errorf("unable to remove registry value %q/%q: %v", credentialSpecRegistryLocation, valueName, err))
  181. }
  182. }
  183. }
  184. return
  185. }