gmsa_full.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399
  1. /*
  2. Copyright 2019 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. // This test ensures that the whole GMSA process works as intended.
  14. // However, it does require a pretty heavy weight set up to run correctly;
  15. // in particular, it does make a number of assumptions about the cluster it
  16. // runs against:
  17. // * there exists a Windows worker node with the agentpool=windowsgmsa label on it
  18. // * that node is joined to a working Active Directory domain.
  19. // * a GMSA account has been created in that AD domain, and then installed on that
  20. // same worker.
  21. // * a valid k8s manifest file containing a single CRD definition has been generated using
  22. // https://github.com/kubernetes-sigs/windows-gmsa/blob/master/scripts/GenerateCredentialSpecResource.ps1
  23. // with the credential specs of that GMSA account, or type GMSACredentialSpec and named gmsa-e2e;
  24. // and that manifest file has been written to C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml
  25. // on that same worker node.
  26. // * the API has both MutatingAdmissionWebhook and ValidatingAdmissionWebhook
  27. // admission controllers enabled.
  28. // * the cluster comprises at least one Linux node that accepts workloads - it
  29. // can be the master, but any other Linux node is fine too. This is needed for
  30. // the webhook's pod.
  31. // All these assumptions are fulfilled by an AKS extension when setting up the AKS
  32. // cluster we run daily e2e tests against, but they do make running this test
  33. // outside of that very specific context pretty hard.
  34. package windows
  35. import (
  36. "context"
  37. "fmt"
  38. "io/ioutil"
  39. "os"
  40. "os/exec"
  41. "path"
  42. "strings"
  43. "time"
  44. v1 "k8s.io/api/core/v1"
  45. rbacv1 "k8s.io/api/rbac/v1"
  46. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  47. clientset "k8s.io/client-go/kubernetes"
  48. "k8s.io/kubernetes/test/e2e/framework"
  49. e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
  50. imageutils "k8s.io/kubernetes/test/utils/image"
  51. "github.com/onsi/ginkgo"
  52. "github.com/onsi/gomega"
  53. "github.com/pkg/errors"
  54. )
  55. const (
  56. // gmsaFullNodeLabel is the label we expect to find on at least one node
  57. // that is then expected to fulfill all the expectations explained above.
  58. gmsaFullNodeLabel = "agentpool=windowsgmsa"
  59. // gmsaCrdManifestPath is where we expect to find the manifest file for
  60. // the GMSA cred spec on that node - see explanations above.
  61. gmsaCrdManifestPath = `C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml`
  62. // gmsaCustomResourceName is the expected name of the GMSA custom resource
  63. // defined at gmsaCrdManifestPath
  64. gmsaCustomResourceName = "gmsa-e2e"
  65. // gmsaWebhookDeployScriptURL is the URL of the deploy script for the GMSA webook
  66. // TODO(wk8): we should pin versions.
  67. gmsaWebhookDeployScriptURL = "https://raw.githubusercontent.com/kubernetes-sigs/windows-gmsa/master/admission-webhook/deploy/deploy-gmsa-webhook.sh"
  68. )
  69. var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA Full [Slow]", func() {
  70. f := framework.NewDefaultFramework("gmsa-full-test-windows")
  71. ginkgo.Describe("GMSA support", func() {
  72. ginkgo.It("works end to end", func() {
  73. defer ginkgo.GinkgoRecover()
  74. ginkgo.By("finding the worker node that fulfills this test's assumptions")
  75. nodes := findPreconfiguredGmsaNodes(f.ClientSet)
  76. if len(nodes) != 1 {
  77. e2eskipper.Skipf("Expected to find exactly one node with the %q label, found %d", gmsaFullNodeLabel, len(nodes))
  78. }
  79. node := nodes[0]
  80. ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node")
  81. crdManifestContents := retrieveCRDManifestFileContents(f, node)
  82. ginkgo.By("downloading the GMSA webhook deploy script")
  83. deployScriptPath, err := downloadFile(gmsaWebhookDeployScriptURL)
  84. defer func() { os.Remove(deployScriptPath) }()
  85. if err != nil {
  86. framework.Failf(err.Error())
  87. }
  88. ginkgo.By("deploying the GMSA webhook")
  89. webhookCleanUp, err := deployGmsaWebhook(f, deployScriptPath)
  90. defer webhookCleanUp()
  91. if err != nil {
  92. framework.Failf(err.Error())
  93. }
  94. ginkgo.By("creating the GMSA custom resource")
  95. customResourceCleanup, err := createGmsaCustomResource(f.Namespace.Name, crdManifestContents)
  96. defer customResourceCleanup()
  97. if err != nil {
  98. framework.Failf(err.Error())
  99. }
  100. ginkgo.By("creating an RBAC role to grant use access to that GMSA resource")
  101. rbacRoleName, rbacRoleCleanup, err := createRBACRoleForGmsa(f)
  102. defer rbacRoleCleanup()
  103. if err != nil {
  104. framework.Failf(err.Error())
  105. }
  106. ginkgo.By("creating a service account")
  107. serviceAccountName := createServiceAccount(f)
  108. ginkgo.By("binding the RBAC role to the service account")
  109. bindRBACRoleToServiceAccount(f, serviceAccountName, rbacRoleName)
  110. ginkgo.By("creating a pod using the GMSA cred spec")
  111. podName := createPodWithGmsa(f, serviceAccountName)
  112. // nltest /QUERY will only return successfully if there is a GMSA
  113. // identity configured, _and_ it succeeds in contacting the AD controller
  114. // and authenticating with it.
  115. ginkgo.By("checking that nltest /QUERY returns successfully")
  116. var output string
  117. gomega.Eventually(func() bool {
  118. output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", "/QUERY")
  119. return err == nil
  120. }, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue())
  121. expectedSubstr := "The command completed successfully"
  122. if !strings.Contains(output, expectedSubstr) {
  123. framework.Failf("Expected %q to contain %q", output, expectedSubstr)
  124. }
  125. })
  126. })
  127. })
  128. // findPreconfiguredGmsaNode finds node with the gmsaFullNodeLabel label on it.
  129. func findPreconfiguredGmsaNodes(c clientset.Interface) []v1.Node {
  130. nodeOpts := metav1.ListOptions{
  131. LabelSelector: gmsaFullNodeLabel,
  132. }
  133. nodes, err := c.CoreV1().Nodes().List(context.TODO(), nodeOpts)
  134. if err != nil {
  135. framework.Failf("Unable to list nodes: %v", err)
  136. }
  137. return nodes.Items
  138. }
  139. // retrieveCRDManifestFileContents retrieves the contents of the file
  140. // at gmsaCrdManifestPath on node; it does so by scheduling a single pod
  141. // on nodes with the gmsaFullNodeLabel label with that file's directory
  142. // mounted on it, and then exec-ing into that pod to retrieve the file's
  143. // contents.
  144. func retrieveCRDManifestFileContents(f *framework.Framework, node v1.Node) string {
  145. podName := "retrieve-gmsa-crd-contents"
  146. // we can't use filepath.Dir here since the test itself runs on a Linux machine
  147. splitPath := strings.Split(gmsaCrdManifestPath, `\`)
  148. dirPath := strings.Join(splitPath[:len(splitPath)-1], `\`)
  149. volumeName := "retrieve-gmsa-crd-contents-volume"
  150. pod := &v1.Pod{
  151. ObjectMeta: metav1.ObjectMeta{
  152. Name: podName,
  153. Namespace: f.Namespace.Name,
  154. },
  155. Spec: v1.PodSpec{
  156. NodeSelector: node.Labels,
  157. Containers: []v1.Container{
  158. {
  159. Name: podName,
  160. Image: imageutils.GetPauseImageName(),
  161. VolumeMounts: []v1.VolumeMount{
  162. {
  163. Name: volumeName,
  164. MountPath: dirPath,
  165. },
  166. },
  167. },
  168. },
  169. Volumes: []v1.Volume{
  170. {
  171. Name: volumeName,
  172. VolumeSource: v1.VolumeSource{
  173. HostPath: &v1.HostPathVolumeSource{
  174. Path: dirPath,
  175. },
  176. },
  177. },
  178. },
  179. },
  180. }
  181. f.PodClient().CreateSync(pod)
  182. // using powershell and using forward slashes avoids the nightmare of having to properly
  183. // escape quotes and backward slashes
  184. output, err := runKubectlExecInNamespace(f.Namespace.Name, podName, "powershell", "Get-Content", strings.ReplaceAll(gmsaCrdManifestPath, `\`, "/"))
  185. if err != nil {
  186. framework.Failf("failed to retrieve the contents of %q on node %q: %v", gmsaCrdManifestPath, node.Name, err)
  187. }
  188. // Windows to linux new lines
  189. return strings.ReplaceAll(output, "\r\n", "\n")
  190. }
  191. // deployGmsaWebhook deploys the GMSA webhook, and returns a cleanup function
  192. // to be called when done with testing, that removes the temp files it's created
  193. // on disks as well as the API resources it's created.
  194. func deployGmsaWebhook(f *framework.Framework, deployScriptPath string) (func(), error) {
  195. cleanUpFunc := func() {}
  196. tempDir, err := ioutil.TempDir("", "")
  197. if err != nil {
  198. return cleanUpFunc, errors.Wrapf(err, "unable to create temp dir")
  199. }
  200. manifestsFile := path.Join(tempDir, "manifests.yml")
  201. name := "gmsa-webhook"
  202. namespace := f.Namespace.Name + "-webhook"
  203. certsDir := path.Join(tempDir, "certs")
  204. // regardless of whether the deployment succeeded, let's do a best effort at cleanup
  205. cleanUpFunc = func() {
  206. framework.RunKubectl(f.Namespace.Name, "delete", "--filename", manifestsFile)
  207. framework.RunKubectl(f.Namespace.Name, "delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io")
  208. framework.RunKubectl(f.Namespace.Name, "delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", name, namespace))
  209. os.RemoveAll(tempDir)
  210. }
  211. cmd := exec.Command("bash", deployScriptPath,
  212. "--file", manifestsFile,
  213. "--name", name,
  214. "--namespace", namespace,
  215. "--certs-dir", certsDir,
  216. "--tolerate-master")
  217. output, err := cmd.CombinedOutput()
  218. if err == nil {
  219. framework.Logf("GMSA webhook successfully deployed, output:\n%s", string(output))
  220. } else {
  221. err = errors.Wrapf(err, "unable to deploy GMSA webhook, output:\n%s", string(output))
  222. }
  223. return cleanUpFunc, err
  224. }
  225. // createGmsaCustomResource creates the GMSA API object from the contents
  226. // of the manifest file retrieved from the worker node.
  227. // It returns a function to clean up both the temp file it creates and
  228. // the API object it creates when done with testing.
  229. func createGmsaCustomResource(ns string, crdManifestContents string) (func(), error) {
  230. cleanUpFunc := func() {}
  231. tempFile, err := ioutil.TempFile("", "")
  232. if err != nil {
  233. return cleanUpFunc, errors.Wrapf(err, "unable to create temp file")
  234. }
  235. defer tempFile.Close()
  236. cleanUpFunc = func() {
  237. framework.RunKubectl(ns, "delete", "--filename", tempFile.Name())
  238. os.Remove(tempFile.Name())
  239. }
  240. _, err = tempFile.WriteString(crdManifestContents)
  241. if err != nil {
  242. err = errors.Wrapf(err, "unable to write GMSA contents to %q", tempFile.Name())
  243. return cleanUpFunc, err
  244. }
  245. output, err := framework.RunKubectl(ns, "apply", "--filename", tempFile.Name())
  246. if err != nil {
  247. err = errors.Wrapf(err, "unable to create custom resource, output:\n%s", output)
  248. }
  249. return cleanUpFunc, err
  250. }
  251. // createRBACRoleForGmsa creates an RBAC cluster role to grant use
  252. // access to our test credential spec.
  253. // It returns the role's name, as well as a function to delete it when done.
  254. func createRBACRoleForGmsa(f *framework.Framework) (string, func(), error) {
  255. roleName := f.Namespace.Name + "-rbac-role"
  256. role := &rbacv1.ClusterRole{
  257. ObjectMeta: metav1.ObjectMeta{
  258. Name: roleName,
  259. },
  260. Rules: []rbacv1.PolicyRule{
  261. {
  262. APIGroups: []string{"windows.k8s.io"},
  263. Resources: []string{"gmsacredentialspecs"},
  264. Verbs: []string{"use"},
  265. ResourceNames: []string{gmsaCustomResourceName},
  266. },
  267. },
  268. }
  269. cleanUpFunc := func() {
  270. f.ClientSet.RbacV1().ClusterRoles().Delete(context.TODO(), roleName, &metav1.DeleteOptions{})
  271. }
  272. _, err := f.ClientSet.RbacV1().ClusterRoles().Create(context.TODO(), role, metav1.CreateOptions{})
  273. if err != nil {
  274. err = errors.Wrapf(err, "unable to create RBAC cluster role %q", roleName)
  275. }
  276. return roleName, cleanUpFunc, err
  277. }
  278. // createServiceAccount creates a service account, and returns its name.
  279. func createServiceAccount(f *framework.Framework) string {
  280. accountName := f.Namespace.Name + "-sa"
  281. account := &v1.ServiceAccount{
  282. ObjectMeta: metav1.ObjectMeta{
  283. Name: accountName,
  284. Namespace: f.Namespace.Name,
  285. },
  286. }
  287. if _, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(context.TODO(), account, metav1.CreateOptions{}); err != nil {
  288. framework.Failf("unable to create service account %q: %v", accountName, err)
  289. }
  290. return accountName
  291. }
  292. // bindRBACRoleToServiceAccount binds the given RBAC cluster role to the given service account.
  293. func bindRBACRoleToServiceAccount(f *framework.Framework, serviceAccountName, rbacRoleName string) {
  294. binding := &rbacv1.RoleBinding{
  295. ObjectMeta: metav1.ObjectMeta{
  296. Name: f.Namespace.Name + "-rbac-binding",
  297. Namespace: f.Namespace.Name,
  298. },
  299. Subjects: []rbacv1.Subject{
  300. {
  301. Kind: "ServiceAccount",
  302. Name: serviceAccountName,
  303. Namespace: f.Namespace.Name,
  304. },
  305. },
  306. RoleRef: rbacv1.RoleRef{
  307. APIGroup: "rbac.authorization.k8s.io",
  308. Kind: "ClusterRole",
  309. Name: rbacRoleName,
  310. },
  311. }
  312. f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(context.TODO(), binding, metav1.CreateOptions{})
  313. }
  314. // createPodWithGmsa creates a pod using the test GMSA cred spec, and returns its name.
  315. func createPodWithGmsa(f *framework.Framework, serviceAccountName string) string {
  316. podName := "pod-with-gmsa"
  317. credSpecName := gmsaCustomResourceName
  318. pod := &v1.Pod{
  319. ObjectMeta: metav1.ObjectMeta{
  320. Name: podName,
  321. Namespace: f.Namespace.Name,
  322. },
  323. Spec: v1.PodSpec{
  324. ServiceAccountName: serviceAccountName,
  325. Containers: []v1.Container{
  326. {
  327. Name: podName,
  328. Image: imageutils.GetPauseImageName(),
  329. },
  330. },
  331. SecurityContext: &v1.PodSecurityContext{
  332. WindowsOptions: &v1.WindowsSecurityContextOptions{
  333. GMSACredentialSpecName: &credSpecName,
  334. },
  335. },
  336. },
  337. }
  338. f.PodClient().CreateSync(pod)
  339. return podName
  340. }
  341. func runKubectlExecInNamespace(namespace string, args ...string) (string, error) {
  342. namespaceOption := fmt.Sprintf("--namespace=%s", namespace)
  343. return framework.RunKubectl(namespace, append([]string{"exec", namespaceOption}, args...)...)
  344. }