123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399 |
- /*
- Copyright 2019 The Kubernetes Authors.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
- // This test ensures that the whole GMSA process works as intended.
- // However, it does require a pretty heavy weight set up to run correctly;
- // in particular, it does make a number of assumptions about the cluster it
- // runs against:
- // * there exists a Windows worker node with the agentpool=windowsgmsa label on it
- // * that node is joined to a working Active Directory domain.
- // * a GMSA account has been created in that AD domain, and then installed on that
- // same worker.
- // * a valid k8s manifest file containing a single CRD definition has been generated using
- // https://github.com/kubernetes-sigs/windows-gmsa/blob/master/scripts/GenerateCredentialSpecResource.ps1
- // with the credential specs of that GMSA account, or type GMSACredentialSpec and named gmsa-e2e;
- // and that manifest file has been written to C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml
- // on that same worker node.
- // * the API has both MutatingAdmissionWebhook and ValidatingAdmissionWebhook
- // admission controllers enabled.
- // * the cluster comprises at least one Linux node that accepts workloads - it
- // can be the master, but any other Linux node is fine too. This is needed for
- // the webhook's pod.
- // All these assumptions are fulfilled by an AKS extension when setting up the AKS
- // cluster we run daily e2e tests against, but they do make running this test
- // outside of that very specific context pretty hard.
- package windows
- import (
- "context"
- "fmt"
- "io/ioutil"
- "os"
- "os/exec"
- "path"
- "strings"
- "time"
- v1 "k8s.io/api/core/v1"
- rbacv1 "k8s.io/api/rbac/v1"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- clientset "k8s.io/client-go/kubernetes"
- "k8s.io/kubernetes/test/e2e/framework"
- e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
- imageutils "k8s.io/kubernetes/test/utils/image"
- "github.com/onsi/ginkgo"
- "github.com/onsi/gomega"
- "github.com/pkg/errors"
- )
- const (
- // gmsaFullNodeLabel is the label we expect to find on at least one node
- // that is then expected to fulfill all the expectations explained above.
- gmsaFullNodeLabel = "agentpool=windowsgmsa"
- // gmsaCrdManifestPath is where we expect to find the manifest file for
- // the GMSA cred spec on that node - see explanations above.
- gmsaCrdManifestPath = `C:\gmsa\gmsa-cred-spec-gmsa-e2e.yml`
- // gmsaCustomResourceName is the expected name of the GMSA custom resource
- // defined at gmsaCrdManifestPath
- gmsaCustomResourceName = "gmsa-e2e"
- // gmsaWebhookDeployScriptURL is the URL of the deploy script for the GMSA webook
- // TODO(wk8): we should pin versions.
- gmsaWebhookDeployScriptURL = "https://raw.githubusercontent.com/kubernetes-sigs/windows-gmsa/master/admission-webhook/deploy/deploy-gmsa-webhook.sh"
- )
- var _ = SIGDescribe("[Feature:Windows] [Feature:WindowsGMSA] GMSA Full [Slow]", func() {
- f := framework.NewDefaultFramework("gmsa-full-test-windows")
- ginkgo.Describe("GMSA support", func() {
- ginkgo.It("works end to end", func() {
- defer ginkgo.GinkgoRecover()
- ginkgo.By("finding the worker node that fulfills this test's assumptions")
- nodes := findPreconfiguredGmsaNodes(f.ClientSet)
- if len(nodes) != 1 {
- e2eskipper.Skipf("Expected to find exactly one node with the %q label, found %d", gmsaFullNodeLabel, len(nodes))
- }
- node := nodes[0]
- ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node")
- crdManifestContents := retrieveCRDManifestFileContents(f, node)
- ginkgo.By("downloading the GMSA webhook deploy script")
- deployScriptPath, err := downloadFile(gmsaWebhookDeployScriptURL)
- defer func() { os.Remove(deployScriptPath) }()
- if err != nil {
- framework.Failf(err.Error())
- }
- ginkgo.By("deploying the GMSA webhook")
- webhookCleanUp, err := deployGmsaWebhook(f, deployScriptPath)
- defer webhookCleanUp()
- if err != nil {
- framework.Failf(err.Error())
- }
- ginkgo.By("creating the GMSA custom resource")
- customResourceCleanup, err := createGmsaCustomResource(f.Namespace.Name, crdManifestContents)
- defer customResourceCleanup()
- if err != nil {
- framework.Failf(err.Error())
- }
- ginkgo.By("creating an RBAC role to grant use access to that GMSA resource")
- rbacRoleName, rbacRoleCleanup, err := createRBACRoleForGmsa(f)
- defer rbacRoleCleanup()
- if err != nil {
- framework.Failf(err.Error())
- }
- ginkgo.By("creating a service account")
- serviceAccountName := createServiceAccount(f)
- ginkgo.By("binding the RBAC role to the service account")
- bindRBACRoleToServiceAccount(f, serviceAccountName, rbacRoleName)
- ginkgo.By("creating a pod using the GMSA cred spec")
- podName := createPodWithGmsa(f, serviceAccountName)
- // nltest /QUERY will only return successfully if there is a GMSA
- // identity configured, _and_ it succeeds in contacting the AD controller
- // and authenticating with it.
- ginkgo.By("checking that nltest /QUERY returns successfully")
- var output string
- gomega.Eventually(func() bool {
- output, err = runKubectlExecInNamespace(f.Namespace.Name, podName, "nltest", "/QUERY")
- return err == nil
- }, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue())
- expectedSubstr := "The command completed successfully"
- if !strings.Contains(output, expectedSubstr) {
- framework.Failf("Expected %q to contain %q", output, expectedSubstr)
- }
- })
- })
- })
- // findPreconfiguredGmsaNode finds node with the gmsaFullNodeLabel label on it.
- func findPreconfiguredGmsaNodes(c clientset.Interface) []v1.Node {
- nodeOpts := metav1.ListOptions{
- LabelSelector: gmsaFullNodeLabel,
- }
- nodes, err := c.CoreV1().Nodes().List(context.TODO(), nodeOpts)
- if err != nil {
- framework.Failf("Unable to list nodes: %v", err)
- }
- return nodes.Items
- }
- // retrieveCRDManifestFileContents retrieves the contents of the file
- // at gmsaCrdManifestPath on node; it does so by scheduling a single pod
- // on nodes with the gmsaFullNodeLabel label with that file's directory
- // mounted on it, and then exec-ing into that pod to retrieve the file's
- // contents.
- func retrieveCRDManifestFileContents(f *framework.Framework, node v1.Node) string {
- podName := "retrieve-gmsa-crd-contents"
- // we can't use filepath.Dir here since the test itself runs on a Linux machine
- splitPath := strings.Split(gmsaCrdManifestPath, `\`)
- dirPath := strings.Join(splitPath[:len(splitPath)-1], `\`)
- volumeName := "retrieve-gmsa-crd-contents-volume"
- pod := &v1.Pod{
- ObjectMeta: metav1.ObjectMeta{
- Name: podName,
- Namespace: f.Namespace.Name,
- },
- Spec: v1.PodSpec{
- NodeSelector: node.Labels,
- Containers: []v1.Container{
- {
- Name: podName,
- Image: imageutils.GetPauseImageName(),
- VolumeMounts: []v1.VolumeMount{
- {
- Name: volumeName,
- MountPath: dirPath,
- },
- },
- },
- },
- Volumes: []v1.Volume{
- {
- Name: volumeName,
- VolumeSource: v1.VolumeSource{
- HostPath: &v1.HostPathVolumeSource{
- Path: dirPath,
- },
- },
- },
- },
- },
- }
- f.PodClient().CreateSync(pod)
- // using powershell and using forward slashes avoids the nightmare of having to properly
- // escape quotes and backward slashes
- output, err := runKubectlExecInNamespace(f.Namespace.Name, podName, "powershell", "Get-Content", strings.ReplaceAll(gmsaCrdManifestPath, `\`, "/"))
- if err != nil {
- framework.Failf("failed to retrieve the contents of %q on node %q: %v", gmsaCrdManifestPath, node.Name, err)
- }
- // Windows to linux new lines
- return strings.ReplaceAll(output, "\r\n", "\n")
- }
- // deployGmsaWebhook deploys the GMSA webhook, and returns a cleanup function
- // to be called when done with testing, that removes the temp files it's created
- // on disks as well as the API resources it's created.
- func deployGmsaWebhook(f *framework.Framework, deployScriptPath string) (func(), error) {
- cleanUpFunc := func() {}
- tempDir, err := ioutil.TempDir("", "")
- if err != nil {
- return cleanUpFunc, errors.Wrapf(err, "unable to create temp dir")
- }
- manifestsFile := path.Join(tempDir, "manifests.yml")
- name := "gmsa-webhook"
- namespace := f.Namespace.Name + "-webhook"
- certsDir := path.Join(tempDir, "certs")
- // regardless of whether the deployment succeeded, let's do a best effort at cleanup
- cleanUpFunc = func() {
- framework.RunKubectl(f.Namespace.Name, "delete", "--filename", manifestsFile)
- framework.RunKubectl(f.Namespace.Name, "delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io")
- framework.RunKubectl(f.Namespace.Name, "delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", name, namespace))
- os.RemoveAll(tempDir)
- }
- cmd := exec.Command("bash", deployScriptPath,
- "--file", manifestsFile,
- "--name", name,
- "--namespace", namespace,
- "--certs-dir", certsDir,
- "--tolerate-master")
- output, err := cmd.CombinedOutput()
- if err == nil {
- framework.Logf("GMSA webhook successfully deployed, output:\n%s", string(output))
- } else {
- err = errors.Wrapf(err, "unable to deploy GMSA webhook, output:\n%s", string(output))
- }
- return cleanUpFunc, err
- }
- // createGmsaCustomResource creates the GMSA API object from the contents
- // of the manifest file retrieved from the worker node.
- // It returns a function to clean up both the temp file it creates and
- // the API object it creates when done with testing.
- func createGmsaCustomResource(ns string, crdManifestContents string) (func(), error) {
- cleanUpFunc := func() {}
- tempFile, err := ioutil.TempFile("", "")
- if err != nil {
- return cleanUpFunc, errors.Wrapf(err, "unable to create temp file")
- }
- defer tempFile.Close()
- cleanUpFunc = func() {
- framework.RunKubectl(ns, "delete", "--filename", tempFile.Name())
- os.Remove(tempFile.Name())
- }
- _, err = tempFile.WriteString(crdManifestContents)
- if err != nil {
- err = errors.Wrapf(err, "unable to write GMSA contents to %q", tempFile.Name())
- return cleanUpFunc, err
- }
- output, err := framework.RunKubectl(ns, "apply", "--filename", tempFile.Name())
- if err != nil {
- err = errors.Wrapf(err, "unable to create custom resource, output:\n%s", output)
- }
- return cleanUpFunc, err
- }
- // createRBACRoleForGmsa creates an RBAC cluster role to grant use
- // access to our test credential spec.
- // It returns the role's name, as well as a function to delete it when done.
- func createRBACRoleForGmsa(f *framework.Framework) (string, func(), error) {
- roleName := f.Namespace.Name + "-rbac-role"
- role := &rbacv1.ClusterRole{
- ObjectMeta: metav1.ObjectMeta{
- Name: roleName,
- },
- Rules: []rbacv1.PolicyRule{
- {
- APIGroups: []string{"windows.k8s.io"},
- Resources: []string{"gmsacredentialspecs"},
- Verbs: []string{"use"},
- ResourceNames: []string{gmsaCustomResourceName},
- },
- },
- }
- cleanUpFunc := func() {
- f.ClientSet.RbacV1().ClusterRoles().Delete(context.TODO(), roleName, &metav1.DeleteOptions{})
- }
- _, err := f.ClientSet.RbacV1().ClusterRoles().Create(context.TODO(), role, metav1.CreateOptions{})
- if err != nil {
- err = errors.Wrapf(err, "unable to create RBAC cluster role %q", roleName)
- }
- return roleName, cleanUpFunc, err
- }
- // createServiceAccount creates a service account, and returns its name.
- func createServiceAccount(f *framework.Framework) string {
- accountName := f.Namespace.Name + "-sa"
- account := &v1.ServiceAccount{
- ObjectMeta: metav1.ObjectMeta{
- Name: accountName,
- Namespace: f.Namespace.Name,
- },
- }
- if _, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(context.TODO(), account, metav1.CreateOptions{}); err != nil {
- framework.Failf("unable to create service account %q: %v", accountName, err)
- }
- return accountName
- }
- // bindRBACRoleToServiceAccount binds the given RBAC cluster role to the given service account.
- func bindRBACRoleToServiceAccount(f *framework.Framework, serviceAccountName, rbacRoleName string) {
- binding := &rbacv1.RoleBinding{
- ObjectMeta: metav1.ObjectMeta{
- Name: f.Namespace.Name + "-rbac-binding",
- Namespace: f.Namespace.Name,
- },
- Subjects: []rbacv1.Subject{
- {
- Kind: "ServiceAccount",
- Name: serviceAccountName,
- Namespace: f.Namespace.Name,
- },
- },
- RoleRef: rbacv1.RoleRef{
- APIGroup: "rbac.authorization.k8s.io",
- Kind: "ClusterRole",
- Name: rbacRoleName,
- },
- }
- f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(context.TODO(), binding, metav1.CreateOptions{})
- }
- // createPodWithGmsa creates a pod using the test GMSA cred spec, and returns its name.
- func createPodWithGmsa(f *framework.Framework, serviceAccountName string) string {
- podName := "pod-with-gmsa"
- credSpecName := gmsaCustomResourceName
- pod := &v1.Pod{
- ObjectMeta: metav1.ObjectMeta{
- Name: podName,
- Namespace: f.Namespace.Name,
- },
- Spec: v1.PodSpec{
- ServiceAccountName: serviceAccountName,
- Containers: []v1.Container{
- {
- Name: podName,
- Image: imageutils.GetPauseImageName(),
- },
- },
- SecurityContext: &v1.PodSecurityContext{
- WindowsOptions: &v1.WindowsSecurityContextOptions{
- GMSACredentialSpecName: &credSpecName,
- },
- },
- },
- }
- f.PodClient().CreateSync(pod)
- return podName
- }
- func runKubectlExecInNamespace(namespace string, args ...string) (string, error) {
- namespaceOption := fmt.Sprintf("--namespace=%s", namespace)
- return framework.RunKubectl(namespace, append([]string{"exec", namespaceOption}, args...)...)
- }
|