custom-kube-scheduler/kubernetes-v1.15.4/cmd/kubeadm/app/phases/selfhosting/selfhosting.go

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package selfhosting

import (
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"k8s.io/klog"

	"github.com/pkg/errors"
	apps "k8s.io/api/apps/v1"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	clientset "k8s.io/client-go/kubernetes"
	clientscheme "k8s.io/client-go/kubernetes/scheme"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)

const (
	// selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out
	selfHostingWaitTimeout = 2 * time.Minute

	// selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets
	selfHostingFailureThreshold int = 5
)

// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one
// It achieves that task this way:
// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests)
// 2. Extract the PodSpec from that Static Pod specification
// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly)
// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec
// 5. Create the DaemonSet resource. Wait until the Pods are running.
// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running.
// 7. The self-hosted containers should now step up and take over.
// 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue
//      Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes
// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool, certsInSecrets bool) error {
	klog.V(1).Infoln("creating self hosted control plane")
	// Adjust the timeout slightly to something self-hosting specific
	waiter.SetTimeout(selfHostingWaitTimeout)

	// Here the map of different mutators to use for the control plane's PodSpec is stored
	klog.V(1).Infoln("getting mutators")
	mutators := GetMutatorsFromFeatureGates(certsInSecrets)

	if certsInSecrets {
		// Upload the certificates and kubeconfig files from disk to the cluster as Secrets
		if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil {
			return err
		}
		if err := uploadKubeConfigSecrets(client, kubeConfigDir); err != nil {
			return err
		}
	}

	for _, componentName := range kubeadmconstants.ControlPlaneComponents {
		start := time.Now()
		manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, manifestsDir)

		// Since we want this function to be idempotent; just continue and try the next component if this file doesn't exist
		if _, err := os.Stat(manifestPath); err != nil {
			fmt.Printf("[self-hosted] The Static Pod for the component %q doesn't seem to be on the disk; trying the next one\n", componentName)
			continue
		}

		// Load the Static Pod spec in order to be able to create a self-hosted variant of that file
		podSpec, err := loadPodSpecFromFile(manifestPath)
		if err != nil {
			return err
		}

		// Build a DaemonSet object from the loaded PodSpec
		ds := BuildDaemonSet(componentName, podSpec, mutators)

		// Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out
		if err := apiclient.TryRunCommand(func() error {
			return apiclient.CreateOrUpdateDaemonSet(client, ds)
		}, selfHostingFailureThreshold); err != nil {
			return err
		}

		// Wait for the self-hosted component to come up
		if err := waiter.WaitForPodsWithLabel(BuildSelfHostedComponentLabelQuery(componentName)); err != nil {
			return err
		}

		// Remove the old Static Pod manifest if not dryrunning
		if !dryRun {
			if err := os.RemoveAll(manifestPath); err != nil {
				return errors.Wrapf(err, "unable to delete static pod manifest for %s ", componentName)
			}
		}

		// Wait for the mirror Pod hash to be removed; otherwise we'll run into race conditions here when the kubelet hasn't had time to
		// remove the Static Pod (or the mirror Pod respectively). This implicitly also tests that the API server endpoint is healthy,
		// because this blocks until the API server returns a 404 Not Found when getting the Static Pod
		staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeRegistration.Name)
		if err := waiter.WaitForPodToDisappear(staticPodName); err != nil {
			return err
		}

		// Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint (although we know it could return a GET answer for a Pod above)
		if err := waiter.WaitForAPI(); err != nil {
			return err
		}

		fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds())
	}
	return nil
}

// BuildDaemonSet is responsible for mutating the PodSpec and returns a DaemonSet which is suitable for self-hosting
func BuildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodSpecMutatorFunc) *apps.DaemonSet {

	// Mutate the PodSpec so it's suitable for self-hosting
	mutatePodSpec(mutators, name, podSpec)

	// Return a DaemonSet based on that Spec
	return &apps.DaemonSet{
		ObjectMeta: metav1.ObjectMeta{
			Name:      kubeadmconstants.AddSelfHostedPrefix(name),
			Namespace: metav1.NamespaceSystem,
			Labels:    BuildSelfhostedComponentLabels(name),
		},
		Spec: apps.DaemonSetSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: BuildSelfhostedComponentLabels(name),
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: BuildSelfhostedComponentLabels(name),
				},
				Spec: *podSpec,
			},
			UpdateStrategy: apps.DaemonSetUpdateStrategy{
				// Make the DaemonSet utilize the RollingUpdate rollout strategy
				Type: apps.RollingUpdateDaemonSetStrategyType,
			},
		},
	}
}

// BuildSelfhostedComponentLabels returns the labels for a self-hosted component
func BuildSelfhostedComponentLabels(component string) map[string]string {
	return map[string]string{
		"k8s-app": kubeadmconstants.AddSelfHostedPrefix(component),
	}
}

// BuildSelfHostedComponentLabelQuery creates the right query for matching a self-hosted Pod
func BuildSelfHostedComponentLabelQuery(componentName string) string {
	return fmt.Sprintf("k8s-app=%s", kubeadmconstants.AddSelfHostedPrefix(componentName))
}

func loadPodSpecFromFile(filePath string) (*v1.PodSpec, error) {
	podDef, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to read file path %s", filePath)
	}

	if len(podDef) == 0 {
		return nil, errors.Errorf("file was empty: %s", filePath)
	}

	codec := clientscheme.Codecs.UniversalDecoder()
	pod := &v1.Pod{}

	if err = runtime.DecodeInto(codec, podDef, pod); err != nil {
		return nil, errors.Wrap(err, "failed decoding pod")
	}

	return &pod.Spec, nil
}