custom-kube-scheduler/kubernetes/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package selfhosting

import (
	"path/filepath"
	"strings"

	v1 "k8s.io/api/core/v1"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)

const (
	// selfHostedKubeConfigDir sets the directory where kubeconfig files for the scheduler and controller-manager should be mounted
	// Due to how the projected volume mount works (can only be a full directory, not mount individual files), we must change this from
	// the default as mounts cannot be nested (/etc/kubernetes would override /etc/kubernetes/pki)
	selfHostedKubeConfigDir = "/etc/kubernetes/kubeconfig"
)

// PodSpecMutatorFunc is a function capable of mutating a PodSpec
type PodSpecMutatorFunc func(*v1.PodSpec)

// GetDefaultMutators gets the mutator functions that always should be used
func GetDefaultMutators() map[string][]PodSpecMutatorFunc {
	return map[string][]PodSpecMutatorFunc{
		kubeadmconstants.KubeAPIServer: {
			addNodeSelectorToPodSpec,
			setControlPlaneTolerationOnPodSpec,
			setRightDNSPolicyOnPodSpec,
			setHostIPOnPodSpec,
		},
		kubeadmconstants.KubeControllerManager: {
			addNodeSelectorToPodSpec,
			setControlPlaneTolerationOnPodSpec,
			setRightDNSPolicyOnPodSpec,
		},
		kubeadmconstants.KubeScheduler: {
			addNodeSelectorToPodSpec,
			setControlPlaneTolerationOnPodSpec,
			setRightDNSPolicyOnPodSpec,
		},
	}
}

// GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed
func GetMutatorsFromFeatureGates(certsInSecrets bool) map[string][]PodSpecMutatorFunc {
	// Here the map of different mutators to use for the control plane's podspec is stored
	mutators := GetDefaultMutators()

	if certsInSecrets {
		// Some extra work to be done if we should store the control plane certificates in Secrets
		// Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them
		mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer)
		mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager)
		mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler)
	}

	return mutators
}

// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) {
	// Get the mutator functions for the component in question, then loop through and execute them
	mutatorsForComponent := mutators[name]
	for _, mutateFunc := range mutatorsForComponent {
		mutateFunc(podSpec)
	}
}

// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the control-plane label
func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) {
	if podSpec.NodeSelector == nil {
		podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}
		return
	}

	podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = ""
}

// setControlPlaneTolerationOnPodSpec makes the Pod tolerate the control-plane taint
func setControlPlaneTolerationOnPodSpec(podSpec *v1.PodSpec) {
	if podSpec.Tolerations == nil {
		podSpec.Tolerations = []v1.Toleration{kubeadmconstants.ControlPlaneToleration}
		return
	}

	podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.ControlPlaneToleration)
}

// setHostIPOnPodSpec sets the environment variable HOST_IP using downward API
func setHostIPOnPodSpec(podSpec *v1.PodSpec) {
	envVar := v1.EnvVar{
		Name: "HOST_IP",
		ValueFrom: &v1.EnvVarSource{
			FieldRef: &v1.ObjectFieldSelector{
				FieldPath: "status.hostIP",
			},
		},
	}

	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, envVar)

	for i := range podSpec.Containers[0].Command {
		if strings.Contains(podSpec.Containers[0].Command[i], "advertise-address") {
			podSpec.Containers[0].Command[i] = "--advertise-address=$(HOST_IP)"
		}
	}
}

// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary
func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) {
	podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}

// setSelfHostedVolumesForAPIServer makes sure the self-hosted api server has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForAPIServer(podSpec *v1.PodSpec) {
	for i, v := range podSpec.Volumes {
		// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
		if v.Name == kubeadmconstants.KubeCertificatesVolumeName {
			podSpec.Volumes[i].VolumeSource = apiServerCertificatesVolumeSource()
		}
	}
}

// setSelfHostedVolumesForControllerManager makes sure the self-hosted controller manager has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForControllerManager(podSpec *v1.PodSpec) {
	for i, v := range podSpec.Volumes {
		// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
		if v.Name == kubeadmconstants.KubeCertificatesVolumeName {
			podSpec.Volumes[i].VolumeSource = controllerManagerCertificatesVolumeSource()
		} else if v.Name == kubeadmconstants.KubeConfigVolumeName {
			podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName)
		}
	}

	// Change directory for the kubeconfig directory to selfHostedKubeConfigDir
	for i, vm := range podSpec.Containers[0].VolumeMounts {
		if vm.Name == kubeadmconstants.KubeConfigVolumeName {
			podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir
		}
	}

	// Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki)
	// This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes
	// don't support that.
	podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string {
		controllerManagerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
		argMap["kubeconfig"] = controllerManagerKubeConfigPath
		if _, ok := argMap["authentication-kubeconfig"]; ok {
			argMap["authentication-kubeconfig"] = controllerManagerKubeConfigPath
		}
		if _, ok := argMap["authorization-kubeconfig"]; ok {
			argMap["authorization-kubeconfig"] = controllerManagerKubeConfigPath
		}
		return argMap
	})
}

// setSelfHostedVolumesForScheduler makes sure the self-hosted scheduler has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForScheduler(podSpec *v1.PodSpec) {
	for i, v := range podSpec.Volumes {
		// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
		if v.Name == kubeadmconstants.KubeConfigVolumeName {
			podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName)
		}
	}

	// Change directory for the kubeconfig directory to selfHostedKubeConfigDir
	for i, vm := range podSpec.Containers[0].VolumeMounts {
		if vm.Name == kubeadmconstants.KubeConfigVolumeName {
			podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir
		}
	}

	// Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki)
	// This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes
	// don't support that.
	podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string {
		schedulerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.SchedulerKubeConfigFileName)
		argMap["kubeconfig"] = schedulerKubeConfigPath
		if _, ok := argMap["authentication-kubeconfig"]; ok {
			argMap["authentication-kubeconfig"] = schedulerKubeConfigPath
		}
		if _, ok := argMap["authorization-kubeconfig"]; ok {
			argMap["authorization-kubeconfig"] = schedulerKubeConfigPath
		}
		return argMap
	})
}