custom-kube-scheduler/kubernetes/test/kubemark/iks/util.sh

#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..

# Creates a new kube-spawn cluster
function create-clusters {
	# shellcheck disable=SC2154	# Color defined in sourced script
	echo -e "${color_yellow}CHECKING CLUSTERS${color_norm}"
	if ibmcloud ks clusters | grep -Fq 'deleting'; then
		echo -n "Deleting old clusters"
	fi
	while ibmcloud ks clusters | grep -Fq 'deleting'
	do
		echo -n "."
		sleep 10
	done
	echo ""
	ibmcloud ks region-set us-east >/dev/null
	ibmcloud ks vlans wdc06 >/dev/null
	PRIVLAN=$(ibmcloud ks vlans wdc06 --json | jq '. | .[] | select(.type == "private") | .id' | sed -e "s/\"//g")
	PUBVLAN=$(ibmcloud ks vlans wdc06 --json | jq '. | .[] | select(.type == "public") | .id' | sed -e "s/\"//g")
	if ! ibmcloud ks clusters | grep -Fq 'kubeSpawnTester'; then
		echo "Creating spawning cluster"
		# make number and spec of node workers configurable
		# otherwise it can't afford tests like kubemark-5000
		# TODO: dynamically adjust the number and spec
		ibmcloud ks cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers "${NUM_NODES:-2}" --machine-type "${NODE_SIZE}" --name kubeSpawnTester
	fi
	if ! ibmcloud ks clusters | grep -Fq 'kubeMasterTester'; then
		echo "Creating master cluster"
		# if we can't make it a bare master (workers = 0)
		# then make workers = 1 with the smallest machine spec
		ibmcloud ks cluster-create --location "${CLUSTER_LOCATION}" --public-vlan "${PUBVLAN}" --private-vlan "${PRIVLAN}" --workers 1 --machine-type u2c.2x4 --name kubeMasterTester
	fi
	push-image
	if ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'; then
		# shellcheck disable=SC2154 # Color defined in sourced script
		echo -e "${color_cyan}Warning: new clusters may take up to 60 minutes to be ready${color_norm}"
		echo -n "Clusters loading"
	fi
	while ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'normal'
	do
		echo -n "."
		sleep 5
	done
	while ! ibmcloud ks clusters | grep 'kubeMasterTester' | grep -Fq 'normal'
	do
		echo -n "."
		sleep 5
	done
	echo -e "${color_yellow}CLUSTER CREATION COMPLETE${color_norm}"
}

# Builds and pushes image to registry
function push-image {
	if [[ "${ISBUILD}" = "y" ]]; then
		if ! ibmcloud cr namespaces | grep -Fq "${KUBE_NAMESPACE}"; then
			echo "Creating registry namespace"
			ibmcloud cr namespace-add "${KUBE_NAMESPACE}"
			echo "ibmcloud cr namespace-rm ${KUBE_NAMESPACE}" >> "${RESOURCE_DIRECTORY}/iks-namespacelist.sh"
		fi
		docker build -t "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_LOCATION}"
		docker tag "${KUBEMARK_INIT_TAG}" "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}"
		docker push "${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}/${PROJECT}:${KUBEMARK_IMAGE_TAG}"
		echo "Image pushed"
	else
		KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-brandondr96}"
		KUBE_NAMESPACE=""
	fi
}

# Allow user to use existing clusters if desired
function choose-clusters {
	echo -n -e "Do you want to use custom clusters? [y/N]${color_cyan}>${color_norm} "
	read -r USE_EXISTING
	if [[ "${USE_EXISTING}" = "y" ]]; then
		echo -e "${color_yellow}Enter path for desired hollow-node spawning cluster kubeconfig file:${color_norm}"
		read -r CUSTOM_SPAWN_CONFIG
		echo -e "${color_yellow}Enter path for desired hollow-node hosting cluster kubeconfig file:${color_norm}"
		read -r CUSTOM_MASTER_CONFIG
		push-image
	elif [[ "${USE_EXISTING}" = "N" ]]; then
		create-clusters
	else
		# shellcheck disable=SC2154 # Color defined in sourced script
		echo -e "${color_red}Invalid response, please try again:${color_norm}"
		choose-clusters
	fi
}

# Ensure secrets are correctly set
function set-registry-secrets {
	spawn-config
	kubectl get secret bluemix-default-secret-regional -o yaml | sed 's/default/kubemark/g' | kubectl -n kubemark create -f -
	kubectl patch serviceaccount -n kubemark default -p '{"imagePullSecrets": [{"name": "bluemix-kubemark-secret"}]}'
	kubectl -n kubemark get serviceaccounts default -o json | jq 'del(.metadata.resourceVersion)' | jq 'setpath(["imagePullSecrets"];[{"name":"bluemix-kubemark-secret-regional"}])' | kubectl -n kubemark replace serviceaccount default -f -
}

# Sets the hollow-node master
# Exported variables:
#   MASTER_IP - IP Address of the Kubemark master
function set-hollow-master {
	echo -e "${color_yellow}CONFIGURING MASTER${color_norm}"
	master-config
	MASTER_IP=$(grep server "$KUBECONFIG" | awk -F "/" '{print $3}')
	export MASTER_IP
}

# Set up master cluster environment
# Exported variables:
#   KUBECONFIG - Overrides default kube config for the purpose of setting up the Kubemark master components.
function master-config {
	if [[ "${USE_EXISTING}" = "y" ]]; then
		export KUBECONFIG=${CUSTOM_MASTER_CONFIG}
	else
		eval "$(ibmcloud ks cluster-config kubeMasterTester --admin | grep export)"
	fi
}

# Set up spawn cluster environment
# Exported variables:
#    KUBECONFIG - Overrides default kube config for the purpose of setting up the hollow-node cluster.
function spawn-config {
	if [[ "${USE_EXISTING}" = "y" ]]; then
		export KUBECONFIG=${CUSTOM_SPAWN_CONFIG}
	else
		eval "$(ibmcloud ks cluster-config kubeSpawnTester --admin | grep export)"
	fi
}

# Deletes existing clusters
function delete-clusters {
	echo "DELETING CLUSTERS"
	ibmcloud ks cluster-rm kubeSpawnTester
	ibmcloud ks cluster-rm kubeMasterTester
	while ! ibmcloud ks clusters | grep 'kubeSpawnTester' | grep -Fq 'deleting'
	do
		sleep 5
	done
	while ! ibmcloud ks clusters | grep 'kubeMasterTester' | grep -Fq 'deleting'
	do
		sleep 5
	done
	kubectl delete ns kubemark
}

# Login to cloud services
function complete-login {
	echo -e "${color_yellow}LOGGING INTO CLOUD SERVICES${color_norm}"
	echo -n -e "Do you have a federated IBM cloud login? [y/N]${color_cyan}>${color_norm} "
	read -r ISFED
	if [[ "${ISFED}" = "y" ]]; then
		ibmcloud login --sso -a "${REGISTRY_LOGIN_URL}"
	elif [[ "${ISFED}" = "N" ]]; then
		ibmcloud login -a "${REGISTRY_LOGIN_URL}"
	else
		echo -e "${color_red}Invalid response, please try again:${color_norm}"
		complete-login
	fi
	ibmcloud cr login
}

# Generate values to fill the hollow-node configuration templates.
# Exported variables:
#   KUBECTL - The name or path to the kubernetes client binary.
#   TEST_CLUSTER_API_CONTENT_TYPE - Defines the content-type of the requests used by the Kubemark components.
function generate-values {
	echo "Generating values"
	master-config
	KUBECTL=kubectl
	export KUBECTL
	KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
	RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
	TEST_CLUSTER_API_CONTENT_TYPE="bluemix" #Determine correct usage of this
	export TEST_CLUSTER_API_CONTENT_TYPE
	CONFIGPATH=${KUBECONFIG%/*}
	KUBELET_CERT_BASE64="${KUBELET_CERT_BASE64:-$(base64 "${CONFIGPATH}/admin.pem" | tr -d '\r\n')}"
	KUBELET_KEY_BASE64="${KUBELET_KEY_BASE64:-$(base64 "${CONFIGPATH}/admin-key.pem" | tr -d '\r\n')}"
	CA_CERT_BASE64="${CA_CERT_BASE64:-$( base64 "$(find "${CONFIGPATH}" -name "*ca*" | head -n 1)" | tr -d '\r\n')}"

}

# Build image for kubemark
function build-kubemark-image {
	echo -n -e "Do you want to build the kubemark image? [y/N]${color_cyan}>${color_norm} "
	read -r ISBUILD
	if [[ "${ISBUILD}" = "y" ]]; then
		echo -e "${color_yellow}BUILDING IMAGE${color_norm}"
		"${KUBE_ROOT}/build/run.sh" make kubemark
		cp "${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/kubemark" "${KUBEMARK_IMAGE_LOCATION}"
	elif [[ "${ISBUILD}" = "N" ]]; then
		echo -n ""
	else
		echo -e "${color_red}Invalid response, please try again:${color_norm}"
		build-kubemark-image
	fi
}

# Clean up repository
function clean-repo {
	echo -n -e "Do you want to remove build output and binary? [y/N]${color_cyan}>${color_norm} "
	read -r ISCLEAN
	if [[ "${ISCLEAN}" = "y" ]]; then
		echo -e "${color_yellow}CLEANING REPO${color_norm}"
		rm -rf "${KUBE_ROOT}/_output"
		rm -f "${KUBEMARK_IMAGE_LOCATION}/kubemark"
	elif [[ "${ISCLEAN}" = "N" ]]; then
		echo -n ""
	else
		echo -e "${color_red}Invalid response, please try again:${color_norm}"
		clean-repo
	fi
}