manifests_test.go 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147
  1. /*
  2. Copyright 2016 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package controlplane
  14. import (
  15. "fmt"
  16. "io/ioutil"
  17. "os"
  18. "path/filepath"
  19. "reflect"
  20. "sort"
  21. "strings"
  22. "testing"
  23. "github.com/lithammer/dedent"
  24. "k8s.io/apimachinery/pkg/util/sets"
  25. kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
  26. kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
  27. "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
  28. staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
  29. testutil "k8s.io/kubernetes/cmd/kubeadm/test"
  30. "k8s.io/kubernetes/pkg/features"
  31. )
  32. const (
  33. testCertsDir = "/var/lib/certs"
  34. )
  35. var cpVersion = kubeadmconstants.MinimumControlPlaneVersion.WithPreRelease("beta.2").String()
  36. func TestGetStaticPodSpecs(t *testing.T) {
  37. // Creates a Cluster Configuration
  38. cfg := &kubeadmapi.ClusterConfiguration{
  39. KubernetesVersion: "v1.9.0",
  40. }
  41. // Executes GetStaticPodSpecs
  42. specs := GetStaticPodSpecs(cfg, &kubeadmapi.APIEndpoint{})
  43. var tests = []struct {
  44. name string
  45. staticPodName string
  46. }{
  47. {
  48. name: "KubeAPIServer",
  49. staticPodName: kubeadmconstants.KubeAPIServer,
  50. },
  51. {
  52. name: "KubeControllerManager",
  53. staticPodName: kubeadmconstants.KubeControllerManager,
  54. },
  55. {
  56. name: "KubeScheduler",
  57. staticPodName: kubeadmconstants.KubeScheduler,
  58. },
  59. }
  60. for _, tc := range tests {
  61. t.Run(tc.name, func(t *testing.T) {
  62. // assert the spec for the staticPodName exists
  63. if spec, ok := specs[tc.staticPodName]; ok {
  64. // Assert each specs refers to the right pod
  65. if spec.Spec.Containers[0].Name != tc.staticPodName {
  66. t.Errorf("getKubeConfigSpecs spec for %s contains pod %s, expects %s", tc.staticPodName, spec.Spec.Containers[0].Name, tc.staticPodName)
  67. }
  68. } else {
  69. t.Errorf("getStaticPodSpecs didn't create spec for %s ", tc.staticPodName)
  70. }
  71. })
  72. }
  73. }
  74. func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
  75. var tests = []struct {
  76. name string
  77. components []string
  78. }{
  79. {
  80. name: "KubeAPIServer KubeAPIServer KubeScheduler",
  81. components: []string{kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler},
  82. },
  83. {
  84. name: "KubeAPIServer",
  85. components: []string{kubeadmconstants.KubeAPIServer},
  86. },
  87. {
  88. name: "KubeControllerManager",
  89. components: []string{kubeadmconstants.KubeControllerManager},
  90. },
  91. {
  92. name: "KubeScheduler",
  93. components: []string{kubeadmconstants.KubeScheduler},
  94. },
  95. }
  96. for _, test := range tests {
  97. t.Run(test.name, func(t *testing.T) {
  98. // Create temp folder for the test case
  99. tmpdir := testutil.SetupTempDir(t)
  100. defer os.RemoveAll(tmpdir)
  101. // Creates a Cluster Configuration
  102. cfg := &kubeadmapi.ClusterConfiguration{
  103. KubernetesVersion: "v1.9.0",
  104. }
  105. // Execute createStaticPodFunction
  106. manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
  107. err := CreateStaticPodFiles(manifestPath, "", cfg, &kubeadmapi.APIEndpoint{}, test.components...)
  108. if err != nil {
  109. t.Errorf("Error executing createStaticPodFunction: %v", err)
  110. return
  111. }
  112. // Assert expected files are there
  113. testutil.AssertFilesCount(t, manifestPath, len(test.components))
  114. for _, fileName := range test.components {
  115. testutil.AssertFileExists(t, manifestPath, fileName+".yaml")
  116. }
  117. })
  118. }
  119. }
  120. func TestCreateStaticPodFilesKustomize(t *testing.T) {
  121. // Create temp folder for the test case
  122. tmpdir := testutil.SetupTempDir(t)
  123. defer os.RemoveAll(tmpdir)
  124. // Creates a Cluster Configuration
  125. cfg := &kubeadmapi.ClusterConfiguration{
  126. KubernetesVersion: "v1.9.0",
  127. }
  128. kustomizePath := filepath.Join(tmpdir, "kustomize")
  129. err := os.MkdirAll(kustomizePath, 0777)
  130. if err != nil {
  131. t.Fatalf("Couldn't create %s", kustomizePath)
  132. }
  133. patchString := dedent.Dedent(`
  134. apiVersion: v1
  135. kind: Pod
  136. metadata:
  137. name: kube-apiserver
  138. namespace: kube-system
  139. annotations:
  140. kustomize: patch for kube-apiserver
  141. `)
  142. err = ioutil.WriteFile(filepath.Join(kustomizePath, "patch.yaml"), []byte(patchString), 0644)
  143. if err != nil {
  144. t.Fatalf("WriteFile returned unexpected error: %v", err)
  145. }
  146. // Execute createStaticPodFunction with kustomizations
  147. manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
  148. err = CreateStaticPodFiles(manifestPath, kustomizePath, cfg, &kubeadmapi.APIEndpoint{}, kubeadmconstants.KubeAPIServer)
  149. if err != nil {
  150. t.Errorf("Error executing createStaticPodFunction: %v", err)
  151. return
  152. }
  153. pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.KubeAPIServer)))
  154. if err != nil {
  155. t.Errorf("Error executing ReadStaticPodFromDisk: %v", err)
  156. return
  157. }
  158. if _, ok := pod.ObjectMeta.Annotations["kustomize"]; !ok {
  159. t.Error("Kustomize did not apply patches corresponding to the resource")
  160. }
  161. }
  162. func TestGetAPIServerCommand(t *testing.T) {
  163. var tests = []struct {
  164. name string
  165. cfg *kubeadmapi.ClusterConfiguration
  166. endpoint *kubeadmapi.APIEndpoint
  167. expected []string
  168. }{
  169. {
  170. name: "testing defaults",
  171. cfg: &kubeadmapi.ClusterConfiguration{
  172. Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
  173. CertificatesDir: testCertsDir,
  174. },
  175. endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
  176. expected: []string{
  177. "kube-apiserver",
  178. "--insecure-port=0",
  179. "--enable-admission-plugins=NodeRestriction",
  180. "--service-cluster-ip-range=bar",
  181. "--service-account-key-file=" + testCertsDir + "/sa.pub",
  182. "--client-ca-file=" + testCertsDir + "/ca.crt",
  183. "--tls-cert-file=" + testCertsDir + "/apiserver.crt",
  184. "--tls-private-key-file=" + testCertsDir + "/apiserver.key",
  185. "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
  186. "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
  187. "--enable-bootstrap-token-auth=true",
  188. "--secure-port=123",
  189. "--allow-privileged=true",
  190. "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
  191. "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
  192. "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
  193. "--requestheader-username-headers=X-Remote-User",
  194. "--requestheader-group-headers=X-Remote-Group",
  195. "--requestheader-extra-headers-prefix=X-Remote-Extra-",
  196. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  197. "--requestheader-allowed-names=front-proxy-client",
  198. "--authorization-mode=Node,RBAC",
  199. "--advertise-address=1.2.3.4",
  200. fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
  201. "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
  202. "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
  203. "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
  204. },
  205. },
  206. {
  207. name: "ipv6 advertise address",
  208. cfg: &kubeadmapi.ClusterConfiguration{
  209. Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
  210. CertificatesDir: testCertsDir,
  211. },
  212. endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
  213. expected: []string{
  214. "kube-apiserver",
  215. "--insecure-port=0",
  216. "--enable-admission-plugins=NodeRestriction",
  217. "--service-cluster-ip-range=bar",
  218. "--service-account-key-file=" + testCertsDir + "/sa.pub",
  219. "--client-ca-file=" + testCertsDir + "/ca.crt",
  220. "--tls-cert-file=" + testCertsDir + "/apiserver.crt",
  221. "--tls-private-key-file=" + testCertsDir + "/apiserver.key",
  222. "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
  223. "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
  224. "--enable-bootstrap-token-auth=true",
  225. fmt.Sprintf("--secure-port=%d", 123),
  226. "--allow-privileged=true",
  227. "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
  228. "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
  229. "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
  230. "--requestheader-username-headers=X-Remote-User",
  231. "--requestheader-group-headers=X-Remote-Group",
  232. "--requestheader-extra-headers-prefix=X-Remote-Extra-",
  233. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  234. "--requestheader-allowed-names=front-proxy-client",
  235. "--authorization-mode=Node,RBAC",
  236. "--advertise-address=2001:db8::1",
  237. fmt.Sprintf("--etcd-servers=https://[::1]:%d", kubeadmconstants.EtcdListenClientPort),
  238. "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
  239. "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
  240. "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
  241. },
  242. },
  243. {
  244. name: "an external etcd with custom ca, certs and keys",
  245. cfg: &kubeadmapi.ClusterConfiguration{
  246. Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
  247. Etcd: kubeadmapi.Etcd{
  248. External: &kubeadmapi.ExternalEtcd{
  249. Endpoints: []string{"https://[2001:abcd:bcda::1]:2379", "https://[2001:abcd:bcda::2]:2379"},
  250. CAFile: "fuz",
  251. CertFile: "fiz",
  252. KeyFile: "faz",
  253. },
  254. },
  255. CertificatesDir: testCertsDir,
  256. },
  257. endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
  258. expected: []string{
  259. "kube-apiserver",
  260. "--insecure-port=0",
  261. "--enable-admission-plugins=NodeRestriction",
  262. "--service-cluster-ip-range=bar",
  263. "--service-account-key-file=" + testCertsDir + "/sa.pub",
  264. "--client-ca-file=" + testCertsDir + "/ca.crt",
  265. "--tls-cert-file=" + testCertsDir + "/apiserver.crt",
  266. "--tls-private-key-file=" + testCertsDir + "/apiserver.key",
  267. "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
  268. "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
  269. fmt.Sprintf("--secure-port=%d", 123),
  270. "--allow-privileged=true",
  271. "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
  272. "--enable-bootstrap-token-auth=true",
  273. "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
  274. "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
  275. "--requestheader-username-headers=X-Remote-User",
  276. "--requestheader-group-headers=X-Remote-Group",
  277. "--requestheader-extra-headers-prefix=X-Remote-Extra-",
  278. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  279. "--requestheader-allowed-names=front-proxy-client",
  280. "--authorization-mode=Node,RBAC",
  281. "--advertise-address=2001:db8::1",
  282. "--etcd-servers=https://[2001:abcd:bcda::1]:2379,https://[2001:abcd:bcda::2]:2379",
  283. "--etcd-cafile=fuz",
  284. "--etcd-certfile=fiz",
  285. "--etcd-keyfile=faz",
  286. },
  287. },
  288. {
  289. name: "an insecure etcd",
  290. cfg: &kubeadmapi.ClusterConfiguration{
  291. Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
  292. Etcd: kubeadmapi.Etcd{
  293. External: &kubeadmapi.ExternalEtcd{
  294. Endpoints: []string{"http://[::1]:2379", "http://[::1]:2380"},
  295. },
  296. },
  297. CertificatesDir: testCertsDir,
  298. },
  299. endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
  300. expected: []string{
  301. "kube-apiserver",
  302. "--insecure-port=0",
  303. "--enable-admission-plugins=NodeRestriction",
  304. "--service-cluster-ip-range=bar",
  305. "--service-account-key-file=" + testCertsDir + "/sa.pub",
  306. "--client-ca-file=" + testCertsDir + "/ca.crt",
  307. "--tls-cert-file=" + testCertsDir + "/apiserver.crt",
  308. "--tls-private-key-file=" + testCertsDir + "/apiserver.key",
  309. "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
  310. "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
  311. fmt.Sprintf("--secure-port=%d", 123),
  312. "--allow-privileged=true",
  313. "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
  314. "--enable-bootstrap-token-auth=true",
  315. "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
  316. "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
  317. "--requestheader-username-headers=X-Remote-User",
  318. "--requestheader-group-headers=X-Remote-Group",
  319. "--requestheader-extra-headers-prefix=X-Remote-Extra-",
  320. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  321. "--requestheader-allowed-names=front-proxy-client",
  322. "--authorization-mode=Node,RBAC",
  323. "--advertise-address=2001:db8::1",
  324. "--etcd-servers=http://[::1]:2379,http://[::1]:2380",
  325. },
  326. },
  327. {
  328. name: "test APIServer.ExtraArgs works as expected",
  329. cfg: &kubeadmapi.ClusterConfiguration{
  330. Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
  331. CertificatesDir: testCertsDir,
  332. APIServer: kubeadmapi.APIServer{
  333. ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
  334. ExtraArgs: map[string]string{
  335. "service-cluster-ip-range": "baz",
  336. "advertise-address": "9.9.9.9",
  337. "audit-policy-file": "/etc/config/audit.yaml",
  338. "audit-log-path": "/var/log/kubernetes",
  339. },
  340. },
  341. },
  342. },
  343. endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
  344. expected: []string{
  345. "kube-apiserver",
  346. "--insecure-port=0",
  347. "--enable-admission-plugins=NodeRestriction",
  348. "--service-cluster-ip-range=baz",
  349. "--service-account-key-file=" + testCertsDir + "/sa.pub",
  350. "--client-ca-file=" + testCertsDir + "/ca.crt",
  351. "--tls-cert-file=" + testCertsDir + "/apiserver.crt",
  352. "--tls-private-key-file=" + testCertsDir + "/apiserver.key",
  353. "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
  354. "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
  355. "--enable-bootstrap-token-auth=true",
  356. "--secure-port=123",
  357. "--allow-privileged=true",
  358. "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
  359. "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
  360. "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
  361. "--requestheader-username-headers=X-Remote-User",
  362. "--requestheader-group-headers=X-Remote-Group",
  363. "--requestheader-extra-headers-prefix=X-Remote-Extra-",
  364. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  365. "--requestheader-allowed-names=front-proxy-client",
  366. "--authorization-mode=Node,RBAC",
  367. "--advertise-address=9.9.9.9",
  368. fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
  369. "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
  370. "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
  371. "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
  372. "--audit-policy-file=/etc/config/audit.yaml",
  373. "--audit-log-path=/var/log/kubernetes",
  374. },
  375. },
  376. {
  377. name: "authorization-mode extra-args ABAC",
  378. cfg: &kubeadmapi.ClusterConfiguration{
  379. Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
  380. CertificatesDir: testCertsDir,
  381. APIServer: kubeadmapi.APIServer{
  382. ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
  383. ExtraArgs: map[string]string{
  384. "authorization-mode": kubeadmconstants.ModeABAC,
  385. },
  386. },
  387. },
  388. },
  389. endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
  390. expected: []string{
  391. "kube-apiserver",
  392. "--insecure-port=0",
  393. "--enable-admission-plugins=NodeRestriction",
  394. "--service-cluster-ip-range=bar",
  395. "--service-account-key-file=" + testCertsDir + "/sa.pub",
  396. "--client-ca-file=" + testCertsDir + "/ca.crt",
  397. "--tls-cert-file=" + testCertsDir + "/apiserver.crt",
  398. "--tls-private-key-file=" + testCertsDir + "/apiserver.key",
  399. "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
  400. "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
  401. "--enable-bootstrap-token-auth=true",
  402. "--secure-port=123",
  403. "--allow-privileged=true",
  404. "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
  405. "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
  406. "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
  407. "--requestheader-username-headers=X-Remote-User",
  408. "--requestheader-group-headers=X-Remote-Group",
  409. "--requestheader-extra-headers-prefix=X-Remote-Extra-",
  410. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  411. "--requestheader-allowed-names=front-proxy-client",
  412. "--authorization-mode=ABAC",
  413. "--advertise-address=1.2.3.4",
  414. fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
  415. "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
  416. "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
  417. "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
  418. },
  419. },
  420. {
  421. name: "insecure-port extra-args",
  422. cfg: &kubeadmapi.ClusterConfiguration{
  423. Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
  424. CertificatesDir: testCertsDir,
  425. APIServer: kubeadmapi.APIServer{
  426. ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
  427. ExtraArgs: map[string]string{
  428. "insecure-port": "1234",
  429. },
  430. },
  431. },
  432. },
  433. endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
  434. expected: []string{
  435. "kube-apiserver",
  436. "--insecure-port=1234",
  437. "--enable-admission-plugins=NodeRestriction",
  438. "--service-cluster-ip-range=bar",
  439. "--service-account-key-file=" + testCertsDir + "/sa.pub",
  440. "--client-ca-file=" + testCertsDir + "/ca.crt",
  441. "--tls-cert-file=" + testCertsDir + "/apiserver.crt",
  442. "--tls-private-key-file=" + testCertsDir + "/apiserver.key",
  443. "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
  444. "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
  445. "--enable-bootstrap-token-auth=true",
  446. "--secure-port=123",
  447. "--allow-privileged=true",
  448. "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
  449. "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
  450. "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
  451. "--requestheader-username-headers=X-Remote-User",
  452. "--requestheader-group-headers=X-Remote-Group",
  453. "--requestheader-extra-headers-prefix=X-Remote-Extra-",
  454. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  455. "--requestheader-allowed-names=front-proxy-client",
  456. "--authorization-mode=Node,RBAC",
  457. "--advertise-address=1.2.3.4",
  458. fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
  459. "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
  460. "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
  461. "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
  462. },
  463. },
  464. {
  465. name: "authorization-mode extra-args Webhook",
  466. cfg: &kubeadmapi.ClusterConfiguration{
  467. Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
  468. CertificatesDir: testCertsDir,
  469. APIServer: kubeadmapi.APIServer{
  470. ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
  471. ExtraArgs: map[string]string{
  472. "authorization-mode": strings.Join([]string{
  473. kubeadmconstants.ModeNode,
  474. kubeadmconstants.ModeRBAC,
  475. kubeadmconstants.ModeWebhook,
  476. }, ","),
  477. },
  478. },
  479. },
  480. },
  481. endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
  482. expected: []string{
  483. "kube-apiserver",
  484. "--insecure-port=0",
  485. "--enable-admission-plugins=NodeRestriction",
  486. "--service-cluster-ip-range=bar",
  487. "--service-account-key-file=" + testCertsDir + "/sa.pub",
  488. "--client-ca-file=" + testCertsDir + "/ca.crt",
  489. "--tls-cert-file=" + testCertsDir + "/apiserver.crt",
  490. "--tls-private-key-file=" + testCertsDir + "/apiserver.key",
  491. "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt",
  492. "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key",
  493. "--enable-bootstrap-token-auth=true",
  494. "--secure-port=123",
  495. "--allow-privileged=true",
  496. "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
  497. "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt",
  498. "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key",
  499. "--requestheader-username-headers=X-Remote-User",
  500. "--requestheader-group-headers=X-Remote-Group",
  501. "--requestheader-extra-headers-prefix=X-Remote-Extra-",
  502. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  503. "--requestheader-allowed-names=front-proxy-client",
  504. "--authorization-mode=Node,RBAC,Webhook",
  505. "--advertise-address=1.2.3.4",
  506. fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort),
  507. "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt",
  508. "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt",
  509. "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key",
  510. },
  511. },
  512. }
  513. for _, rt := range tests {
  514. t.Run(rt.name, func(t *testing.T) {
  515. actual := getAPIServerCommand(rt.cfg, rt.endpoint)
  516. sort.Strings(actual)
  517. sort.Strings(rt.expected)
  518. if !reflect.DeepEqual(actual, rt.expected) {
  519. errorDiffArguments(t, rt.name, actual, rt.expected)
  520. }
  521. })
  522. }
  523. }
  524. func errorDiffArguments(t *testing.T, name string, actual, expected []string) {
  525. expectedShort := removeCommon(expected, actual)
  526. actualShort := removeCommon(actual, expected)
  527. t.Errorf(
  528. "[%s] failed getAPIServerCommand:\nexpected:\n%v\nsaw:\n%v"+
  529. "\nexpectedShort:\n%v\nsawShort:\n%v\n",
  530. name, expected, actual,
  531. expectedShort, actualShort)
  532. }
  533. // removeCommon removes common items from left list
  534. // makes compairing two cmdline (with lots of arguments) easier
  535. func removeCommon(left, right []string) []string {
  536. origSet := sets.NewString(left...)
  537. origSet.Delete(right...)
  538. return origSet.List()
  539. }
  540. func TestGetControllerManagerCommand(t *testing.T) {
  541. var tests = []struct {
  542. name string
  543. cfg *kubeadmapi.ClusterConfiguration
  544. expected []string
  545. }{
  546. {
  547. name: "custom cluster name for " + cpVersion,
  548. cfg: &kubeadmapi.ClusterConfiguration{
  549. KubernetesVersion: cpVersion,
  550. CertificatesDir: testCertsDir,
  551. ClusterName: "some-other-cluster-name",
  552. },
  553. expected: []string{
  554. "kube-controller-manager",
  555. "--bind-address=127.0.0.1",
  556. "--leader-elect=true",
  557. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  558. "--root-ca-file=" + testCertsDir + "/ca.crt",
  559. "--service-account-private-key-file=" + testCertsDir + "/sa.key",
  560. "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
  561. "--cluster-signing-key-file=" + testCertsDir + "/ca.key",
  562. "--use-service-account-credentials=true",
  563. "--controllers=*,bootstrapsigner,tokencleaner",
  564. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  565. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  566. "--client-ca-file=" + testCertsDir + "/ca.crt",
  567. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  568. "--cluster-name=some-other-cluster-name",
  569. },
  570. },
  571. {
  572. name: "custom certs dir for " + cpVersion,
  573. cfg: &kubeadmapi.ClusterConfiguration{
  574. CertificatesDir: testCertsDir,
  575. KubernetesVersion: cpVersion,
  576. },
  577. expected: []string{
  578. "kube-controller-manager",
  579. "--bind-address=127.0.0.1",
  580. "--leader-elect=true",
  581. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  582. "--root-ca-file=" + testCertsDir + "/ca.crt",
  583. "--service-account-private-key-file=" + testCertsDir + "/sa.key",
  584. "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
  585. "--cluster-signing-key-file=" + testCertsDir + "/ca.key",
  586. "--use-service-account-credentials=true",
  587. "--controllers=*,bootstrapsigner,tokencleaner",
  588. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  589. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  590. "--client-ca-file=" + testCertsDir + "/ca.crt",
  591. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  592. },
  593. },
  594. {
  595. name: "custom cluster-cidr for " + cpVersion,
  596. cfg: &kubeadmapi.ClusterConfiguration{
  597. Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16"},
  598. CertificatesDir: testCertsDir,
  599. KubernetesVersion: cpVersion,
  600. },
  601. expected: []string{
  602. "kube-controller-manager",
  603. "--bind-address=127.0.0.1",
  604. "--leader-elect=true",
  605. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  606. "--root-ca-file=" + testCertsDir + "/ca.crt",
  607. "--service-account-private-key-file=" + testCertsDir + "/sa.key",
  608. "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
  609. "--cluster-signing-key-file=" + testCertsDir + "/ca.key",
  610. "--use-service-account-credentials=true",
  611. "--controllers=*,bootstrapsigner,tokencleaner",
  612. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  613. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  614. "--client-ca-file=" + testCertsDir + "/ca.crt",
  615. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  616. "--allocate-node-cidrs=true",
  617. "--cluster-cidr=10.0.1.15/16",
  618. "--node-cidr-mask-size=24",
  619. },
  620. },
  621. {
  622. name: "custom service-cluster-ip-range for " + cpVersion,
  623. cfg: &kubeadmapi.ClusterConfiguration{
  624. Networking: kubeadmapi.Networking{
  625. PodSubnet: "10.0.1.15/16",
  626. ServiceSubnet: "172.20.0.0/24"},
  627. CertificatesDir: testCertsDir,
  628. KubernetesVersion: cpVersion,
  629. },
  630. expected: []string{
  631. "kube-controller-manager",
  632. "--bind-address=127.0.0.1",
  633. "--leader-elect=true",
  634. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  635. "--root-ca-file=" + testCertsDir + "/ca.crt",
  636. "--service-account-private-key-file=" + testCertsDir + "/sa.key",
  637. "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
  638. "--cluster-signing-key-file=" + testCertsDir + "/ca.key",
  639. "--use-service-account-credentials=true",
  640. "--controllers=*,bootstrapsigner,tokencleaner",
  641. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  642. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  643. "--client-ca-file=" + testCertsDir + "/ca.crt",
  644. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  645. "--allocate-node-cidrs=true",
  646. "--cluster-cidr=10.0.1.15/16",
  647. "--node-cidr-mask-size=24",
  648. "--service-cluster-ip-range=172.20.0.0/24",
  649. },
  650. },
  651. {
  652. name: "custom extra-args for " + cpVersion,
  653. cfg: &kubeadmapi.ClusterConfiguration{
  654. Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16"},
  655. ControllerManager: kubeadmapi.ControlPlaneComponent{
  656. ExtraArgs: map[string]string{"node-cidr-mask-size": "20"},
  657. },
  658. CertificatesDir: testCertsDir,
  659. KubernetesVersion: cpVersion,
  660. },
  661. expected: []string{
  662. "kube-controller-manager",
  663. "--bind-address=127.0.0.1",
  664. "--leader-elect=true",
  665. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  666. "--root-ca-file=" + testCertsDir + "/ca.crt",
  667. "--service-account-private-key-file=" + testCertsDir + "/sa.key",
  668. "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
  669. "--cluster-signing-key-file=" + testCertsDir + "/ca.key",
  670. "--use-service-account-credentials=true",
  671. "--controllers=*,bootstrapsigner,tokencleaner",
  672. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  673. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  674. "--client-ca-file=" + testCertsDir + "/ca.crt",
  675. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  676. "--allocate-node-cidrs=true",
  677. "--cluster-cidr=10.0.1.15/16",
  678. "--node-cidr-mask-size=20",
  679. },
  680. },
  681. {
  682. name: "custom IPv6 networking for " + cpVersion,
  683. cfg: &kubeadmapi.ClusterConfiguration{
  684. Networking: kubeadmapi.Networking{
  685. PodSubnet: "2001:db8::/64",
  686. ServiceSubnet: "fd03::/112",
  687. },
  688. CertificatesDir: testCertsDir,
  689. KubernetesVersion: cpVersion,
  690. },
  691. expected: []string{
  692. "kube-controller-manager",
  693. "--bind-address=127.0.0.1",
  694. "--leader-elect=true",
  695. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  696. "--root-ca-file=" + testCertsDir + "/ca.crt",
  697. "--service-account-private-key-file=" + testCertsDir + "/sa.key",
  698. "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
  699. "--cluster-signing-key-file=" + testCertsDir + "/ca.key",
  700. "--use-service-account-credentials=true",
  701. "--controllers=*,bootstrapsigner,tokencleaner",
  702. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  703. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  704. "--client-ca-file=" + testCertsDir + "/ca.crt",
  705. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  706. "--allocate-node-cidrs=true",
  707. "--cluster-cidr=2001:db8::/64",
  708. "--node-cidr-mask-size=80",
  709. "--service-cluster-ip-range=fd03::/112",
  710. },
  711. },
  712. {
  713. name: "dual-stack networking for " + cpVersion,
  714. cfg: &kubeadmapi.ClusterConfiguration{
  715. Networking: kubeadmapi.Networking{
  716. PodSubnet: "2001:db8::/64,10.1.0.0/16",
  717. ServiceSubnet: "fd03::/112,192.168.0.0/16",
  718. },
  719. CertificatesDir: testCertsDir,
  720. KubernetesVersion: cpVersion,
  721. FeatureGates: map[string]bool{string(features.IPv6DualStack): true},
  722. },
  723. expected: []string{
  724. "kube-controller-manager",
  725. "--bind-address=127.0.0.1",
  726. "--leader-elect=true",
  727. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  728. "--root-ca-file=" + testCertsDir + "/ca.crt",
  729. "--service-account-private-key-file=" + testCertsDir + "/sa.key",
  730. "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
  731. "--cluster-signing-key-file=" + testCertsDir + "/ca.key",
  732. "--use-service-account-credentials=true",
  733. "--controllers=*,bootstrapsigner,tokencleaner",
  734. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  735. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  736. "--client-ca-file=" + testCertsDir + "/ca.crt",
  737. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  738. "--feature-gates=IPv6DualStack=true",
  739. "--allocate-node-cidrs=true",
  740. "--cluster-cidr=2001:db8::/64,10.1.0.0/16",
  741. "--node-cidr-mask-size-ipv4=24",
  742. "--node-cidr-mask-size-ipv6=80",
  743. "--service-cluster-ip-range=fd03::/112,192.168.0.0/16",
  744. },
  745. },
  746. {
  747. name: "dual-stack networking custom extra-args for " + cpVersion,
  748. cfg: &kubeadmapi.ClusterConfiguration{
  749. Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16,2001:db8::/64"},
  750. ControllerManager: kubeadmapi.ControlPlaneComponent{
  751. ExtraArgs: map[string]string{"node-cidr-mask-size-ipv4": "20", "node-cidr-mask-size-ipv6": "96"},
  752. },
  753. CertificatesDir: testCertsDir,
  754. KubernetesVersion: cpVersion,
  755. FeatureGates: map[string]bool{string(features.IPv6DualStack): true},
  756. },
  757. expected: []string{
  758. "kube-controller-manager",
  759. "--bind-address=127.0.0.1",
  760. "--leader-elect=true",
  761. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  762. "--root-ca-file=" + testCertsDir + "/ca.crt",
  763. "--service-account-private-key-file=" + testCertsDir + "/sa.key",
  764. "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
  765. "--cluster-signing-key-file=" + testCertsDir + "/ca.key",
  766. "--use-service-account-credentials=true",
  767. "--controllers=*,bootstrapsigner,tokencleaner",
  768. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  769. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  770. "--client-ca-file=" + testCertsDir + "/ca.crt",
  771. "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
  772. "--feature-gates=IPv6DualStack=true",
  773. "--allocate-node-cidrs=true",
  774. "--cluster-cidr=10.0.1.15/16,2001:db8::/64",
  775. "--node-cidr-mask-size-ipv4=20",
  776. "--node-cidr-mask-size-ipv6=96",
  777. },
  778. },
  779. }
  780. for _, rt := range tests {
  781. t.Run(rt.name, func(t *testing.T) {
  782. actual := getControllerManagerCommand(rt.cfg)
  783. sort.Strings(actual)
  784. sort.Strings(rt.expected)
  785. if !reflect.DeepEqual(actual, rt.expected) {
  786. errorDiffArguments(t, rt.name, actual, rt.expected)
  787. }
  788. })
  789. }
  790. }
  791. func TestCalcNodeCidrSize(t *testing.T) {
  792. tests := []struct {
  793. name string
  794. podSubnet string
  795. expectedPrefix string
  796. expectedIPv6 bool
  797. }{
  798. {
  799. name: "Malformed pod subnet",
  800. podSubnet: "10.10.10/160",
  801. expectedPrefix: "24",
  802. expectedIPv6: false,
  803. },
  804. {
  805. name: "V4: Always uses 24",
  806. podSubnet: "10.10.10.10/16",
  807. expectedPrefix: "24",
  808. expectedIPv6: false,
  809. },
  810. {
  811. name: "V6: Use pod subnet size, when not enough space",
  812. podSubnet: "2001:db8::/128",
  813. expectedPrefix: "128",
  814. expectedIPv6: true,
  815. },
  816. {
  817. name: "V6: Use pod subnet size, when not enough space",
  818. podSubnet: "2001:db8::/113",
  819. expectedPrefix: "113",
  820. expectedIPv6: true,
  821. },
  822. {
  823. name: "V6: Special case with 256 nodes",
  824. podSubnet: "2001:db8::/112",
  825. expectedPrefix: "120",
  826. expectedIPv6: true,
  827. },
  828. {
  829. name: "V6: Using /120 for node CIDR",
  830. podSubnet: "2001:db8::/104",
  831. expectedPrefix: "120",
  832. expectedIPv6: true,
  833. },
  834. {
  835. name: "V6: Using /112 for node CIDR",
  836. podSubnet: "2001:db8::/103",
  837. expectedPrefix: "112",
  838. expectedIPv6: true,
  839. },
  840. {
  841. name: "V6: Using /112 for node CIDR",
  842. podSubnet: "2001:db8::/96",
  843. expectedPrefix: "112",
  844. expectedIPv6: true,
  845. },
  846. {
  847. name: "V6: Using /104 for node CIDR",
  848. podSubnet: "2001:db8::/95",
  849. expectedPrefix: "104",
  850. expectedIPv6: true,
  851. },
  852. {
  853. name: "V6: For /64 pod net, use /80",
  854. podSubnet: "2001:db8::/64",
  855. expectedPrefix: "80",
  856. expectedIPv6: true,
  857. },
  858. {
  859. name: "V6: For /48 pod net, use /64",
  860. podSubnet: "2001:db8::/48",
  861. expectedPrefix: "64",
  862. expectedIPv6: true,
  863. },
  864. {
  865. name: "V6: For /32 pod net, use /48",
  866. podSubnet: "2001:db8::/32",
  867. expectedPrefix: "48",
  868. expectedIPv6: true,
  869. },
  870. }
  871. for _, test := range tests {
  872. t.Run(test.name, func(t *testing.T) {
  873. actualPrefix, actualIPv6 := calcNodeCidrSize(test.podSubnet)
  874. if actualPrefix != test.expectedPrefix {
  875. t.Errorf("Case [%s]\nCalc of node CIDR size for pod subnet %q failed: Expected %q, saw %q",
  876. test.name, test.podSubnet, test.expectedPrefix, actualPrefix)
  877. }
  878. if actualIPv6 != test.expectedIPv6 {
  879. t.Errorf("Case [%s]\nCalc of node CIDR size for pod subnet %q failed: Expected isIPv6=%v, saw isIPv6=%v",
  880. test.name, test.podSubnet, test.expectedIPv6, actualIPv6)
  881. }
  882. })
  883. }
  884. }
  885. func TestGetControllerManagerCommandExternalCA(t *testing.T) {
  886. tests := []struct {
  887. name string
  888. cfg *kubeadmapi.InitConfiguration
  889. caKeyPresent bool
  890. expectedArgFunc func(dir string) []string
  891. }{
  892. {
  893. name: "caKeyPresent-false for " + cpVersion,
  894. cfg: &kubeadmapi.InitConfiguration{
  895. LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
  896. ClusterConfiguration: kubeadmapi.ClusterConfiguration{
  897. KubernetesVersion: cpVersion,
  898. Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
  899. },
  900. },
  901. caKeyPresent: false,
  902. expectedArgFunc: func(tmpdir string) []string {
  903. return []string{
  904. "kube-controller-manager",
  905. "--bind-address=127.0.0.1",
  906. "--leader-elect=true",
  907. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  908. "--root-ca-file=" + tmpdir + "/ca.crt",
  909. "--service-account-private-key-file=" + tmpdir + "/sa.key",
  910. "--cluster-signing-cert-file=",
  911. "--cluster-signing-key-file=",
  912. "--use-service-account-credentials=true",
  913. "--controllers=*,bootstrapsigner,tokencleaner",
  914. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  915. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  916. "--client-ca-file=" + tmpdir + "/ca.crt",
  917. "--requestheader-client-ca-file=" + tmpdir + "/front-proxy-ca.crt",
  918. }
  919. },
  920. },
  921. {
  922. name: "caKeyPresent true for " + cpVersion,
  923. cfg: &kubeadmapi.InitConfiguration{
  924. LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
  925. ClusterConfiguration: kubeadmapi.ClusterConfiguration{
  926. KubernetesVersion: cpVersion,
  927. Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
  928. },
  929. },
  930. caKeyPresent: true,
  931. expectedArgFunc: func(tmpdir string) []string {
  932. return []string{
  933. "kube-controller-manager",
  934. "--bind-address=127.0.0.1",
  935. "--leader-elect=true",
  936. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  937. "--root-ca-file=" + tmpdir + "/ca.crt",
  938. "--service-account-private-key-file=" + tmpdir + "/sa.key",
  939. "--cluster-signing-cert-file=" + tmpdir + "/ca.crt",
  940. "--cluster-signing-key-file=" + tmpdir + "/ca.key",
  941. "--use-service-account-credentials=true",
  942. "--controllers=*,bootstrapsigner,tokencleaner",
  943. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  944. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
  945. "--client-ca-file=" + tmpdir + "/ca.crt",
  946. "--requestheader-client-ca-file=" + tmpdir + "/front-proxy-ca.crt",
  947. }
  948. },
  949. },
  950. }
  951. for _, test := range tests {
  952. t.Run(test.name, func(t *testing.T) {
  953. // Create temp folder for the test case
  954. tmpdir := testutil.SetupTempDir(t)
  955. defer os.RemoveAll(tmpdir)
  956. test.cfg.CertificatesDir = tmpdir
  957. if err := certs.CreatePKIAssets(test.cfg); err != nil {
  958. t.Errorf("failed creating pki assets: %v", err)
  959. }
  960. // delete ca.key and front-proxy-ca.key if test.caKeyPresent is false
  961. if !test.caKeyPresent {
  962. if err := os.Remove(filepath.Join(test.cfg.CertificatesDir, kubeadmconstants.CAKeyName)); err != nil {
  963. t.Errorf("failed removing %s: %v", kubeadmconstants.CAKeyName, err)
  964. }
  965. if err := os.Remove(filepath.Join(test.cfg.CertificatesDir, kubeadmconstants.FrontProxyCAKeyName)); err != nil {
  966. t.Errorf("failed removing %s: %v", kubeadmconstants.FrontProxyCAKeyName, err)
  967. }
  968. }
  969. actual := getControllerManagerCommand(&test.cfg.ClusterConfiguration)
  970. expected := test.expectedArgFunc(tmpdir)
  971. sort.Strings(actual)
  972. sort.Strings(expected)
  973. if !reflect.DeepEqual(actual, expected) {
  974. errorDiffArguments(t, test.name, actual, expected)
  975. }
  976. })
  977. }
  978. }
  979. func TestGetSchedulerCommand(t *testing.T) {
  980. var tests = []struct {
  981. name string
  982. cfg *kubeadmapi.ClusterConfiguration
  983. expected []string
  984. }{
  985. {
  986. name: "scheduler defaults",
  987. cfg: &kubeadmapi.ClusterConfiguration{},
  988. expected: []string{
  989. "kube-scheduler",
  990. "--bind-address=127.0.0.1",
  991. "--leader-elect=true",
  992. "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/scheduler.conf",
  993. "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/scheduler.conf",
  994. "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/scheduler.conf",
  995. },
  996. },
  997. }
  998. for _, rt := range tests {
  999. t.Run(rt.name, func(t *testing.T) {
  1000. actual := getSchedulerCommand(rt.cfg)
  1001. sort.Strings(actual)
  1002. sort.Strings(rt.expected)
  1003. if !reflect.DeepEqual(actual, rt.expected) {
  1004. errorDiffArguments(t, rt.name, actual, rt.expected)
  1005. }
  1006. })
  1007. }
  1008. }
  1009. func TestGetAuthzModes(t *testing.T) {
  1010. var tests = []struct {
  1011. name string
  1012. authMode []string
  1013. expected string
  1014. }{
  1015. {
  1016. name: "default if empty",
  1017. authMode: []string{},
  1018. expected: "Node,RBAC",
  1019. },
  1020. {
  1021. name: "default non empty",
  1022. authMode: []string{kubeadmconstants.ModeNode, kubeadmconstants.ModeRBAC},
  1023. expected: "Node,RBAC",
  1024. },
  1025. {
  1026. name: "single unspecified returning default",
  1027. authMode: []string{"FooAuthzMode"},
  1028. expected: "Node,RBAC",
  1029. },
  1030. {
  1031. name: "multiple ignored",
  1032. authMode: []string{kubeadmconstants.ModeNode, "foo", kubeadmconstants.ModeRBAC, "bar"},
  1033. expected: "Node,RBAC",
  1034. },
  1035. {
  1036. name: "single mode",
  1037. authMode: []string{kubeadmconstants.ModeAlwaysDeny},
  1038. expected: "AlwaysDeny",
  1039. },
  1040. {
  1041. name: "multiple special order",
  1042. authMode: []string{kubeadmconstants.ModeNode, kubeadmconstants.ModeWebhook, kubeadmconstants.ModeRBAC, kubeadmconstants.ModeABAC},
  1043. expected: "Node,Webhook,RBAC,ABAC",
  1044. },
  1045. }
  1046. for _, rt := range tests {
  1047. t.Run(rt.name, func(t *testing.T) {
  1048. actual := getAuthzModes(strings.Join(rt.authMode, ","))
  1049. if actual != rt.expected {
  1050. t.Errorf("failed getAuthzModes:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
  1051. }
  1052. })
  1053. }
  1054. }
  1055. func TestIsValidAuthzMode(t *testing.T) {
  1056. var tests = []struct {
  1057. mode string
  1058. valid bool
  1059. }{
  1060. {
  1061. mode: "Node",
  1062. valid: true,
  1063. },
  1064. {
  1065. mode: "RBAC",
  1066. valid: true,
  1067. },
  1068. {
  1069. mode: "ABAC",
  1070. valid: true,
  1071. },
  1072. {
  1073. mode: "AlwaysAllow",
  1074. valid: true,
  1075. },
  1076. {
  1077. mode: "Webhook",
  1078. valid: true,
  1079. },
  1080. {
  1081. mode: "AlwaysDeny",
  1082. valid: true,
  1083. },
  1084. {
  1085. mode: "Foo",
  1086. valid: false,
  1087. },
  1088. }
  1089. for _, rt := range tests {
  1090. t.Run(rt.mode, func(t *testing.T) {
  1091. isValid := isValidAuthzMode(rt.mode)
  1092. if isValid != rt.valid {
  1093. t.Errorf("failed isValidAuthzMode:\nexpected:\n%v\nsaw:\n%v", rt.valid, isValid)
  1094. }
  1095. })
  1096. }
  1097. }