deployment_test.go 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278
  1. /*
  2. Copyright 2017 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package deployment
  14. import (
  15. "context"
  16. "fmt"
  17. "strings"
  18. "testing"
  19. apps "k8s.io/api/apps/v1"
  20. "k8s.io/api/core/v1"
  21. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  22. "k8s.io/apimachinery/pkg/util/intstr"
  23. "k8s.io/apimachinery/pkg/util/uuid"
  24. "k8s.io/apimachinery/pkg/util/wait"
  25. "k8s.io/client-go/util/retry"
  26. deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
  27. "k8s.io/kubernetes/test/integration/framework"
  28. "k8s.io/utils/pointer"
  29. )
  30. func TestNewDeployment(t *testing.T) {
  31. s, closeFn, rm, dc, informers, c := dcSetup(t)
  32. defer closeFn()
  33. name := "test-new-deployment"
  34. ns := framework.CreateTestingNamespace(name, s, t)
  35. defer framework.DeleteTestingNamespace(ns, s, t)
  36. replicas := int32(20)
  37. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
  38. tester.deployment.Spec.MinReadySeconds = 4
  39. tester.deployment.Annotations = map[string]string{"test": "should-copy-to-replica-set", v1.LastAppliedConfigAnnotation: "should-not-copy-to-replica-set"}
  40. var err error
  41. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  42. if err != nil {
  43. t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
  44. }
  45. // Start informer and controllers
  46. stopCh := make(chan struct{})
  47. defer close(stopCh)
  48. informers.Start(stopCh)
  49. go rm.Run(5, stopCh)
  50. go dc.Run(5, stopCh)
  51. // Wait for the Deployment to be updated to revision 1
  52. if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  53. t.Fatal(err)
  54. }
  55. // Make sure the Deployment completes while manually marking Deployment pods as ready at the same time.
  56. // Use soft check because this deployment was just created and rolling update strategy might be violated.
  57. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  58. t.Fatal(err)
  59. }
  60. // Check new RS annotations
  61. newRS, err := tester.expectNewReplicaSet()
  62. if err != nil {
  63. t.Fatal(err)
  64. }
  65. if newRS.Annotations["test"] != "should-copy-to-replica-set" {
  66. t.Errorf("expected new ReplicaSet annotations copied from Deployment %s, got: %v", tester.deployment.Name, newRS.Annotations)
  67. }
  68. if newRS.Annotations[v1.LastAppliedConfigAnnotation] != "" {
  69. t.Errorf("expected new ReplicaSet last-applied annotation not copied from Deployment %s", tester.deployment.Name)
  70. }
  71. // New RS should contain pod-template-hash in its selector, label, and template label
  72. rsHash, err := checkRSHashLabels(newRS)
  73. if err != nil {
  74. t.Error(err)
  75. }
  76. // All pods targeted by the deployment should contain pod-template-hash in their labels
  77. selector, err := metav1.LabelSelectorAsSelector(tester.deployment.Spec.Selector)
  78. if err != nil {
  79. t.Fatalf("failed to parse deployment %s selector: %v", name, err)
  80. }
  81. pods, err := c.CoreV1().Pods(ns.Name).List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()})
  82. if err != nil {
  83. t.Fatalf("failed to list pods of deployment %s: %v", name, err)
  84. }
  85. if len(pods.Items) != int(replicas) {
  86. t.Errorf("expected %d pods, got %d pods", replicas, len(pods.Items))
  87. }
  88. podHash, err := checkPodsHashLabel(pods)
  89. if err != nil {
  90. t.Error(err)
  91. }
  92. if rsHash != podHash {
  93. t.Errorf("found mismatching pod-template-hash value: rs hash = %s whereas pod hash = %s", rsHash, podHash)
  94. }
  95. }
  96. // Deployments should support roll out, roll back, and roll over.
  97. // TODO: drop the rollback portions of this test when extensions/v1beta1 is no longer served
  98. // and rollback endpoint is no longer supported.
  99. func TestDeploymentRollingUpdate(t *testing.T) {
  100. s, closeFn, rm, dc, informers, c := dcSetup(t)
  101. defer closeFn()
  102. name := "test-rolling-update-deployment"
  103. ns := framework.CreateTestingNamespace(name, s, t)
  104. defer framework.DeleteTestingNamespace(ns, s, t)
  105. // Start informer and controllers
  106. stopCh := make(chan struct{})
  107. defer close(stopCh)
  108. informers.Start(stopCh)
  109. go rm.Run(5, stopCh)
  110. go dc.Run(5, stopCh)
  111. replicas := int32(20)
  112. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
  113. tester.deployment.Spec.MinReadySeconds = 4
  114. quarter := intstr.FromString("25%")
  115. tester.deployment.Spec.Strategy.RollingUpdate = &apps.RollingUpdateDeployment{
  116. MaxUnavailable: &quarter,
  117. MaxSurge: &quarter,
  118. }
  119. // Create a deployment.
  120. var err error
  121. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  122. if err != nil {
  123. t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
  124. }
  125. oriImage := tester.deployment.Spec.Template.Spec.Containers[0].Image
  126. if err := tester.waitForDeploymentRevisionAndImage("1", oriImage); err != nil {
  127. t.Fatal(err)
  128. }
  129. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  130. t.Fatal(err)
  131. }
  132. // 1. Roll out a new image.
  133. image := "new-image"
  134. if oriImage == image {
  135. t.Fatalf("bad test setup, deployment %s roll out with the same image", tester.deployment.Name)
  136. }
  137. imageFn := func(update *apps.Deployment) {
  138. update.Spec.Template.Spec.Containers[0].Image = image
  139. }
  140. tester.deployment, err = tester.updateDeployment(imageFn)
  141. if err != nil {
  142. t.Fatalf("failed to update deployment %s: %v", tester.deployment.Name, err)
  143. }
  144. if err := tester.waitForDeploymentRevisionAndImage("2", image); err != nil {
  145. t.Fatal(err)
  146. }
  147. if err := tester.waitForDeploymentCompleteAndCheckRollingAndMarkPodsReady(); err != nil {
  148. t.Fatal(err)
  149. }
  150. // 2. Roll over a deployment before the previous rolling update finishes.
  151. image = "dont-finish"
  152. imageFn = func(update *apps.Deployment) {
  153. update.Spec.Template.Spec.Containers[0].Image = image
  154. }
  155. tester.deployment, err = tester.updateDeployment(imageFn)
  156. if err != nil {
  157. t.Fatalf("failed to update deployment %s: %v", tester.deployment.Name, err)
  158. }
  159. if err := tester.waitForDeploymentRevisionAndImage("3", image); err != nil {
  160. t.Fatal(err)
  161. }
  162. // We don't mark pods as ready so that rollout won't finish.
  163. // Before the rollout finishes, trigger another rollout.
  164. image = "rollover"
  165. imageFn = func(update *apps.Deployment) {
  166. update.Spec.Template.Spec.Containers[0].Image = image
  167. }
  168. tester.deployment, err = tester.updateDeployment(imageFn)
  169. if err != nil {
  170. t.Fatalf("failed to update deployment %s: %v", tester.deployment.Name, err)
  171. }
  172. if err := tester.waitForDeploymentRevisionAndImage("4", image); err != nil {
  173. t.Fatal(err)
  174. }
  175. if err := tester.waitForDeploymentCompleteAndCheckRollingAndMarkPodsReady(); err != nil {
  176. t.Fatal(err)
  177. }
  178. _, allOldRSs, err := deploymentutil.GetOldReplicaSets(tester.deployment, c.AppsV1())
  179. if err != nil {
  180. t.Fatalf("failed retrieving old replicasets of deployment %s: %v", tester.deployment.Name, err)
  181. }
  182. for _, oldRS := range allOldRSs {
  183. if *oldRS.Spec.Replicas != 0 {
  184. t.Errorf("expected old replicaset %s of deployment %s to have 0 replica, got %d", oldRS.Name, tester.deployment.Name, *oldRS.Spec.Replicas)
  185. }
  186. }
  187. }
  188. // selectors are IMMUTABLE for all API versions except apps/v1beta1 and extensions/v1beta1
  189. func TestDeploymentSelectorImmutability(t *testing.T) {
  190. s, closeFn, c := dcSimpleSetup(t)
  191. defer closeFn()
  192. name := "test-deployment-selector-immutability"
  193. ns := framework.CreateTestingNamespace(name, s, t)
  194. defer framework.DeleteTestingNamespace(ns, s, t)
  195. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, int32(20))}
  196. var err error
  197. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  198. if err != nil {
  199. t.Fatalf("failed to create apps/v1 deployment %s: %v", tester.deployment.Name, err)
  200. }
  201. // test to ensure apps/v1 selector is immutable
  202. newSelectorLabels := map[string]string{"name_apps_v1beta1": "test_apps_v1beta1"}
  203. deploymentAppsV1, err := c.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{})
  204. if err != nil {
  205. t.Fatalf("failed to get apps/v1 deployment %s: %v", name, err)
  206. }
  207. newSelectorLabels = map[string]string{"name_apps_v1": "test_apps_v1"}
  208. deploymentAppsV1.Spec.Selector.MatchLabels = newSelectorLabels
  209. deploymentAppsV1.Spec.Template.Labels = newSelectorLabels
  210. _, err = c.AppsV1().Deployments(ns.Name).Update(context.TODO(), deploymentAppsV1, metav1.UpdateOptions{})
  211. if err == nil {
  212. t.Fatalf("failed to provide validation error when changing immutable selector when updating apps/v1 deployment %s", deploymentAppsV1.Name)
  213. }
  214. expectedErrType := "Invalid value"
  215. expectedErrDetail := "field is immutable"
  216. if !strings.Contains(err.Error(), expectedErrType) || !strings.Contains(err.Error(), expectedErrDetail) {
  217. t.Errorf("error message does not match, expected type: %s, expected detail: %s, got: %s", expectedErrType, expectedErrDetail, err.Error())
  218. }
  219. }
  220. // Paused deployment should not start new rollout
  221. func TestPausedDeployment(t *testing.T) {
  222. s, closeFn, rm, dc, informers, c := dcSetup(t)
  223. defer closeFn()
  224. name := "test-paused-deployment"
  225. ns := framework.CreateTestingNamespace(name, s, t)
  226. defer framework.DeleteTestingNamespace(ns, s, t)
  227. replicas := int32(1)
  228. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
  229. tester.deployment.Spec.Paused = true
  230. tgps := int64(1)
  231. tester.deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &tgps
  232. var err error
  233. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  234. if err != nil {
  235. t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
  236. }
  237. // Start informer and controllers
  238. stopCh := make(chan struct{})
  239. defer close(stopCh)
  240. informers.Start(stopCh)
  241. go rm.Run(5, stopCh)
  242. go dc.Run(5, stopCh)
  243. // Verify that the paused deployment won't create new replica set.
  244. if err := tester.expectNoNewReplicaSet(); err != nil {
  245. t.Fatal(err)
  246. }
  247. // Resume the deployment
  248. tester.deployment, err = tester.updateDeployment(resumeFn)
  249. if err != nil {
  250. t.Fatalf("failed to resume deployment %s: %v", tester.deployment.Name, err)
  251. }
  252. // Wait for the controller to notice the resume.
  253. if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
  254. t.Fatal(err)
  255. }
  256. // Wait for the Deployment to be updated to revision 1
  257. if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  258. t.Fatal(err)
  259. }
  260. // Make sure the Deployment completes while manually marking Deployment pods as ready at the same time.
  261. // Use soft check because this deployment was just created and rolling update strategy might be violated.
  262. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  263. t.Fatal(err)
  264. }
  265. // A new replicaset should be created.
  266. if _, err := tester.expectNewReplicaSet(); err != nil {
  267. t.Fatal(err)
  268. }
  269. // Pause the deployment.
  270. // The paused deployment shouldn't trigger a new rollout.
  271. tester.deployment, err = tester.updateDeployment(pauseFn)
  272. if err != nil {
  273. t.Fatalf("failed to pause deployment %s: %v", tester.deployment.Name, err)
  274. }
  275. // Wait for the controller to notice the pause.
  276. if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
  277. t.Fatal(err)
  278. }
  279. // Update the deployment template
  280. newTGPS := int64(0)
  281. tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
  282. update.Spec.Template.Spec.TerminationGracePeriodSeconds = &newTGPS
  283. })
  284. if err != nil {
  285. t.Fatalf("failed updating template of deployment %s: %v", tester.deployment.Name, err)
  286. }
  287. // Wait for the controller to notice the rollout.
  288. if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
  289. t.Fatal(err)
  290. }
  291. // Verify that the paused deployment won't create new replica set.
  292. if err := tester.expectNoNewReplicaSet(); err != nil {
  293. t.Fatal(err)
  294. }
  295. _, allOldRs, err := deploymentutil.GetOldReplicaSets(tester.deployment, c.AppsV1())
  296. if err != nil {
  297. t.Fatalf("failed retrieving old replicasets of deployment %s: %v", tester.deployment.Name, err)
  298. }
  299. if len(allOldRs) != 1 {
  300. t.Errorf("expected an old replica set, got %v", allOldRs)
  301. }
  302. if *allOldRs[0].Spec.Template.Spec.TerminationGracePeriodSeconds == newTGPS {
  303. t.Errorf("TerminationGracePeriodSeconds on the replica set should be %d, got %d", tgps, newTGPS)
  304. }
  305. }
  306. // Paused deployment can be scaled
  307. func TestScalePausedDeployment(t *testing.T) {
  308. s, closeFn, rm, dc, informers, c := dcSetup(t)
  309. defer closeFn()
  310. name := "test-scale-paused-deployment"
  311. ns := framework.CreateTestingNamespace(name, s, t)
  312. defer framework.DeleteTestingNamespace(ns, s, t)
  313. replicas := int32(1)
  314. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
  315. tgps := int64(1)
  316. tester.deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &tgps
  317. var err error
  318. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  319. if err != nil {
  320. t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
  321. }
  322. // Start informer and controllers
  323. stopCh := make(chan struct{})
  324. defer close(stopCh)
  325. informers.Start(stopCh)
  326. go rm.Run(5, stopCh)
  327. go dc.Run(5, stopCh)
  328. // Wait for the Deployment to be updated to revision 1
  329. if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  330. t.Fatal(err)
  331. }
  332. // Make sure the Deployment completes while manually marking Deployment pods as ready at the same time.
  333. // Use soft check because this deployment was just created and rolling update strategy might be violated.
  334. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  335. t.Fatal(err)
  336. }
  337. // A new replicaset should be created.
  338. if _, err := tester.expectNewReplicaSet(); err != nil {
  339. t.Fatal(err)
  340. }
  341. // Pause the deployment.
  342. tester.deployment, err = tester.updateDeployment(pauseFn)
  343. if err != nil {
  344. t.Fatalf("failed to pause deployment %s: %v", tester.deployment.Name, err)
  345. }
  346. // Wait for the controller to notice the scale.
  347. if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
  348. t.Fatal(err)
  349. }
  350. // Scale the paused deployment.
  351. newReplicas := int32(10)
  352. tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
  353. update.Spec.Replicas = &newReplicas
  354. })
  355. if err != nil {
  356. t.Fatalf("failed updating deployment %s: %v", tester.deployment.Name, err)
  357. }
  358. // Wait for the controller to notice the scale.
  359. if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
  360. t.Fatal(err)
  361. }
  362. // Verify that the new replicaset is scaled.
  363. rs, err := tester.expectNewReplicaSet()
  364. if err != nil {
  365. t.Fatal(err)
  366. }
  367. if *rs.Spec.Replicas != newReplicas {
  368. t.Errorf("expected new replicaset replicas = %d, got %d", newReplicas, *rs.Spec.Replicas)
  369. }
  370. // Make sure the Deployment completes while manually marking Deployment pods as ready at the same time.
  371. // Use soft check because this deployment was just scaled and rolling update strategy might be violated.
  372. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  373. t.Fatal(err)
  374. }
  375. }
  376. // Deployment rollout shouldn't be blocked on hash collisions
  377. func TestDeploymentHashCollision(t *testing.T) {
  378. s, closeFn, rm, dc, informers, c := dcSetup(t)
  379. defer closeFn()
  380. name := "test-hash-collision-deployment"
  381. ns := framework.CreateTestingNamespace(name, s, t)
  382. defer framework.DeleteTestingNamespace(ns, s, t)
  383. replicas := int32(1)
  384. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
  385. var err error
  386. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  387. if err != nil {
  388. t.Fatalf("failed to create deployment %s: %v", tester.deployment.Name, err)
  389. }
  390. // Start informer and controllers
  391. stopCh := make(chan struct{})
  392. defer close(stopCh)
  393. informers.Start(stopCh)
  394. go rm.Run(5, stopCh)
  395. go dc.Run(5, stopCh)
  396. // Wait for the Deployment to be updated to revision 1
  397. if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  398. t.Fatal(err)
  399. }
  400. // Mock a hash collision
  401. newRS, err := deploymentutil.GetNewReplicaSet(tester.deployment, c.AppsV1())
  402. if err != nil {
  403. t.Fatalf("failed getting new replicaset of deployment %s: %v", tester.deployment.Name, err)
  404. }
  405. if newRS == nil {
  406. t.Fatalf("unable to find new replicaset of deployment %s", tester.deployment.Name)
  407. }
  408. _, err = tester.updateReplicaSet(newRS.Name, func(update *apps.ReplicaSet) {
  409. *update.Spec.Template.Spec.TerminationGracePeriodSeconds = int64(5)
  410. })
  411. if err != nil {
  412. t.Fatalf("failed updating replicaset %s template: %v", newRS.Name, err)
  413. }
  414. // Expect deployment collision counter to increment
  415. if err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  416. d, err := c.AppsV1().Deployments(ns.Name).Get(context.TODO(), tester.deployment.Name, metav1.GetOptions{})
  417. if err != nil {
  418. return false, nil
  419. }
  420. return d.Status.CollisionCount != nil && *d.Status.CollisionCount == int32(1), nil
  421. }); err != nil {
  422. t.Fatalf("Failed to increment collision counter for deployment %q: %v", tester.deployment.Name, err)
  423. }
  424. // Expect a new ReplicaSet to be created
  425. if err := tester.waitForDeploymentRevisionAndImage("2", fakeImage); err != nil {
  426. t.Fatal(err)
  427. }
  428. }
  429. func checkRSHashLabels(rs *apps.ReplicaSet) (string, error) {
  430. hash := rs.Labels[apps.DefaultDeploymentUniqueLabelKey]
  431. selectorHash := rs.Spec.Selector.MatchLabels[apps.DefaultDeploymentUniqueLabelKey]
  432. templateLabelHash := rs.Spec.Template.Labels[apps.DefaultDeploymentUniqueLabelKey]
  433. if hash != selectorHash || selectorHash != templateLabelHash {
  434. return "", fmt.Errorf("mismatching hash value found in replicaset %s: %#v", rs.Name, rs)
  435. }
  436. if len(hash) == 0 {
  437. return "", fmt.Errorf("unexpected replicaset %s missing required pod-template-hash labels", rs.Name)
  438. }
  439. if !strings.HasSuffix(rs.Name, hash) {
  440. return "", fmt.Errorf("unexpected replicaset %s name suffix doesn't match hash %s", rs.Name, hash)
  441. }
  442. return hash, nil
  443. }
  444. func checkPodsHashLabel(pods *v1.PodList) (string, error) {
  445. if len(pods.Items) == 0 {
  446. return "", fmt.Errorf("no pods given")
  447. }
  448. var hash string
  449. for _, pod := range pods.Items {
  450. podHash := pod.Labels[apps.DefaultDeploymentUniqueLabelKey]
  451. if len(podHash) == 0 {
  452. return "", fmt.Errorf("found pod %s missing pod-template-hash label: %#v", pod.Name, pods)
  453. }
  454. // Save the first valid hash
  455. if len(hash) == 0 {
  456. hash = podHash
  457. }
  458. if podHash != hash {
  459. return "", fmt.Errorf("found pod %s with mismatching pod-template-hash value %s: %#v", pod.Name, podHash, pods)
  460. }
  461. }
  462. return hash, nil
  463. }
  464. // Deployment should have a timeout condition when it fails to progress after given deadline.
  465. func TestFailedDeployment(t *testing.T) {
  466. s, closeFn, rm, dc, informers, c := dcSetup(t)
  467. defer closeFn()
  468. name := "test-failed-deployment"
  469. ns := framework.CreateTestingNamespace(name, s, t)
  470. defer framework.DeleteTestingNamespace(ns, s, t)
  471. deploymentName := "progress-check"
  472. replicas := int32(1)
  473. three := int32(3)
  474. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  475. tester.deployment.Spec.ProgressDeadlineSeconds = &three
  476. var err error
  477. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  478. if err != nil {
  479. t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  480. }
  481. // Start informer and controllers
  482. stopCh := make(chan struct{})
  483. defer close(stopCh)
  484. informers.Start(stopCh)
  485. go rm.Run(5, stopCh)
  486. go dc.Run(5, stopCh)
  487. if err = tester.waitForDeploymentUpdatedReplicasGTE(replicas); err != nil {
  488. t.Fatal(err)
  489. }
  490. // Pods are not marked as Ready, therefore the deployment progress will eventually timeout after progressDeadlineSeconds has passed.
  491. // Wait for the deployment to have a progress timeout condition.
  492. if err = tester.waitForDeploymentWithCondition(deploymentutil.TimedOutReason, apps.DeploymentProgressing); err != nil {
  493. t.Fatal(err)
  494. }
  495. // Manually mark pods as Ready and wait for deployment to complete.
  496. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  497. t.Fatalf("deployment %q fails to have its status becoming valid: %v", deploymentName, err)
  498. }
  499. // Wait for the deployment to have a progress complete condition.
  500. if err = tester.waitForDeploymentWithCondition(deploymentutil.NewRSAvailableReason, apps.DeploymentProgressing); err != nil {
  501. t.Fatal(err)
  502. }
  503. }
  504. func TestOverlappingDeployments(t *testing.T) {
  505. s, closeFn, rm, dc, informers, c := dcSetup(t)
  506. defer closeFn()
  507. name := "test-overlapping-deployments"
  508. ns := framework.CreateTestingNamespace(name, s, t)
  509. defer framework.DeleteTestingNamespace(ns, s, t)
  510. replicas := int32(1)
  511. firstDeploymentName := "first-deployment"
  512. secondDeploymentName := "second-deployment"
  513. testers := []*deploymentTester{
  514. {t: t, c: c, deployment: newDeployment(firstDeploymentName, ns.Name, replicas)},
  515. {t: t, c: c, deployment: newDeployment(secondDeploymentName, ns.Name, replicas)},
  516. }
  517. // Start informer and controllers
  518. stopCh := make(chan struct{})
  519. defer close(stopCh)
  520. informers.Start(stopCh)
  521. go rm.Run(5, stopCh)
  522. go dc.Run(5, stopCh)
  523. // Create 2 deployments with overlapping selectors
  524. var err error
  525. var rss []*apps.ReplicaSet
  526. for _, tester := range testers {
  527. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  528. dname := tester.deployment.Name
  529. if err != nil {
  530. t.Fatalf("failed to create deployment %q: %v", dname, err)
  531. }
  532. // Wait for the deployment to be updated to revision 1
  533. if err = tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  534. t.Fatalf("failed to update deployment %q to revision 1: %v", dname, err)
  535. }
  536. // Make sure the deployment completes while manually marking its pods as ready at the same time
  537. if err = tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  538. t.Fatalf("deployment %q failed to complete: %v", dname, err)
  539. }
  540. // Get replicaset of the deployment
  541. newRS, err := tester.getNewReplicaSet()
  542. if err != nil {
  543. t.Fatalf("failed to get new replicaset of deployment %q: %v", dname, err)
  544. }
  545. if newRS == nil {
  546. t.Fatalf("unable to find new replicaset of deployment %q", dname)
  547. }
  548. // Store the replicaset for future usage
  549. rss = append(rss, newRS)
  550. }
  551. // Both deployments should proceed independently, so their respective replicaset should not be the same replicaset
  552. if rss[0].UID == rss[1].UID {
  553. t.Fatalf("overlapping deployments should not share the same replicaset")
  554. }
  555. // Scale only the first deployment by 1
  556. newReplicas := replicas + 1
  557. testers[0].deployment, err = testers[0].updateDeployment(func(update *apps.Deployment) {
  558. update.Spec.Replicas = &newReplicas
  559. })
  560. if err != nil {
  561. t.Fatalf("failed updating deployment %q: %v", firstDeploymentName, err)
  562. }
  563. // Make sure the deployment completes after scaling
  564. if err := testers[0].waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  565. t.Fatalf("deployment %q failed to complete after scaling: %v", firstDeploymentName, err)
  566. }
  567. // Verify replicaset of both deployments has updated number of replicas
  568. for i, tester := range testers {
  569. rs, err := c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), rss[i].Name, metav1.GetOptions{})
  570. if err != nil {
  571. t.Fatalf("failed to get replicaset %q: %v", rss[i].Name, err)
  572. }
  573. if *rs.Spec.Replicas != *tester.deployment.Spec.Replicas {
  574. t.Errorf("expected replicaset %q of deployment %q has %d replicas, but found %d replicas", rs.Name, firstDeploymentName, *tester.deployment.Spec.Replicas, *rs.Spec.Replicas)
  575. }
  576. }
  577. }
  578. // Deployment should not block rollout when updating spec replica number and template at the same time.
  579. func TestScaledRolloutDeployment(t *testing.T) {
  580. s, closeFn, rm, dc, informers, c := dcSetup(t)
  581. defer closeFn()
  582. name := "test-scaled-rollout-deployment"
  583. ns := framework.CreateTestingNamespace(name, s, t)
  584. defer framework.DeleteTestingNamespace(ns, s, t)
  585. stopCh := make(chan struct{})
  586. defer close(stopCh)
  587. informers.Start(stopCh)
  588. go rm.Run(5, stopCh)
  589. go dc.Run(5, stopCh)
  590. // Create a deployment with rolling update strategy, max surge = 3, and max unavailable = 2
  591. var err error
  592. replicas := int32(10)
  593. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(name, ns.Name, replicas)}
  594. tester.deployment.Spec.Strategy.RollingUpdate.MaxSurge = intOrStrP(3)
  595. tester.deployment.Spec.Strategy.RollingUpdate.MaxUnavailable = intOrStrP(2)
  596. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  597. if err != nil {
  598. t.Fatalf("failed to create deployment %q: %v", name, err)
  599. }
  600. if err = tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  601. t.Fatal(err)
  602. }
  603. if err = tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  604. t.Fatalf("deployment %q failed to complete: %v", name, err)
  605. }
  606. // Record current replicaset before starting new rollout
  607. firstRS, err := tester.expectNewReplicaSet()
  608. if err != nil {
  609. t.Fatal(err)
  610. }
  611. // Update the deployment with another new image but do not mark the pods as ready to block new replicaset
  612. fakeImage2 := "fakeimage2"
  613. tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
  614. update.Spec.Template.Spec.Containers[0].Image = fakeImage2
  615. })
  616. if err != nil {
  617. t.Fatalf("failed updating deployment %q: %v", name, err)
  618. }
  619. if err = tester.waitForDeploymentRevisionAndImage("2", fakeImage2); err != nil {
  620. t.Fatal(err)
  621. }
  622. // Verify the deployment has minimum available replicas after 2nd rollout
  623. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{})
  624. if err != nil {
  625. t.Fatalf("failed to get deployment %q: %v", name, err)
  626. }
  627. minAvailableReplicas := deploymentutil.MinAvailable(tester.deployment)
  628. if tester.deployment.Status.AvailableReplicas < minAvailableReplicas {
  629. t.Fatalf("deployment %q does not have minimum number of available replicas after 2nd rollout", name)
  630. }
  631. // Wait for old replicaset of 1st rollout to have desired replicas
  632. firstRS, err = c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), firstRS.Name, metav1.GetOptions{})
  633. if err != nil {
  634. t.Fatalf("failed to get replicaset %q: %v", firstRS.Name, err)
  635. }
  636. if err = tester.waitRSStable(firstRS); err != nil {
  637. t.Fatal(err)
  638. }
  639. // Wait for new replicaset of 2nd rollout to have desired replicas
  640. secondRS, err := tester.expectNewReplicaSet()
  641. if err != nil {
  642. t.Fatal(err)
  643. }
  644. if err = tester.waitRSStable(secondRS); err != nil {
  645. t.Fatal(err)
  646. }
  647. // Scale up the deployment and update its image to another new image simultaneously (this time marks all pods as ready)
  648. newReplicas := int32(20)
  649. fakeImage3 := "fakeimage3"
  650. tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
  651. update.Spec.Replicas = &newReplicas
  652. update.Spec.Template.Spec.Containers[0].Image = fakeImage3
  653. })
  654. if err != nil {
  655. t.Fatalf("failed updating deployment %q: %v", name, err)
  656. }
  657. if err = tester.waitForDeploymentRevisionAndImage("3", fakeImage3); err != nil {
  658. t.Fatal(err)
  659. }
  660. if err = tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  661. t.Fatalf("deployment %q failed to complete: %v", name, err)
  662. }
  663. // Verify every replicaset has correct desiredReplicas annotation after 3rd rollout
  664. thirdRS, err := deploymentutil.GetNewReplicaSet(tester.deployment, c.AppsV1())
  665. if err != nil {
  666. t.Fatalf("failed getting new revision 3 replicaset for deployment %q: %v", name, err)
  667. }
  668. rss := []*apps.ReplicaSet{firstRS, secondRS, thirdRS}
  669. for _, curRS := range rss {
  670. curRS, err = c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), curRS.Name, metav1.GetOptions{})
  671. if err != nil {
  672. t.Fatalf("failed to get replicaset when checking desired replicas annotation: %v", err)
  673. }
  674. desired, ok := deploymentutil.GetDesiredReplicasAnnotation(curRS)
  675. if !ok {
  676. t.Fatalf("failed to retrieve desiredReplicas annotation for replicaset %q", curRS.Name)
  677. }
  678. if desired != *(tester.deployment.Spec.Replicas) {
  679. t.Fatalf("unexpected desiredReplicas annotation for replicaset %q: expected %d, got %d", curRS.Name, *(tester.deployment.Spec.Replicas), desired)
  680. }
  681. }
  682. // Update the deployment with another new image but do not mark the pods as ready to block new replicaset
  683. fakeImage4 := "fakeimage4"
  684. tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
  685. update.Spec.Template.Spec.Containers[0].Image = fakeImage4
  686. })
  687. if err != nil {
  688. t.Fatalf("failed updating deployment %q: %v", name, err)
  689. }
  690. if err = tester.waitForDeploymentRevisionAndImage("4", fakeImage4); err != nil {
  691. t.Fatal(err)
  692. }
  693. // Verify the deployment has minimum available replicas after 4th rollout
  694. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{})
  695. if err != nil {
  696. t.Fatalf("failed to get deployment %q: %v", name, err)
  697. }
  698. minAvailableReplicas = deploymentutil.MinAvailable(tester.deployment)
  699. if tester.deployment.Status.AvailableReplicas < minAvailableReplicas {
  700. t.Fatalf("deployment %q does not have minimum number of available replicas after 4th rollout", name)
  701. }
  702. // Wait for old replicaset of 3rd rollout to have desired replicas
  703. thirdRS, err = c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), thirdRS.Name, metav1.GetOptions{})
  704. if err != nil {
  705. t.Fatalf("failed to get replicaset %q: %v", thirdRS.Name, err)
  706. }
  707. if err = tester.waitRSStable(thirdRS); err != nil {
  708. t.Fatal(err)
  709. }
  710. // Wait for new replicaset of 4th rollout to have desired replicas
  711. fourthRS, err := tester.expectNewReplicaSet()
  712. if err != nil {
  713. t.Fatal(err)
  714. }
  715. if err = tester.waitRSStable(fourthRS); err != nil {
  716. t.Fatal(err)
  717. }
  718. // Scale down the deployment and update its image to another new image simultaneously (this time marks all pods as ready)
  719. newReplicas = int32(5)
  720. fakeImage5 := "fakeimage5"
  721. tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
  722. update.Spec.Replicas = &newReplicas
  723. update.Spec.Template.Spec.Containers[0].Image = fakeImage5
  724. })
  725. if err != nil {
  726. t.Fatalf("failed updating deployment %q: %v", name, err)
  727. }
  728. if err = tester.waitForDeploymentRevisionAndImage("5", fakeImage5); err != nil {
  729. t.Fatal(err)
  730. }
  731. if err = tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  732. t.Fatalf("deployment %q failed to complete: %v", name, err)
  733. }
  734. // Verify every replicaset has correct desiredReplicas annotation after 5th rollout
  735. fifthRS, err := deploymentutil.GetNewReplicaSet(tester.deployment, c.AppsV1())
  736. if err != nil {
  737. t.Fatalf("failed getting new revision 5 replicaset for deployment %q: %v", name, err)
  738. }
  739. rss = []*apps.ReplicaSet{thirdRS, fourthRS, fifthRS}
  740. for _, curRS := range rss {
  741. curRS, err = c.AppsV1().ReplicaSets(ns.Name).Get(context.TODO(), curRS.Name, metav1.GetOptions{})
  742. if err != nil {
  743. t.Fatalf("failed to get replicaset when checking desired replicas annotation: %v", err)
  744. }
  745. desired, ok := deploymentutil.GetDesiredReplicasAnnotation(curRS)
  746. if !ok {
  747. t.Fatalf("failed to retrieve desiredReplicas annotation for replicaset %q", curRS.Name)
  748. }
  749. if desired != *(tester.deployment.Spec.Replicas) {
  750. t.Fatalf("unexpected desiredReplicas annotation for replicaset %q: expected %d, got %d", curRS.Name, *(tester.deployment.Spec.Replicas), desired)
  751. }
  752. }
  753. }
  754. func TestSpecReplicasChange(t *testing.T) {
  755. s, closeFn, rm, dc, informers, c := dcSetup(t)
  756. defer closeFn()
  757. name := "test-spec-replicas-change"
  758. ns := framework.CreateTestingNamespace(name, s, t)
  759. defer framework.DeleteTestingNamespace(ns, s, t)
  760. deploymentName := "deployment"
  761. replicas := int32(1)
  762. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  763. tester.deployment.Spec.Strategy.Type = apps.RecreateDeploymentStrategyType
  764. tester.deployment.Spec.Strategy.RollingUpdate = nil
  765. var err error
  766. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  767. if err != nil {
  768. t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  769. }
  770. // Start informer and controllers
  771. stopCh := make(chan struct{})
  772. defer close(stopCh)
  773. informers.Start(stopCh)
  774. go rm.Run(5, stopCh)
  775. go dc.Run(5, stopCh)
  776. // Scale up/down deployment and verify its replicaset has matching .spec.replicas
  777. if err = tester.scaleDeployment(2); err != nil {
  778. t.Fatal(err)
  779. }
  780. if err = tester.scaleDeployment(0); err != nil {
  781. t.Fatal(err)
  782. }
  783. if err = tester.scaleDeployment(1); err != nil {
  784. t.Fatal(err)
  785. }
  786. // Add a template annotation change to test deployment's status does update
  787. // without .spec.replicas change
  788. var oldGeneration int64
  789. tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
  790. oldGeneration = update.Generation
  791. update.Spec.RevisionHistoryLimit = pointer.Int32Ptr(4)
  792. })
  793. if err != nil {
  794. t.Fatalf("failed updating deployment %q: %v", tester.deployment.Name, err)
  795. }
  796. savedGeneration := tester.deployment.Generation
  797. if savedGeneration == oldGeneration {
  798. t.Fatalf("Failed to verify .Generation has incremented for deployment %q", deploymentName)
  799. }
  800. if err = tester.waitForObservedDeployment(savedGeneration); err != nil {
  801. t.Fatal(err)
  802. }
  803. }
  804. func TestDeploymentAvailableCondition(t *testing.T) {
  805. s, closeFn, rm, dc, informers, c := dcSetup(t)
  806. defer closeFn()
  807. name := "test-deployment-available-condition"
  808. ns := framework.CreateTestingNamespace(name, s, t)
  809. defer framework.DeleteTestingNamespace(ns, s, t)
  810. deploymentName := "deployment"
  811. replicas := int32(10)
  812. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  813. // Assign a high value to the deployment's minReadySeconds
  814. tester.deployment.Spec.MinReadySeconds = 3600
  815. // progressDeadlineSeconds must be greater than minReadySeconds
  816. tester.deployment.Spec.ProgressDeadlineSeconds = pointer.Int32Ptr(7200)
  817. var err error
  818. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  819. if err != nil {
  820. t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  821. }
  822. // Start informer and controllers
  823. stopCh := make(chan struct{})
  824. defer close(stopCh)
  825. informers.Start(stopCh)
  826. go rm.Run(5, stopCh)
  827. go dc.Run(5, stopCh)
  828. // Wait for the deployment to be observed by the controller and has at least specified number of updated replicas
  829. if err = tester.waitForDeploymentUpdatedReplicasGTE(replicas); err != nil {
  830. t.Fatal(err)
  831. }
  832. // Wait for the deployment to have MinimumReplicasUnavailable reason because the pods are not marked as ready
  833. if err = tester.waitForDeploymentWithCondition(deploymentutil.MinimumReplicasUnavailable, apps.DeploymentAvailable); err != nil {
  834. t.Fatal(err)
  835. }
  836. // Verify all replicas fields of DeploymentStatus have desired counts
  837. if err = tester.checkDeploymentStatusReplicasFields(10, 10, 0, 0, 10); err != nil {
  838. t.Fatal(err)
  839. }
  840. // Mark the pods as ready without waiting for the deployment to complete
  841. if err = tester.markUpdatedPodsReadyWithoutComplete(); err != nil {
  842. t.Fatal(err)
  843. }
  844. // Wait for number of ready replicas to equal number of replicas.
  845. if err = tester.waitForReadyReplicas(); err != nil {
  846. t.Fatal(err)
  847. }
  848. // Wait for the deployment to still have MinimumReplicasUnavailable reason within minReadySeconds period
  849. if err = tester.waitForDeploymentWithCondition(deploymentutil.MinimumReplicasUnavailable, apps.DeploymentAvailable); err != nil {
  850. t.Fatal(err)
  851. }
  852. // Verify all replicas fields of DeploymentStatus have desired counts
  853. if err = tester.checkDeploymentStatusReplicasFields(10, 10, 10, 0, 10); err != nil {
  854. t.Fatal(err)
  855. }
  856. // Update the deployment's minReadySeconds to a small value
  857. tester.deployment, err = tester.updateDeployment(func(update *apps.Deployment) {
  858. update.Spec.MinReadySeconds = 1
  859. })
  860. if err != nil {
  861. t.Fatalf("failed updating deployment %q: %v", deploymentName, err)
  862. }
  863. // Wait for the deployment to notice minReadySeconds has changed
  864. if err := tester.waitForObservedDeployment(tester.deployment.Generation); err != nil {
  865. t.Fatal(err)
  866. }
  867. // Wait for the deployment to have MinimumReplicasAvailable reason after minReadySeconds period
  868. if err = tester.waitForDeploymentWithCondition(deploymentutil.MinimumReplicasAvailable, apps.DeploymentAvailable); err != nil {
  869. t.Fatal(err)
  870. }
  871. // Verify all replicas fields of DeploymentStatus have desired counts
  872. if err = tester.checkDeploymentStatusReplicasFields(10, 10, 10, 10, 0); err != nil {
  873. t.Fatal(err)
  874. }
  875. }
  876. // Wait for deployment to automatically patch incorrect ControllerRef of RS
  877. func testRSControllerRefPatch(t *testing.T, tester *deploymentTester, rs *apps.ReplicaSet, ownerReference *metav1.OwnerReference, expectedOwnerReferenceNum int) {
  878. ns := rs.Namespace
  879. rsClient := tester.c.AppsV1().ReplicaSets(ns)
  880. rs, err := tester.updateReplicaSet(rs.Name, func(update *apps.ReplicaSet) {
  881. update.OwnerReferences = []metav1.OwnerReference{*ownerReference}
  882. })
  883. if err != nil {
  884. t.Fatalf("failed to update replicaset %q: %v", rs.Name, err)
  885. }
  886. if err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  887. newRS, err := rsClient.Get(context.TODO(), rs.Name, metav1.GetOptions{})
  888. if err != nil {
  889. return false, err
  890. }
  891. return metav1.GetControllerOf(newRS) != nil, nil
  892. }); err != nil {
  893. t.Fatalf("failed to wait for controllerRef of the replicaset %q to become nil: %v", rs.Name, err)
  894. }
  895. newRS, err := rsClient.Get(context.TODO(), rs.Name, metav1.GetOptions{})
  896. if err != nil {
  897. t.Fatalf("failed to obtain replicaset %q: %v", rs.Name, err)
  898. }
  899. controllerRef := metav1.GetControllerOf(newRS)
  900. if controllerRef.UID != tester.deployment.UID {
  901. t.Fatalf("controllerRef of replicaset %q has a different UID: Expected %v, got %v", newRS.Name, tester.deployment.UID, controllerRef.UID)
  902. }
  903. ownerReferenceNum := len(newRS.GetOwnerReferences())
  904. if ownerReferenceNum != expectedOwnerReferenceNum {
  905. t.Fatalf("unexpected number of owner references for replicaset %q: Expected %d, got %d", newRS.Name, expectedOwnerReferenceNum, ownerReferenceNum)
  906. }
  907. }
  908. func TestGeneralReplicaSetAdoption(t *testing.T) {
  909. s, closeFn, rm, dc, informers, c := dcSetup(t)
  910. defer closeFn()
  911. name := "test-general-replicaset-adoption"
  912. ns := framework.CreateTestingNamespace(name, s, t)
  913. defer framework.DeleteTestingNamespace(ns, s, t)
  914. deploymentName := "deployment"
  915. replicas := int32(1)
  916. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  917. var err error
  918. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  919. if err != nil {
  920. t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  921. }
  922. // Start informer and controllers
  923. stopCh := make(chan struct{})
  924. defer close(stopCh)
  925. informers.Start(stopCh)
  926. go rm.Run(5, stopCh)
  927. go dc.Run(5, stopCh)
  928. // Wait for the Deployment to be updated to revision 1
  929. if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  930. t.Fatal(err)
  931. }
  932. // Ensure the deployment completes while marking its pods as ready simultaneously
  933. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  934. t.Fatal(err)
  935. }
  936. // Get replicaset of the deployment
  937. rs, err := deploymentutil.GetNewReplicaSet(tester.deployment, c.AppsV1())
  938. if err != nil {
  939. t.Fatalf("failed to get replicaset of deployment %q: %v", deploymentName, err)
  940. }
  941. if rs == nil {
  942. t.Fatalf("unable to find replicaset of deployment %q", deploymentName)
  943. }
  944. // When the only OwnerReference of the RS points to another type of API object such as statefulset
  945. // with Controller=false, the deployment should add a second OwnerReference (ControllerRef) pointing to itself
  946. // with Controller=true
  947. var falseVar = false
  948. ownerReference := metav1.OwnerReference{UID: uuid.NewUUID(), APIVersion: "apps/v1", Kind: "StatefulSet", Name: deploymentName, Controller: &falseVar}
  949. testRSControllerRefPatch(t, tester, rs, &ownerReference, 2)
  950. // When the only OwnerReference of the RS points to the deployment with Controller=false,
  951. // the deployment should set Controller=true for the only OwnerReference
  952. ownerReference = metav1.OwnerReference{UID: tester.deployment.UID, APIVersion: "apps/v1", Kind: "Deployment", Name: deploymentName, Controller: &falseVar}
  953. testRSControllerRefPatch(t, tester, rs, &ownerReference, 1)
  954. }
  955. func testScalingUsingScaleSubresource(t *testing.T, tester *deploymentTester, replicas int32) {
  956. ns := tester.deployment.Namespace
  957. deploymentName := tester.deployment.Name
  958. deploymentClient := tester.c.AppsV1().Deployments(ns)
  959. deployment, err := deploymentClient.Get(context.TODO(), deploymentName, metav1.GetOptions{})
  960. if err != nil {
  961. t.Fatalf("Failed to obtain deployment %q: %v", deploymentName, err)
  962. }
  963. scale, err := tester.c.AppsV1().Deployments(ns).GetScale(context.TODO(), deploymentName, metav1.GetOptions{})
  964. if err != nil {
  965. t.Fatalf("Failed to obtain scale subresource for deployment %q: %v", deploymentName, err)
  966. }
  967. if scale.Spec.Replicas != *deployment.Spec.Replicas {
  968. t.Fatalf("Scale subresource for deployment %q does not match .Spec.Replicas: expected %d, got %d", deploymentName, *deployment.Spec.Replicas, scale.Spec.Replicas)
  969. }
  970. if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
  971. scale, err := tester.c.AppsV1().Deployments(ns).GetScale(context.TODO(), deploymentName, metav1.GetOptions{})
  972. if err != nil {
  973. return err
  974. }
  975. scale.Spec.Replicas = replicas
  976. _, err = tester.c.AppsV1().Deployments(ns).UpdateScale(context.TODO(), deploymentName, scale, metav1.UpdateOptions{})
  977. return err
  978. }); err != nil {
  979. t.Fatalf("Failed to set .Spec.Replicas of scale subresource for deployment %q: %v", deploymentName, err)
  980. }
  981. deployment, err = deploymentClient.Get(context.TODO(), deploymentName, metav1.GetOptions{})
  982. if err != nil {
  983. t.Fatalf("Failed to obtain deployment %q: %v", deploymentName, err)
  984. }
  985. if *deployment.Spec.Replicas != replicas {
  986. t.Fatalf(".Spec.Replicas of deployment %q does not match its scale subresource: expected %d, got %d", deploymentName, replicas, *deployment.Spec.Replicas)
  987. }
  988. }
  989. func TestDeploymentScaleSubresource(t *testing.T) {
  990. s, closeFn, rm, dc, informers, c := dcSetup(t)
  991. defer closeFn()
  992. name := "test-deployment-scale-subresource"
  993. ns := framework.CreateTestingNamespace(name, s, t)
  994. defer framework.DeleteTestingNamespace(ns, s, t)
  995. deploymentName := "deployment"
  996. replicas := int32(2)
  997. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  998. var err error
  999. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  1000. if err != nil {
  1001. t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  1002. }
  1003. // Start informer and controllers
  1004. stopCh := make(chan struct{})
  1005. defer close(stopCh)
  1006. informers.Start(stopCh)
  1007. go rm.Run(5, stopCh)
  1008. go dc.Run(5, stopCh)
  1009. // Wait for the Deployment to be updated to revision 1
  1010. if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  1011. t.Fatal(err)
  1012. }
  1013. // Ensure the deployment completes while marking its pods as ready simultaneously
  1014. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  1015. t.Fatal(err)
  1016. }
  1017. // Use scale subresource to scale the deployment up to 3
  1018. testScalingUsingScaleSubresource(t, tester, 3)
  1019. // Use the scale subresource to scale the deployment down to 0
  1020. testScalingUsingScaleSubresource(t, tester, 0)
  1021. }
  1022. // This test verifies that the Deployment does orphan a ReplicaSet when the ReplicaSet's
  1023. // .Labels field is changed to no longer match the Deployment's selector. It also partially
  1024. // verifies that collision avoidance mechanism is triggered when a Deployment's new ReplicaSet
  1025. // is orphaned, even without PodTemplateSpec change. Refer comment below for more info:
  1026. // https://github.com/kubernetes/kubernetes/pull/59212#discussion_r166465113
  1027. func TestReplicaSetOrphaningAndAdoptionWhenLabelsChange(t *testing.T) {
  1028. s, closeFn, rm, dc, informers, c := dcSetup(t)
  1029. defer closeFn()
  1030. name := "test-replicaset-orphaning-and-adoption-when-labels-change"
  1031. ns := framework.CreateTestingNamespace(name, s, t)
  1032. defer framework.DeleteTestingNamespace(ns, s, t)
  1033. deploymentName := "deployment"
  1034. replicas := int32(1)
  1035. tester := &deploymentTester{t: t, c: c, deployment: newDeployment(deploymentName, ns.Name, replicas)}
  1036. var err error
  1037. tester.deployment, err = c.AppsV1().Deployments(ns.Name).Create(context.TODO(), tester.deployment, metav1.CreateOptions{})
  1038. if err != nil {
  1039. t.Fatalf("failed to create deployment %q: %v", deploymentName, err)
  1040. }
  1041. // Start informer and controllers
  1042. stopCh := make(chan struct{})
  1043. defer close(stopCh)
  1044. informers.Start(stopCh)
  1045. go rm.Run(5, stopCh)
  1046. go dc.Run(5, stopCh)
  1047. // Wait for the Deployment to be updated to revision 1
  1048. if err := tester.waitForDeploymentRevisionAndImage("1", fakeImage); err != nil {
  1049. t.Fatal(err)
  1050. }
  1051. // Ensure the deployment completes while marking its pods as ready simultaneously
  1052. if err := tester.waitForDeploymentCompleteAndMarkPodsReady(); err != nil {
  1053. t.Fatal(err)
  1054. }
  1055. // Orphaning: deployment should remove OwnerReference from a RS when the RS's labels change to not match its labels
  1056. // Get replicaset of the deployment
  1057. rs, err := deploymentutil.GetNewReplicaSet(tester.deployment, c.AppsV1())
  1058. if err != nil {
  1059. t.Fatalf("failed to get replicaset of deployment %q: %v", deploymentName, err)
  1060. }
  1061. if rs == nil {
  1062. t.Fatalf("unable to find replicaset of deployment %q", deploymentName)
  1063. }
  1064. // Verify controllerRef of the replicaset is not nil and pointing to the deployment
  1065. controllerRef := metav1.GetControllerOf(rs)
  1066. if controllerRef == nil {
  1067. t.Fatalf("controllerRef of replicaset %q is nil", rs.Name)
  1068. }
  1069. if controllerRef.UID != tester.deployment.UID {
  1070. t.Fatalf("controllerRef of replicaset %q has a different UID: Expected %v, got %v", rs.Name, tester.deployment.UID, controllerRef.UID)
  1071. }
  1072. // Change the replicaset's labels to not match the deployment's labels
  1073. labelMap := map[string]string{"new-name": "new-test"}
  1074. rs, err = tester.updateReplicaSet(rs.Name, func(update *apps.ReplicaSet) {
  1075. update.Labels = labelMap
  1076. })
  1077. if err != nil {
  1078. t.Fatalf("failed to update replicaset %q: %v", rs.Name, err)
  1079. }
  1080. // Wait for the controllerRef of the replicaset to become nil
  1081. rsClient := tester.c.AppsV1().ReplicaSets(ns.Name)
  1082. if err = wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  1083. rs, err = rsClient.Get(context.TODO(), rs.Name, metav1.GetOptions{})
  1084. if err != nil {
  1085. return false, err
  1086. }
  1087. return metav1.GetControllerOf(rs) == nil, nil
  1088. }); err != nil {
  1089. t.Fatalf("failed to wait for controllerRef of replicaset %q to become nil: %v", rs.Name, err)
  1090. }
  1091. // Wait for the deployment to create a new replicaset
  1092. // This will trigger collision avoidance due to deterministic nature of replicaset name
  1093. // i.e., the new replicaset will have a name with different hash to preserve name uniqueness
  1094. var newRS *apps.ReplicaSet
  1095. if err = wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  1096. newRS, err = deploymentutil.GetNewReplicaSet(tester.deployment, c.AppsV1())
  1097. if err != nil {
  1098. return false, fmt.Errorf("failed to get new replicaset of deployment %q after orphaning: %v", deploymentName, err)
  1099. }
  1100. return newRS != nil, nil
  1101. }); err != nil {
  1102. t.Fatalf("failed to wait for deployment %q to create a new replicaset after orphaning: %v", deploymentName, err)
  1103. }
  1104. if newRS.UID == rs.UID {
  1105. t.Fatalf("expect deployment %q to create a new replicaset different from the orphaned one, but it isn't", deploymentName)
  1106. }
  1107. // Adoption: deployment should add controllerRef to a RS when the RS's labels change to match its labels
  1108. // Change the old replicaset's labels to match the deployment's labels
  1109. rs, err = tester.updateReplicaSet(rs.Name, func(update *apps.ReplicaSet) {
  1110. update.Labels = testLabels()
  1111. })
  1112. if err != nil {
  1113. t.Fatalf("failed to update replicaset %q: %v", rs.Name, err)
  1114. }
  1115. // Wait for the deployment to adopt the old replicaset
  1116. if err = wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
  1117. rs, err := rsClient.Get(context.TODO(), rs.Name, metav1.GetOptions{})
  1118. if err != nil {
  1119. return false, err
  1120. }
  1121. controllerRef = metav1.GetControllerOf(rs)
  1122. return controllerRef != nil && controllerRef.UID == tester.deployment.UID, nil
  1123. }); err != nil {
  1124. t.Fatalf("failed waiting for replicaset adoption by deployment %q to complete: %v", deploymentName, err)
  1125. }
  1126. }