framework_test.go 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551
  1. /*
  2. Copyright 2018 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package scheduler
  14. import (
  15. "context"
  16. "fmt"
  17. "testing"
  18. "time"
  19. v1 "k8s.io/api/core/v1"
  20. "k8s.io/apimachinery/pkg/api/resource"
  21. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  22. "k8s.io/apimachinery/pkg/runtime"
  23. "k8s.io/apimachinery/pkg/util/wait"
  24. clientset "k8s.io/client-go/kubernetes"
  25. scheduler "k8s.io/kubernetes/pkg/scheduler"
  26. schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
  27. "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
  28. framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
  29. schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
  30. )
  31. type PreFilterPlugin struct {
  32. numPreFilterCalled int
  33. failPreFilter bool
  34. rejectPreFilter bool
  35. }
  36. type ScorePlugin struct {
  37. failScore bool
  38. numScoreCalled int
  39. highScoreNode string
  40. }
  41. type ScoreWithNormalizePlugin struct {
  42. numScoreCalled int
  43. numNormalizeScoreCalled int
  44. }
  45. type FilterPlugin struct {
  46. numFilterCalled int
  47. failFilter bool
  48. }
  49. type ReservePlugin struct {
  50. numReserveCalled int
  51. failReserve bool
  52. }
  53. type PreScorePlugin struct {
  54. numPreScoreCalled int
  55. failPreScore bool
  56. }
  57. type PreBindPlugin struct {
  58. numPreBindCalled int
  59. failPreBind bool
  60. rejectPreBind bool
  61. }
  62. type BindPlugin struct {
  63. numBindCalled int
  64. PluginName string
  65. bindStatus *framework.Status
  66. client *clientset.Clientset
  67. pluginInvokeEventChan chan pluginInvokeEvent
  68. }
  69. type PostBindPlugin struct {
  70. name string
  71. numPostBindCalled int
  72. pluginInvokeEventChan chan pluginInvokeEvent
  73. }
  74. type UnreservePlugin struct {
  75. name string
  76. numUnreserveCalled int
  77. pluginInvokeEventChan chan pluginInvokeEvent
  78. }
  79. type PermitPlugin struct {
  80. name string
  81. numPermitCalled int
  82. failPermit bool
  83. rejectPermit bool
  84. timeoutPermit bool
  85. waitAndRejectPermit bool
  86. waitAndAllowPermit bool
  87. allowPermit bool
  88. cancelled bool
  89. fh framework.FrameworkHandle
  90. }
  91. const (
  92. prefilterPluginName = "prefilter-plugin"
  93. scorePluginName = "score-plugin"
  94. scoreWithNormalizePluginName = "score-with-normalize-plugin"
  95. filterPluginName = "filter-plugin"
  96. preScorePluginName = "prescore-plugin"
  97. reservePluginName = "reserve-plugin"
  98. preBindPluginName = "prebind-plugin"
  99. unreservePluginName = "unreserve-plugin"
  100. postBindPluginName = "postbind-plugin"
  101. permitPluginName = "permit-plugin"
  102. )
  103. var _ framework.PreFilterPlugin = &PreFilterPlugin{}
  104. var _ framework.ScorePlugin = &ScorePlugin{}
  105. var _ framework.FilterPlugin = &FilterPlugin{}
  106. var _ framework.ScorePlugin = &ScorePlugin{}
  107. var _ framework.ScorePlugin = &ScoreWithNormalizePlugin{}
  108. var _ framework.ReservePlugin = &ReservePlugin{}
  109. var _ framework.PreScorePlugin = &PreScorePlugin{}
  110. var _ framework.PreBindPlugin = &PreBindPlugin{}
  111. var _ framework.BindPlugin = &BindPlugin{}
  112. var _ framework.PostBindPlugin = &PostBindPlugin{}
  113. var _ framework.UnreservePlugin = &UnreservePlugin{}
  114. var _ framework.PermitPlugin = &PermitPlugin{}
  115. // newPlugin returns a plugin factory with specified Plugin.
  116. func newPlugin(plugin framework.Plugin) framework.PluginFactory {
  117. return func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) {
  118. return plugin, nil
  119. }
  120. }
  121. // Name returns name of the score plugin.
  122. func (sp *ScorePlugin) Name() string {
  123. return scorePluginName
  124. }
  125. // reset returns name of the score plugin.
  126. func (sp *ScorePlugin) reset() {
  127. sp.failScore = false
  128. sp.numScoreCalled = 0
  129. sp.highScoreNode = ""
  130. }
  131. // Score returns the score of scheduling a pod on a specific node.
  132. func (sp *ScorePlugin) Score(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) {
  133. sp.numScoreCalled++
  134. if sp.failScore {
  135. return 0, framework.NewStatus(framework.Error, fmt.Sprintf("injecting failure for pod %v", p.Name))
  136. }
  137. score := int64(1)
  138. if sp.numScoreCalled == 1 {
  139. // The first node is scored the highest, the rest is scored lower.
  140. sp.highScoreNode = nodeName
  141. score = framework.MaxNodeScore
  142. }
  143. return score, nil
  144. }
  145. func (sp *ScorePlugin) ScoreExtensions() framework.ScoreExtensions {
  146. return nil
  147. }
  148. // Name returns name of the score plugin.
  149. func (sp *ScoreWithNormalizePlugin) Name() string {
  150. return scoreWithNormalizePluginName
  151. }
  152. // reset returns name of the score plugin.
  153. func (sp *ScoreWithNormalizePlugin) reset() {
  154. sp.numScoreCalled = 0
  155. sp.numNormalizeScoreCalled = 0
  156. }
  157. // Score returns the score of scheduling a pod on a specific node.
  158. func (sp *ScoreWithNormalizePlugin) Score(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) (int64, *framework.Status) {
  159. sp.numScoreCalled++
  160. score := int64(10)
  161. return score, nil
  162. }
  163. func (sp *ScoreWithNormalizePlugin) NormalizeScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
  164. sp.numNormalizeScoreCalled++
  165. return nil
  166. }
  167. func (sp *ScoreWithNormalizePlugin) ScoreExtensions() framework.ScoreExtensions {
  168. return sp
  169. }
  170. // Name returns name of the plugin.
  171. func (fp *FilterPlugin) Name() string {
  172. return filterPluginName
  173. }
  174. // reset is used to reset filter plugin.
  175. func (fp *FilterPlugin) reset() {
  176. fp.numFilterCalled = 0
  177. fp.failFilter = false
  178. }
  179. // Filter is a test function that returns an error or nil, depending on the
  180. // value of "failFilter".
  181. func (fp *FilterPlugin) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *schedulernodeinfo.NodeInfo) *framework.Status {
  182. fp.numFilterCalled++
  183. if fp.failFilter {
  184. return framework.NewStatus(framework.Error, fmt.Sprintf("injecting failure for pod %v", pod.Name))
  185. }
  186. return nil
  187. }
  188. // Name returns name of the plugin.
  189. func (rp *ReservePlugin) Name() string {
  190. return reservePluginName
  191. }
  192. // Reserve is a test function that returns an error or nil, depending on the
  193. // value of "failReserve".
  194. func (rp *ReservePlugin) Reserve(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) *framework.Status {
  195. rp.numReserveCalled++
  196. if rp.failReserve {
  197. return framework.NewStatus(framework.Error, fmt.Sprintf("injecting failure for pod %v", pod.Name))
  198. }
  199. return nil
  200. }
  201. // reset used to reset reserve plugin.
  202. func (rp *ReservePlugin) reset() {
  203. rp.numReserveCalled = 0
  204. }
  205. // Name returns name of the plugin.
  206. func (*PreScorePlugin) Name() string {
  207. return preScorePluginName
  208. }
  209. // PreScore is a test function.
  210. func (pfp *PreScorePlugin) PreScore(ctx context.Context, _ *framework.CycleState, pod *v1.Pod, _ []*v1.Node) *framework.Status {
  211. pfp.numPreScoreCalled++
  212. if pfp.failPreScore {
  213. return framework.NewStatus(framework.Error, fmt.Sprintf("injecting failure for pod %v", pod.Name))
  214. }
  215. return nil
  216. }
  217. // reset used to reset prescore plugin.
  218. func (pfp *PreScorePlugin) reset() {
  219. pfp.numPreScoreCalled = 0
  220. pfp.failPreScore = false
  221. }
  222. // Name returns name of the plugin.
  223. func (pp *PreBindPlugin) Name() string {
  224. return preBindPluginName
  225. }
  226. // PreBind is a test function that returns (true, nil) or errors for testing.
  227. func (pp *PreBindPlugin) PreBind(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) *framework.Status {
  228. pp.numPreBindCalled++
  229. if pp.failPreBind {
  230. return framework.NewStatus(framework.Error, fmt.Sprintf("injecting failure for pod %v", pod.Name))
  231. }
  232. if pp.rejectPreBind {
  233. return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("reject pod %v", pod.Name))
  234. }
  235. return nil
  236. }
  237. // reset used to reset prebind plugin.
  238. func (pp *PreBindPlugin) reset() {
  239. pp.numPreBindCalled = 0
  240. pp.failPreBind = false
  241. pp.rejectPreBind = false
  242. }
  243. const bindPluginAnnotation = "bindPluginName"
  244. func (bp *BindPlugin) Name() string {
  245. return bp.PluginName
  246. }
  247. func (bp *BindPlugin) Bind(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) *framework.Status {
  248. bp.numBindCalled++
  249. if bp.pluginInvokeEventChan != nil {
  250. bp.pluginInvokeEventChan <- pluginInvokeEvent{pluginName: bp.Name(), val: bp.numBindCalled}
  251. }
  252. if bp.bindStatus.IsSuccess() {
  253. if err := bp.client.CoreV1().Pods(p.Namespace).Bind(&v1.Binding{
  254. ObjectMeta: metav1.ObjectMeta{Namespace: p.Namespace, Name: p.Name, UID: p.UID, Annotations: map[string]string{bindPluginAnnotation: bp.Name()}},
  255. Target: v1.ObjectReference{
  256. Kind: "Node",
  257. Name: nodeName,
  258. },
  259. }); err != nil {
  260. return framework.NewStatus(framework.Error, fmt.Sprintf("bind failed: %v", err))
  261. }
  262. }
  263. return bp.bindStatus
  264. }
  265. // reset used to reset numBindCalled.
  266. func (bp *BindPlugin) reset() {
  267. bp.numBindCalled = 0
  268. }
  269. // Name returns name of the plugin.
  270. func (pp *PostBindPlugin) Name() string {
  271. return pp.name
  272. }
  273. // PostBind is a test function, which counts the number of times called.
  274. func (pp *PostBindPlugin) PostBind(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) {
  275. pp.numPostBindCalled++
  276. if pp.pluginInvokeEventChan != nil {
  277. pp.pluginInvokeEventChan <- pluginInvokeEvent{pluginName: pp.Name(), val: pp.numPostBindCalled}
  278. }
  279. }
  280. // reset used to reset postbind plugin.
  281. func (pp *PostBindPlugin) reset() {
  282. pp.numPostBindCalled = 0
  283. }
  284. // Name returns name of the plugin.
  285. func (pp *PreFilterPlugin) Name() string {
  286. return prefilterPluginName
  287. }
  288. // Extensions returns the PreFilterExtensions interface.
  289. func (pp *PreFilterPlugin) PreFilterExtensions() framework.PreFilterExtensions {
  290. return nil
  291. }
  292. // PreFilter is a test function that returns (true, nil) or errors for testing.
  293. func (pp *PreFilterPlugin) PreFilter(ctx context.Context, state *framework.CycleState, pod *v1.Pod) *framework.Status {
  294. pp.numPreFilterCalled++
  295. if pp.failPreFilter {
  296. return framework.NewStatus(framework.Error, fmt.Sprintf("injecting failure for pod %v", pod.Name))
  297. }
  298. if pp.rejectPreFilter {
  299. return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("reject pod %v", pod.Name))
  300. }
  301. return nil
  302. }
  303. // reset used to reset prefilter plugin.
  304. func (pp *PreFilterPlugin) reset() {
  305. pp.numPreFilterCalled = 0
  306. pp.failPreFilter = false
  307. pp.rejectPreFilter = false
  308. }
  309. // Name returns name of the plugin.
  310. func (up *UnreservePlugin) Name() string {
  311. return up.name
  312. }
  313. // Unreserve is a test function that returns an error or nil, depending on the
  314. // value of "failUnreserve".
  315. func (up *UnreservePlugin) Unreserve(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) {
  316. up.numUnreserveCalled++
  317. if up.pluginInvokeEventChan != nil {
  318. up.pluginInvokeEventChan <- pluginInvokeEvent{pluginName: up.Name(), val: up.numUnreserveCalled}
  319. }
  320. }
  321. // reset used to reset numUnreserveCalled.
  322. func (up *UnreservePlugin) reset() {
  323. up.numUnreserveCalled = 0
  324. }
  325. // Name returns name of the plugin.
  326. func (pp *PermitPlugin) Name() string {
  327. return pp.name
  328. }
  329. // Permit implements the permit test plugin.
  330. func (pp *PermitPlugin) Permit(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (*framework.Status, time.Duration) {
  331. pp.numPermitCalled++
  332. if pp.failPermit {
  333. return framework.NewStatus(framework.Error, fmt.Sprintf("injecting failure for pod %v", pod.Name)), 0
  334. }
  335. if pp.rejectPermit {
  336. return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("reject pod %v", pod.Name)), 0
  337. }
  338. if pp.timeoutPermit {
  339. go func() {
  340. select {
  341. case <-ctx.Done():
  342. pp.cancelled = true
  343. }
  344. }()
  345. return framework.NewStatus(framework.Wait, ""), 3 * time.Second
  346. }
  347. if pp.allowPermit && pod.Name != "waiting-pod" {
  348. return nil, 0
  349. }
  350. if pp.waitAndRejectPermit || pp.waitAndAllowPermit {
  351. if pod.Name == "waiting-pod" {
  352. return framework.NewStatus(framework.Wait, ""), 30 * time.Second
  353. }
  354. // This is the signalling pod, wait until the waiting-pod is actually waiting and then either reject or allow it.
  355. wait.Poll(10*time.Millisecond, 30*time.Second, func() (bool, error) {
  356. w := false
  357. pp.fh.IterateOverWaitingPods(func(wp framework.WaitingPod) { w = true })
  358. return w, nil
  359. })
  360. if pp.waitAndRejectPermit {
  361. pp.fh.IterateOverWaitingPods(func(wp framework.WaitingPod) {
  362. wp.Reject(fmt.Sprintf("reject pod %v", wp.GetPod().Name))
  363. })
  364. return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("reject pod %v", pod.Name)), 0
  365. }
  366. if pp.waitAndAllowPermit {
  367. pp.allowAllPods()
  368. return nil, 0
  369. }
  370. }
  371. return nil, 0
  372. }
  373. // allowAllPods allows all waiting pods.
  374. func (pp *PermitPlugin) allowAllPods() {
  375. pp.fh.IterateOverWaitingPods(func(wp framework.WaitingPod) { wp.Allow(pp.name) })
  376. }
  377. // rejectAllPods rejects all waiting pods.
  378. func (pp *PermitPlugin) rejectAllPods() {
  379. pp.fh.IterateOverWaitingPods(func(wp framework.WaitingPod) { wp.Reject("rejectAllPods") })
  380. }
  381. // reset used to reset permit plugin.
  382. func (pp *PermitPlugin) reset() {
  383. pp.numPermitCalled = 0
  384. pp.failPermit = false
  385. pp.rejectPermit = false
  386. pp.timeoutPermit = false
  387. pp.waitAndRejectPermit = false
  388. pp.waitAndAllowPermit = false
  389. pp.allowPermit = false
  390. pp.cancelled = false
  391. }
  392. // newPermitPlugin returns a factory for permit plugin with specified PermitPlugin.
  393. func newPermitPlugin(permitPlugin *PermitPlugin) framework.PluginFactory {
  394. return func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) {
  395. permitPlugin.fh = fh
  396. return permitPlugin, nil
  397. }
  398. }
  399. // TestPreFilterPlugin tests invocation of prefilter plugins.
  400. func TestPreFilterPlugin(t *testing.T) {
  401. // Create a plugin registry for testing. Register only a pre-filter plugin.
  402. preFilterPlugin := &PreFilterPlugin{}
  403. registry := framework.Registry{prefilterPluginName: newPlugin(preFilterPlugin)}
  404. // Setup initial prefilter plugin for testing.
  405. plugins := &schedulerconfig.Plugins{
  406. PreFilter: &schedulerconfig.PluginSet{
  407. Enabled: []schedulerconfig.Plugin{
  408. {
  409. Name: prefilterPluginName,
  410. },
  411. },
  412. },
  413. }
  414. // Create the master and the scheduler with the test plugin set.
  415. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "prefilter-plugin", nil), 2,
  416. scheduler.WithFrameworkPlugins(plugins),
  417. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  418. defer cleanupTest(t, testCtx)
  419. tests := []struct {
  420. fail bool
  421. reject bool
  422. }{
  423. {
  424. fail: false,
  425. reject: false,
  426. },
  427. {
  428. fail: true,
  429. reject: false,
  430. },
  431. {
  432. fail: false,
  433. reject: true,
  434. },
  435. }
  436. for i, test := range tests {
  437. preFilterPlugin.failPreFilter = test.fail
  438. preFilterPlugin.rejectPreFilter = test.reject
  439. // Create a best effort pod.
  440. pod, err := createPausePod(testCtx.clientSet,
  441. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  442. if err != nil {
  443. t.Errorf("Error while creating a test pod: %v", err)
  444. }
  445. if test.reject || test.fail {
  446. if err = waitForPodUnschedulable(testCtx.clientSet, pod); err != nil {
  447. t.Errorf("test #%v: Didn't expect the pod to be scheduled. error: %v", i, err)
  448. }
  449. } else {
  450. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  451. t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err)
  452. }
  453. }
  454. if preFilterPlugin.numPreFilterCalled == 0 {
  455. t.Errorf("Expected the prefilter plugin to be called.")
  456. }
  457. preFilterPlugin.reset()
  458. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  459. }
  460. }
  461. // TestScorePlugin tests invocation of score plugins.
  462. func TestScorePlugin(t *testing.T) {
  463. // Create a plugin registry for testing. Register only a score plugin.
  464. scorePlugin := &ScorePlugin{}
  465. registry := framework.Registry{
  466. scorePluginName: newPlugin(scorePlugin),
  467. }
  468. // Setup initial score plugin for testing.
  469. plugins := &schedulerconfig.Plugins{
  470. Score: &schedulerconfig.PluginSet{
  471. Enabled: []schedulerconfig.Plugin{
  472. {
  473. Name: scorePluginName,
  474. },
  475. },
  476. },
  477. }
  478. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "score-plugin", nil), 10,
  479. scheduler.WithFrameworkPlugins(plugins),
  480. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  481. defer cleanupTest(t, testCtx)
  482. for i, fail := range []bool{false, true} {
  483. scorePlugin.failScore = fail
  484. // Create a best effort pod.
  485. pod, err := createPausePod(testCtx.clientSet,
  486. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  487. if err != nil {
  488. t.Fatalf("Error while creating a test pod: %v", err)
  489. }
  490. if fail {
  491. if err = waitForPodUnschedulable(testCtx.clientSet, pod); err != nil {
  492. t.Errorf("test #%v: Didn't expect the pod to be scheduled. error: %v", i, err)
  493. }
  494. } else {
  495. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  496. t.Errorf("Expected the pod to be scheduled. error: %v", err)
  497. } else {
  498. p, err := getPod(testCtx.clientSet, pod.Name, pod.Namespace)
  499. if err != nil {
  500. t.Errorf("Failed to retrieve the pod. error: %v", err)
  501. } else if p.Spec.NodeName != scorePlugin.highScoreNode {
  502. t.Errorf("Expected the pod to be scheduled on node %q, got %q", scorePlugin.highScoreNode, p.Spec.NodeName)
  503. }
  504. }
  505. }
  506. if scorePlugin.numScoreCalled == 0 {
  507. t.Errorf("Expected the score plugin to be called.")
  508. }
  509. scorePlugin.reset()
  510. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  511. }
  512. }
  513. // TestNormalizeScorePlugin tests invocation of normalize score plugins.
  514. func TestNormalizeScorePlugin(t *testing.T) {
  515. // Create a plugin registry for testing. Register only a normalize score plugin.
  516. scoreWithNormalizePlugin := &ScoreWithNormalizePlugin{}
  517. registry := framework.Registry{
  518. scoreWithNormalizePluginName: newPlugin(scoreWithNormalizePlugin),
  519. }
  520. // Setup initial score plugin for testing.
  521. plugins := &schedulerconfig.Plugins{
  522. Score: &schedulerconfig.PluginSet{
  523. Enabled: []schedulerconfig.Plugin{
  524. {
  525. Name: scoreWithNormalizePluginName,
  526. },
  527. },
  528. },
  529. }
  530. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "score-plugin", nil), 10,
  531. scheduler.WithFrameworkPlugins(plugins),
  532. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  533. defer cleanupTest(t, testCtx)
  534. // Create a best effort pod.
  535. pod, err := createPausePod(testCtx.clientSet,
  536. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  537. if err != nil {
  538. t.Fatalf("Error while creating a test pod: %v", err)
  539. }
  540. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  541. t.Errorf("Expected the pod to be scheduled. error: %v", err)
  542. }
  543. if scoreWithNormalizePlugin.numScoreCalled == 0 {
  544. t.Errorf("Expected the score plugin to be called.")
  545. }
  546. if scoreWithNormalizePlugin.numNormalizeScoreCalled == 0 {
  547. t.Error("Expected the normalize score plugin to be called")
  548. }
  549. scoreWithNormalizePlugin.reset()
  550. }
  551. // TestReservePlugin tests invocation of reserve plugins.
  552. func TestReservePlugin(t *testing.T) {
  553. // Create a plugin registry for testing. Register only a reserve plugin.
  554. reservePlugin := &ReservePlugin{}
  555. registry := framework.Registry{reservePluginName: newPlugin(reservePlugin)}
  556. // Setup initial reserve plugin for testing.
  557. plugins := &schedulerconfig.Plugins{
  558. Reserve: &schedulerconfig.PluginSet{
  559. Enabled: []schedulerconfig.Plugin{
  560. {
  561. Name: reservePluginName,
  562. },
  563. },
  564. },
  565. }
  566. // Create the master and the scheduler with the test plugin set.
  567. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "reserve-plugin", nil), 2,
  568. scheduler.WithFrameworkPlugins(plugins),
  569. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  570. defer cleanupTest(t, testCtx)
  571. for _, fail := range []bool{false, true} {
  572. reservePlugin.failReserve = fail
  573. // Create a best effort pod.
  574. pod, err := createPausePod(testCtx.clientSet,
  575. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  576. if err != nil {
  577. t.Errorf("Error while creating a test pod: %v", err)
  578. }
  579. if fail {
  580. if err = wait.Poll(10*time.Millisecond, 30*time.Second,
  581. podSchedulingError(testCtx.clientSet, pod.Namespace, pod.Name)); err != nil {
  582. t.Errorf("Didn't expect the pod to be scheduled. error: %v", err)
  583. }
  584. } else {
  585. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  586. t.Errorf("Expected the pod to be scheduled. error: %v", err)
  587. }
  588. }
  589. if reservePlugin.numReserveCalled == 0 {
  590. t.Errorf("Expected the reserve plugin to be called.")
  591. }
  592. reservePlugin.reset()
  593. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  594. }
  595. }
  596. // TestPrebindPlugin tests invocation of prebind plugins.
  597. func TestPrebindPlugin(t *testing.T) {
  598. // Create a plugin registry for testing. Register only a prebind plugin.
  599. preBindPlugin := &PreBindPlugin{}
  600. registry := framework.Registry{preBindPluginName: newPlugin(preBindPlugin)}
  601. // Setup initial prebind plugin for testing.
  602. plugins := &schedulerconfig.Plugins{
  603. PreBind: &schedulerconfig.PluginSet{
  604. Enabled: []schedulerconfig.Plugin{
  605. {
  606. Name: preBindPluginName,
  607. },
  608. },
  609. },
  610. }
  611. // Create the master and the scheduler with the test plugin set.
  612. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "prebind-plugin", nil), 2,
  613. scheduler.WithFrameworkPlugins(plugins),
  614. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  615. defer cleanupTest(t, testCtx)
  616. tests := []struct {
  617. fail bool
  618. reject bool
  619. }{
  620. {
  621. fail: false,
  622. reject: false,
  623. },
  624. {
  625. fail: true,
  626. reject: false,
  627. },
  628. {
  629. fail: false,
  630. reject: true,
  631. },
  632. {
  633. fail: true,
  634. reject: true,
  635. },
  636. }
  637. for i, test := range tests {
  638. preBindPlugin.failPreBind = test.fail
  639. preBindPlugin.rejectPreBind = test.reject
  640. // Create a best effort pod.
  641. pod, err := createPausePod(testCtx.clientSet,
  642. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  643. if err != nil {
  644. t.Errorf("Error while creating a test pod: %v", err)
  645. }
  646. if test.fail || test.reject {
  647. if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(testCtx.clientSet, pod.Namespace, pod.Name)); err != nil {
  648. t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err)
  649. }
  650. } else if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  651. t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err)
  652. }
  653. if preBindPlugin.numPreBindCalled == 0 {
  654. t.Errorf("Expected the prebind plugin to be called.")
  655. }
  656. preBindPlugin.reset()
  657. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  658. }
  659. }
  660. // TestUnreservePlugin tests invocation of un-reserve plugin
  661. func TestUnreservePlugin(t *testing.T) {
  662. // TODO: register more plugin which would trigger un-reserve plugin
  663. preBindPlugin := &PreBindPlugin{}
  664. unreservePlugin := &UnreservePlugin{name: unreservePluginName}
  665. registry := framework.Registry{
  666. unreservePluginName: newPlugin(unreservePlugin),
  667. preBindPluginName: newPlugin(preBindPlugin),
  668. }
  669. // Setup initial unreserve and prebind plugin for testing.
  670. plugins := &schedulerconfig.Plugins{
  671. Unreserve: &schedulerconfig.PluginSet{
  672. Enabled: []schedulerconfig.Plugin{
  673. {
  674. Name: unreservePluginName,
  675. },
  676. },
  677. },
  678. PreBind: &schedulerconfig.PluginSet{
  679. Enabled: []schedulerconfig.Plugin{
  680. {
  681. Name: preBindPluginName,
  682. },
  683. },
  684. },
  685. }
  686. // Create the master and the scheduler with the test plugin set.
  687. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "unreserve-plugin", nil), 2,
  688. scheduler.WithFrameworkPlugins(plugins),
  689. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  690. defer cleanupTest(t, testCtx)
  691. tests := []struct {
  692. preBindFail bool
  693. }{
  694. {
  695. preBindFail: false,
  696. },
  697. {
  698. preBindFail: true,
  699. },
  700. }
  701. for i, test := range tests {
  702. preBindPlugin.failPreBind = test.preBindFail
  703. // Create a best effort pod.
  704. pod, err := createPausePod(testCtx.clientSet,
  705. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  706. if err != nil {
  707. t.Errorf("Error while creating a test pod: %v", err)
  708. }
  709. if test.preBindFail {
  710. if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(testCtx.clientSet, pod.Namespace, pod.Name)); err != nil {
  711. t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err)
  712. }
  713. if unreservePlugin.numUnreserveCalled == 0 || unreservePlugin.numUnreserveCalled != preBindPlugin.numPreBindCalled {
  714. t.Errorf("test #%v: Expected the unreserve plugin to be called %d times, was called %d times.", i, preBindPlugin.numPreBindCalled, unreservePlugin.numUnreserveCalled)
  715. }
  716. } else {
  717. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  718. t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err)
  719. }
  720. if unreservePlugin.numUnreserveCalled > 0 {
  721. t.Errorf("test #%v: Didn't expected the unreserve plugin to be called, was called %d times.", i, unreservePlugin.numUnreserveCalled)
  722. }
  723. }
  724. unreservePlugin.reset()
  725. preBindPlugin.reset()
  726. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  727. }
  728. }
  729. type pluginInvokeEvent struct {
  730. pluginName string
  731. val int
  732. }
  733. // TestBindPlugin tests invocation of bind plugins.
  734. func TestBindPlugin(t *testing.T) {
  735. testContext := initTestMaster(t, "bind-plugin", nil)
  736. bindPlugin1 := &BindPlugin{PluginName: "bind-plugin-1", client: testContext.clientSet}
  737. bindPlugin2 := &BindPlugin{PluginName: "bind-plugin-2", client: testContext.clientSet}
  738. unreservePlugin := &UnreservePlugin{name: "mock-unreserve-plugin"}
  739. postBindPlugin := &PostBindPlugin{name: "mock-post-bind-plugin"}
  740. // Create a plugin registry for testing. Register an unreserve, a bind plugin and a postBind plugin.
  741. registry := framework.Registry{
  742. unreservePlugin.Name(): func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
  743. return unreservePlugin, nil
  744. },
  745. bindPlugin1.Name(): func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
  746. return bindPlugin1, nil
  747. },
  748. bindPlugin2.Name(): func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
  749. return bindPlugin2, nil
  750. },
  751. postBindPlugin.Name(): func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) {
  752. return postBindPlugin, nil
  753. },
  754. }
  755. // Setup initial unreserve and bind plugins for testing.
  756. plugins := &schedulerconfig.Plugins{
  757. Unreserve: &schedulerconfig.PluginSet{
  758. Enabled: []schedulerconfig.Plugin{{Name: unreservePlugin.Name()}},
  759. },
  760. Bind: &schedulerconfig.PluginSet{
  761. // Put DefaultBinder last.
  762. Enabled: []schedulerconfig.Plugin{{Name: bindPlugin1.Name()}, {Name: bindPlugin2.Name()}, {Name: defaultbinder.Name}},
  763. Disabled: []schedulerconfig.Plugin{{Name: defaultbinder.Name}},
  764. },
  765. PostBind: &schedulerconfig.PluginSet{
  766. Enabled: []schedulerconfig.Plugin{{Name: postBindPlugin.Name()}},
  767. },
  768. }
  769. // Create the master and the scheduler with the test plugin set.
  770. testCtx := initTestSchedulerWithOptions(t, testContext, false, nil, time.Second,
  771. scheduler.WithFrameworkPlugins(plugins),
  772. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  773. defer cleanupTest(t, testCtx)
  774. // Add a few nodes.
  775. _, err := createNodes(testCtx.clientSet, "test-node", nil, 2)
  776. if err != nil {
  777. t.Fatalf("Cannot create nodes: %v", err)
  778. }
  779. tests := []struct {
  780. bindPluginStatuses []*framework.Status
  781. expectBoundByScheduler bool // true means this test case expecting scheduler would bind pods
  782. expectBoundByPlugin bool // true means this test case expecting a plugin would bind pods
  783. expectBindPluginName string // expecting plugin name to bind pods
  784. expectInvokeEvents []pluginInvokeEvent
  785. }{
  786. // bind plugins skipped to bind the pod and scheduler bond the pod
  787. {
  788. bindPluginStatuses: []*framework.Status{framework.NewStatus(framework.Skip, ""), framework.NewStatus(framework.Skip, "")},
  789. expectBoundByScheduler: true,
  790. expectInvokeEvents: []pluginInvokeEvent{{pluginName: bindPlugin1.Name(), val: 1}, {pluginName: bindPlugin2.Name(), val: 1}, {pluginName: postBindPlugin.Name(), val: 1}},
  791. },
  792. // bindplugin2 succeeded to bind the pod
  793. {
  794. bindPluginStatuses: []*framework.Status{framework.NewStatus(framework.Skip, ""), framework.NewStatus(framework.Success, "")},
  795. expectBoundByPlugin: true,
  796. expectBindPluginName: bindPlugin2.Name(),
  797. expectInvokeEvents: []pluginInvokeEvent{{pluginName: bindPlugin1.Name(), val: 1}, {pluginName: bindPlugin2.Name(), val: 1}, {pluginName: postBindPlugin.Name(), val: 1}},
  798. },
  799. // bindplugin1 succeeded to bind the pod
  800. {
  801. bindPluginStatuses: []*framework.Status{framework.NewStatus(framework.Success, ""), framework.NewStatus(framework.Success, "")},
  802. expectBoundByPlugin: true,
  803. expectBindPluginName: bindPlugin1.Name(),
  804. expectInvokeEvents: []pluginInvokeEvent{{pluginName: bindPlugin1.Name(), val: 1}, {pluginName: postBindPlugin.Name(), val: 1}},
  805. },
  806. // bind plugin fails to bind the pod
  807. {
  808. bindPluginStatuses: []*framework.Status{framework.NewStatus(framework.Error, "failed to bind"), framework.NewStatus(framework.Success, "")},
  809. expectInvokeEvents: []pluginInvokeEvent{{pluginName: bindPlugin1.Name(), val: 1}, {pluginName: unreservePlugin.Name(), val: 1}, {pluginName: bindPlugin1.Name(), val: 2}, {pluginName: unreservePlugin.Name(), val: 2}},
  810. },
  811. }
  812. var pluginInvokeEventChan chan pluginInvokeEvent
  813. for i, test := range tests {
  814. bindPlugin1.bindStatus = test.bindPluginStatuses[0]
  815. bindPlugin2.bindStatus = test.bindPluginStatuses[1]
  816. pluginInvokeEventChan = make(chan pluginInvokeEvent, 10)
  817. bindPlugin1.pluginInvokeEventChan = pluginInvokeEventChan
  818. bindPlugin2.pluginInvokeEventChan = pluginInvokeEventChan
  819. unreservePlugin.pluginInvokeEventChan = pluginInvokeEventChan
  820. postBindPlugin.pluginInvokeEventChan = pluginInvokeEventChan
  821. // Create a best effort pod.
  822. pod, err := createPausePod(testCtx.clientSet,
  823. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  824. if err != nil {
  825. t.Errorf("Error while creating a test pod: %v", err)
  826. }
  827. if test.expectBoundByScheduler || test.expectBoundByPlugin {
  828. // bind plugins skipped to bind the pod
  829. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  830. t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err)
  831. continue
  832. }
  833. pod, err = testCtx.clientSet.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
  834. if err != nil {
  835. t.Errorf("can't get pod: %v", err)
  836. }
  837. if test.expectBoundByScheduler {
  838. if pod.Annotations[bindPluginAnnotation] != "" {
  839. t.Errorf("test #%v: Expected the pod to be bound by scheduler instead of by bindplugin %s", i, pod.Annotations[bindPluginAnnotation])
  840. }
  841. if bindPlugin1.numBindCalled != 1 || bindPlugin2.numBindCalled != 1 {
  842. t.Errorf("test #%v: Expected each bind plugin to be called once, was called %d and %d times.", i, bindPlugin1.numBindCalled, bindPlugin2.numBindCalled)
  843. }
  844. } else {
  845. if pod.Annotations[bindPluginAnnotation] != test.expectBindPluginName {
  846. t.Errorf("test #%v: Expected the pod to be bound by bindplugin %s instead of by bindplugin %s", i, test.expectBindPluginName, pod.Annotations[bindPluginAnnotation])
  847. }
  848. if bindPlugin1.numBindCalled != 1 {
  849. t.Errorf("test #%v: Expected %s to be called once, was called %d times.", i, bindPlugin1.Name(), bindPlugin1.numBindCalled)
  850. }
  851. if test.expectBindPluginName == bindPlugin1.Name() && bindPlugin2.numBindCalled > 0 {
  852. // expect bindplugin1 succeeded to bind the pod and bindplugin2 should not be called.
  853. t.Errorf("test #%v: Expected %s not to be called, was called %d times.", i, bindPlugin2.Name(), bindPlugin1.numBindCalled)
  854. }
  855. }
  856. if err = wait.Poll(10*time.Millisecond, 30*time.Second, func() (done bool, err error) {
  857. return postBindPlugin.numPostBindCalled == 1, nil
  858. }); err != nil {
  859. t.Errorf("test #%v: Expected the postbind plugin to be called once, was called %d times.", i, postBindPlugin.numPostBindCalled)
  860. }
  861. if unreservePlugin.numUnreserveCalled != 0 {
  862. t.Errorf("test #%v: Expected the unreserve plugin not to be called, was called %d times.", i, unreservePlugin.numUnreserveCalled)
  863. }
  864. } else {
  865. // bind plugin fails to bind the pod
  866. if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(testCtx.clientSet, pod.Namespace, pod.Name)); err != nil {
  867. t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err)
  868. }
  869. if postBindPlugin.numPostBindCalled > 0 {
  870. t.Errorf("test #%v: Didn't expected the postbind plugin to be called %d times.", i, postBindPlugin.numPostBindCalled)
  871. }
  872. }
  873. for j := range test.expectInvokeEvents {
  874. expectEvent := test.expectInvokeEvents[j]
  875. select {
  876. case event := <-pluginInvokeEventChan:
  877. if event.pluginName != expectEvent.pluginName {
  878. t.Errorf("test #%v: Expect invoke event %d from plugin %s instead of %s", i, j, expectEvent.pluginName, event.pluginName)
  879. }
  880. if event.val != expectEvent.val {
  881. t.Errorf("test #%v: Expect val of invoke event %d to be %d instead of %d", i, j, expectEvent.val, event.val)
  882. }
  883. case <-time.After(time.Second * 30):
  884. t.Errorf("test #%v: Waiting for invoke event %d timeout.", i, j)
  885. }
  886. }
  887. postBindPlugin.reset()
  888. bindPlugin1.reset()
  889. bindPlugin2.reset()
  890. unreservePlugin.reset()
  891. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  892. }
  893. }
  894. // TestPostBindPlugin tests invocation of postbind plugins.
  895. func TestPostBindPlugin(t *testing.T) {
  896. // Create a plugin registry for testing. Register a prebind and a postbind plugin.
  897. preBindPlugin := &PreBindPlugin{}
  898. postBindPlugin := &PostBindPlugin{name: postBindPluginName}
  899. registry := framework.Registry{
  900. preBindPluginName: newPlugin(preBindPlugin),
  901. postBindPluginName: newPlugin(postBindPlugin),
  902. }
  903. // Setup initial prebind and postbind plugin for testing.
  904. plugins := &schedulerconfig.Plugins{
  905. PreBind: &schedulerconfig.PluginSet{
  906. Enabled: []schedulerconfig.Plugin{
  907. {
  908. Name: preBindPluginName,
  909. },
  910. },
  911. },
  912. PostBind: &schedulerconfig.PluginSet{
  913. Enabled: []schedulerconfig.Plugin{
  914. {
  915. Name: postBindPluginName,
  916. },
  917. },
  918. },
  919. }
  920. // Create the master and the scheduler with the test plugin set.
  921. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "postbind-plugin", nil), 2,
  922. scheduler.WithFrameworkPlugins(plugins),
  923. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  924. defer cleanupTest(t, testCtx)
  925. tests := []struct {
  926. preBindFail bool
  927. preBindReject bool
  928. }{
  929. {
  930. preBindFail: false,
  931. },
  932. {
  933. preBindFail: true,
  934. },
  935. }
  936. for i, test := range tests {
  937. preBindPlugin.failPreBind = test.preBindFail
  938. // Create a best effort pod.
  939. pod, err := createPausePod(testCtx.clientSet,
  940. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  941. if err != nil {
  942. t.Errorf("Error while creating a test pod: %v", err)
  943. }
  944. if test.preBindFail {
  945. if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(testCtx.clientSet, pod.Namespace, pod.Name)); err != nil {
  946. t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err)
  947. }
  948. if postBindPlugin.numPostBindCalled > 0 {
  949. t.Errorf("test #%v: Didn't expected the postbind plugin to be called %d times.", i, postBindPlugin.numPostBindCalled)
  950. }
  951. } else {
  952. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  953. t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err)
  954. }
  955. if postBindPlugin.numPostBindCalled == 0 {
  956. t.Errorf("test #%v: Expected the postbind plugin to be called, was called %d times.", i, postBindPlugin.numPostBindCalled)
  957. }
  958. }
  959. postBindPlugin.reset()
  960. preBindPlugin.reset()
  961. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  962. }
  963. }
  964. // TestPermitPlugin tests invocation of permit plugins.
  965. func TestPermitPlugin(t *testing.T) {
  966. // Create a plugin registry for testing. Register only a permit plugin.
  967. perPlugin := &PermitPlugin{name: permitPluginName}
  968. registry, plugins := initRegistryAndConfig(perPlugin)
  969. // Create the master and the scheduler with the test plugin set.
  970. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "permit-plugin", nil), 2,
  971. scheduler.WithFrameworkPlugins(plugins),
  972. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  973. defer cleanupTest(t, testCtx)
  974. tests := []struct {
  975. fail bool
  976. reject bool
  977. timeout bool
  978. }{
  979. {
  980. fail: false,
  981. reject: false,
  982. timeout: false,
  983. },
  984. {
  985. fail: true,
  986. reject: false,
  987. timeout: false,
  988. },
  989. {
  990. fail: false,
  991. reject: true,
  992. timeout: false,
  993. },
  994. {
  995. fail: true,
  996. reject: true,
  997. timeout: false,
  998. },
  999. {
  1000. fail: false,
  1001. reject: false,
  1002. timeout: true,
  1003. },
  1004. {
  1005. fail: false,
  1006. reject: false,
  1007. timeout: true,
  1008. },
  1009. }
  1010. for i, test := range tests {
  1011. perPlugin.failPermit = test.fail
  1012. perPlugin.rejectPermit = test.reject
  1013. perPlugin.timeoutPermit = test.timeout
  1014. perPlugin.waitAndRejectPermit = false
  1015. perPlugin.waitAndAllowPermit = false
  1016. // Create a best effort pod.
  1017. pod, err := createPausePod(testCtx.clientSet,
  1018. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  1019. if err != nil {
  1020. t.Errorf("Error while creating a test pod: %v", err)
  1021. }
  1022. if test.fail {
  1023. if err = wait.Poll(10*time.Millisecond, 30*time.Second, podSchedulingError(testCtx.clientSet, pod.Namespace, pod.Name)); err != nil {
  1024. t.Errorf("test #%v: Expected a scheduling error, but didn't get it. error: %v", i, err)
  1025. }
  1026. } else {
  1027. if test.reject || test.timeout {
  1028. if err = waitForPodUnschedulable(testCtx.clientSet, pod); err != nil {
  1029. t.Errorf("test #%v: Didn't expect the pod to be scheduled. error: %v", i, err)
  1030. }
  1031. } else {
  1032. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  1033. t.Errorf("test #%v: Expected the pod to be scheduled. error: %v", i, err)
  1034. }
  1035. }
  1036. }
  1037. if perPlugin.numPermitCalled == 0 {
  1038. t.Errorf("Expected the permit plugin to be called.")
  1039. }
  1040. perPlugin.reset()
  1041. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  1042. }
  1043. }
  1044. // TestMultiplePermitPlugins tests multiple permit plugins returning wait for a same pod.
  1045. func TestMultiplePermitPlugins(t *testing.T) {
  1046. // Create a plugin registry for testing.
  1047. perPlugin1 := &PermitPlugin{name: "permit-plugin-1"}
  1048. perPlugin2 := &PermitPlugin{name: "permit-plugin-2"}
  1049. registry, plugins := initRegistryAndConfig(perPlugin1, perPlugin2)
  1050. // Create the master and the scheduler with the test plugin set.
  1051. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "multi-permit-plugin", nil), 2,
  1052. scheduler.WithFrameworkPlugins(plugins),
  1053. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  1054. defer cleanupTest(t, testCtx)
  1055. // Both permit plugins will return Wait for permitting
  1056. perPlugin1.timeoutPermit = true
  1057. perPlugin2.timeoutPermit = true
  1058. // Create a test pod.
  1059. podName := "test-pod"
  1060. pod, err := createPausePod(testCtx.clientSet,
  1061. initPausePod(testCtx.clientSet, &pausePodConfig{Name: podName, Namespace: testCtx.ns.Name}))
  1062. if err != nil {
  1063. t.Errorf("Error while creating a test pod: %v", err)
  1064. }
  1065. var waitingPod framework.WaitingPod
  1066. // Wait until the test pod is actually waiting.
  1067. wait.Poll(10*time.Millisecond, 30*time.Second, func() (bool, error) {
  1068. waitingPod = perPlugin1.fh.GetWaitingPod(pod.UID)
  1069. return waitingPod != nil, nil
  1070. })
  1071. // Check the number of pending permits
  1072. if l := len(waitingPod.GetPendingPlugins()); l != 2 {
  1073. t.Errorf("Expected the number of pending plugins is 2, but got %d", l)
  1074. }
  1075. perPlugin1.allowAllPods()
  1076. // Check the number of pending permits
  1077. if l := len(waitingPod.GetPendingPlugins()); l != 1 {
  1078. t.Errorf("Expected the number of pending plugins is 1, but got %d", l)
  1079. }
  1080. perPlugin2.allowAllPods()
  1081. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  1082. t.Errorf("Expected the pod to be scheduled. error: %v", err)
  1083. }
  1084. if perPlugin1.numPermitCalled == 0 || perPlugin2.numPermitCalled == 0 {
  1085. t.Errorf("Expected the permit plugin to be called.")
  1086. }
  1087. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  1088. }
  1089. // TestPermitPluginsCancelled tests whether all permit plugins are cancelled when pod is rejected.
  1090. func TestPermitPluginsCancelled(t *testing.T) {
  1091. // Create a plugin registry for testing.
  1092. perPlugin1 := &PermitPlugin{name: "permit-plugin-1"}
  1093. perPlugin2 := &PermitPlugin{name: "permit-plugin-2"}
  1094. registry, plugins := initRegistryAndConfig(perPlugin1, perPlugin2)
  1095. // Create the master and the scheduler with the test plugin set.
  1096. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "permit-plugins", nil), 2,
  1097. scheduler.WithFrameworkPlugins(plugins),
  1098. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  1099. defer cleanupTest(t, testCtx)
  1100. // Both permit plugins will return Wait for permitting
  1101. perPlugin1.timeoutPermit = true
  1102. perPlugin2.timeoutPermit = true
  1103. // Create a test pod.
  1104. podName := "test-pod"
  1105. pod, err := createPausePod(testCtx.clientSet,
  1106. initPausePod(testCtx.clientSet, &pausePodConfig{Name: podName, Namespace: testCtx.ns.Name}))
  1107. if err != nil {
  1108. t.Errorf("Error while creating a test pod: %v", err)
  1109. }
  1110. var waitingPod framework.WaitingPod
  1111. // Wait until the test pod is actually waiting.
  1112. wait.Poll(10*time.Millisecond, 30*time.Second, func() (bool, error) {
  1113. waitingPod = perPlugin1.fh.GetWaitingPod(pod.UID)
  1114. return waitingPod != nil, nil
  1115. })
  1116. perPlugin1.rejectAllPods()
  1117. // Wait some time for the permit plugins to be cancelled
  1118. err = wait.Poll(10*time.Millisecond, 30*time.Second, func() (bool, error) {
  1119. return perPlugin1.cancelled && perPlugin2.cancelled, nil
  1120. })
  1121. if err != nil {
  1122. t.Errorf("Expected all permit plugins to be cancelled")
  1123. }
  1124. }
  1125. // TestCoSchedulingWithPermitPlugin tests invocation of permit plugins.
  1126. func TestCoSchedulingWithPermitPlugin(t *testing.T) {
  1127. // Create a plugin registry for testing. Register only a permit plugin.
  1128. permitPlugin := &PermitPlugin{name: permitPluginName}
  1129. registry, plugins := initRegistryAndConfig(permitPlugin)
  1130. // Create the master and the scheduler with the test plugin set.
  1131. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "permit-plugin", nil), 2,
  1132. scheduler.WithFrameworkPlugins(plugins),
  1133. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  1134. defer cleanupTest(t, testCtx)
  1135. tests := []struct {
  1136. waitReject bool
  1137. waitAllow bool
  1138. }{
  1139. {
  1140. waitReject: true,
  1141. waitAllow: false,
  1142. },
  1143. {
  1144. waitReject: false,
  1145. waitAllow: true,
  1146. },
  1147. }
  1148. for i, test := range tests {
  1149. permitPlugin.failPermit = false
  1150. permitPlugin.rejectPermit = false
  1151. permitPlugin.timeoutPermit = false
  1152. permitPlugin.waitAndRejectPermit = test.waitReject
  1153. permitPlugin.waitAndAllowPermit = test.waitAllow
  1154. // Create two pods.
  1155. waitingPod, err := createPausePod(testCtx.clientSet,
  1156. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "waiting-pod", Namespace: testCtx.ns.Name}))
  1157. if err != nil {
  1158. t.Errorf("Error while creating the waiting pod: %v", err)
  1159. }
  1160. signallingPod, err := createPausePod(testCtx.clientSet,
  1161. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "signalling-pod", Namespace: testCtx.ns.Name}))
  1162. if err != nil {
  1163. t.Errorf("Error while creating the signalling pod: %v", err)
  1164. }
  1165. if test.waitReject {
  1166. if err = waitForPodUnschedulable(testCtx.clientSet, waitingPod); err != nil {
  1167. t.Errorf("test #%v: Didn't expect the waiting pod to be scheduled. error: %v", i, err)
  1168. }
  1169. if err = waitForPodUnschedulable(testCtx.clientSet, signallingPod); err != nil {
  1170. t.Errorf("test #%v: Didn't expect the signalling pod to be scheduled. error: %v", i, err)
  1171. }
  1172. } else {
  1173. if err = waitForPodToSchedule(testCtx.clientSet, waitingPod); err != nil {
  1174. t.Errorf("test #%v: Expected the waiting pod to be scheduled. error: %v", i, err)
  1175. }
  1176. if err = waitForPodToSchedule(testCtx.clientSet, signallingPod); err != nil {
  1177. t.Errorf("test #%v: Expected the signalling pod to be scheduled. error: %v", i, err)
  1178. }
  1179. }
  1180. if permitPlugin.numPermitCalled == 0 {
  1181. t.Errorf("Expected the permit plugin to be called.")
  1182. }
  1183. permitPlugin.reset()
  1184. cleanupPods(testCtx.clientSet, t, []*v1.Pod{waitingPod, signallingPod})
  1185. }
  1186. }
  1187. // TestFilterPlugin tests invocation of filter plugins.
  1188. func TestFilterPlugin(t *testing.T) {
  1189. // Create a plugin registry for testing. Register only a filter plugin.
  1190. filterPlugin := &FilterPlugin{}
  1191. registry := framework.Registry{filterPluginName: newPlugin(filterPlugin)}
  1192. // Setup initial filter plugin for testing.
  1193. plugins := &schedulerconfig.Plugins{
  1194. Filter: &schedulerconfig.PluginSet{
  1195. Enabled: []schedulerconfig.Plugin{
  1196. {
  1197. Name: filterPluginName,
  1198. },
  1199. },
  1200. },
  1201. }
  1202. // Create the master and the scheduler with the test plugin set.
  1203. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "filter-plugin", nil), 2,
  1204. scheduler.WithFrameworkPlugins(plugins),
  1205. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  1206. defer cleanupTest(t, testCtx)
  1207. for _, fail := range []bool{false, true} {
  1208. filterPlugin.failFilter = fail
  1209. // Create a best effort pod.
  1210. pod, err := createPausePod(testCtx.clientSet,
  1211. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  1212. if err != nil {
  1213. t.Errorf("Error while creating a test pod: %v", err)
  1214. }
  1215. if fail {
  1216. if err = wait.Poll(10*time.Millisecond, 30*time.Second, podUnschedulable(testCtx.clientSet, pod.Namespace, pod.Name)); err != nil {
  1217. t.Errorf("Didn't expect the pod to be scheduled.")
  1218. }
  1219. } else {
  1220. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  1221. t.Errorf("Expected the pod to be scheduled. error: %v", err)
  1222. }
  1223. }
  1224. if filterPlugin.numFilterCalled == 0 {
  1225. t.Errorf("Expected the filter plugin to be called.")
  1226. }
  1227. filterPlugin.reset()
  1228. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  1229. }
  1230. }
  1231. // TestPreScorePlugin tests invocation of pre-score plugins.
  1232. func TestPreScorePlugin(t *testing.T) {
  1233. // Create a plugin registry for testing. Register only a pre-score plugin.
  1234. preScorePlugin := &PreScorePlugin{}
  1235. registry := framework.Registry{preScorePluginName: newPlugin(preScorePlugin)}
  1236. // Setup initial pre-score plugin for testing.
  1237. plugins := &schedulerconfig.Plugins{
  1238. PreScore: &schedulerconfig.PluginSet{
  1239. Enabled: []schedulerconfig.Plugin{
  1240. {
  1241. Name: preScorePluginName,
  1242. },
  1243. },
  1244. },
  1245. }
  1246. // Create the master and the scheduler with the test plugin set.
  1247. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "pre-score-plugin", nil), 2,
  1248. scheduler.WithFrameworkPlugins(plugins),
  1249. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  1250. defer cleanupTest(t, testCtx)
  1251. for _, fail := range []bool{false, true} {
  1252. preScorePlugin.failPreScore = fail
  1253. // Create a best effort pod.
  1254. pod, err := createPausePod(testCtx.clientSet,
  1255. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "test-pod", Namespace: testCtx.ns.Name}))
  1256. if err != nil {
  1257. t.Errorf("Error while creating a test pod: %v", err)
  1258. }
  1259. if fail {
  1260. if err = waitForPodUnschedulable(testCtx.clientSet, pod); err != nil {
  1261. t.Errorf("Didn't expect the pod to be scheduled. error: %v", err)
  1262. }
  1263. } else {
  1264. if err = waitForPodToSchedule(testCtx.clientSet, pod); err != nil {
  1265. t.Errorf("Expected the pod to be scheduled. error: %v", err)
  1266. }
  1267. }
  1268. if preScorePlugin.numPreScoreCalled == 0 {
  1269. t.Errorf("Expected the pre-score plugin to be called.")
  1270. }
  1271. preScorePlugin.reset()
  1272. cleanupPods(testCtx.clientSet, t, []*v1.Pod{pod})
  1273. }
  1274. }
  1275. // TestPreemptWithPermitPlugin tests preempt with permit plugins.
  1276. func TestPreemptWithPermitPlugin(t *testing.T) {
  1277. // Create a plugin registry for testing. Register only a permit plugin.
  1278. permitPlugin := &PermitPlugin{}
  1279. registry, plugins := initRegistryAndConfig(permitPlugin)
  1280. // Create the master and the scheduler with the test plugin set.
  1281. testCtx := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "preempt-with-permit-plugin", nil), 0,
  1282. scheduler.WithFrameworkPlugins(plugins),
  1283. scheduler.WithFrameworkOutOfTreeRegistry(registry))
  1284. defer cleanupTest(t, testCtx)
  1285. // Add one node.
  1286. nodeRes := &v1.ResourceList{
  1287. v1.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
  1288. v1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI),
  1289. v1.ResourceMemory: *resource.NewQuantity(500, resource.DecimalSI),
  1290. }
  1291. _, err := createNodes(testCtx.clientSet, "test-node", nodeRes, 1)
  1292. if err != nil {
  1293. t.Fatalf("Cannot create nodes: %v", err)
  1294. }
  1295. permitPlugin.failPermit = false
  1296. permitPlugin.rejectPermit = false
  1297. permitPlugin.timeoutPermit = false
  1298. permitPlugin.waitAndRejectPermit = false
  1299. permitPlugin.waitAndAllowPermit = true
  1300. permitPlugin.allowPermit = true
  1301. lowPriority, highPriority := int32(100), int32(300)
  1302. resourceRequest := v1.ResourceRequirements{Requests: v1.ResourceList{
  1303. v1.ResourceCPU: *resource.NewMilliQuantity(400, resource.DecimalSI),
  1304. v1.ResourceMemory: *resource.NewQuantity(400, resource.DecimalSI)},
  1305. }
  1306. // Create two pods.
  1307. waitingPod := initPausePod(testCtx.clientSet, &pausePodConfig{Name: "waiting-pod", Namespace: testCtx.ns.Name, Priority: &lowPriority, Resources: &resourceRequest})
  1308. waitingPod.Spec.TerminationGracePeriodSeconds = new(int64)
  1309. waitingPod, err = createPausePod(testCtx.clientSet, waitingPod)
  1310. if err != nil {
  1311. t.Errorf("Error while creating the waiting pod: %v", err)
  1312. }
  1313. // Wait until the waiting-pod is actually waiting, then create a preemptor pod to preempt it.
  1314. wait.Poll(10*time.Millisecond, 30*time.Second, func() (bool, error) {
  1315. w := false
  1316. permitPlugin.fh.IterateOverWaitingPods(func(wp framework.WaitingPod) { w = true })
  1317. return w, nil
  1318. })
  1319. preemptorPod, err := createPausePod(testCtx.clientSet,
  1320. initPausePod(testCtx.clientSet, &pausePodConfig{Name: "preemptor-pod", Namespace: testCtx.ns.Name, Priority: &highPriority, Resources: &resourceRequest}))
  1321. if err != nil {
  1322. t.Errorf("Error while creating the preemptor pod: %v", err)
  1323. }
  1324. if err = waitForPodToSchedule(testCtx.clientSet, preemptorPod); err != nil {
  1325. t.Errorf("Expected the preemptor pod to be scheduled. error: %v", err)
  1326. }
  1327. if _, err := getPod(testCtx.clientSet, waitingPod.Name, waitingPod.Namespace); err == nil {
  1328. t.Error("Expected the waiting pod to get preempted and deleted")
  1329. }
  1330. if permitPlugin.numPermitCalled == 0 {
  1331. t.Errorf("Expected the permit plugin to be called.")
  1332. }
  1333. permitPlugin.reset()
  1334. cleanupPods(testCtx.clientSet, t, []*v1.Pod{waitingPod, preemptorPod})
  1335. }
  1336. func initTestSchedulerForFrameworkTest(t *testing.T, testCtx *testContext, nodeCount int, opts ...scheduler.Option) *testContext {
  1337. c := initTestSchedulerWithOptions(t, testCtx, false, nil, time.Second, opts...)
  1338. if nodeCount > 0 {
  1339. _, err := createNodes(c.clientSet, "test-node", nil, nodeCount)
  1340. if err != nil {
  1341. t.Fatalf("Cannot create nodes: %v", err)
  1342. }
  1343. }
  1344. return c
  1345. }
  1346. // initRegistryAndConfig returns registry and plugins config based on give plugins.
  1347. // TODO: refactor it to a more generic functions that accepts all kinds of Plugins as arguments
  1348. func initRegistryAndConfig(pp ...*PermitPlugin) (registry framework.Registry, plugins *schedulerconfig.Plugins) {
  1349. if len(pp) == 0 {
  1350. return
  1351. }
  1352. registry = framework.Registry{}
  1353. plugins = &schedulerconfig.Plugins{
  1354. Permit: &schedulerconfig.PluginSet{},
  1355. }
  1356. for _, p := range pp {
  1357. registry.Register(p.Name(), newPermitPlugin(p))
  1358. plugins.Permit.Enabled = append(plugins.Permit.Enabled, schedulerconfig.Plugin{Name: p.Name()})
  1359. }
  1360. return
  1361. }