topology_hints_test.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640
  1. /*
  2. Copyright 2019 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package devicemanager
  14. import (
  15. "reflect"
  16. "sort"
  17. "testing"
  18. v1 "k8s.io/api/core/v1"
  19. "k8s.io/apimachinery/pkg/api/resource"
  20. "k8s.io/apimachinery/pkg/types"
  21. "k8s.io/apimachinery/pkg/util/sets"
  22. pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
  23. "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
  24. "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
  25. )
  26. type mockAffinityStore struct {
  27. hint topologymanager.TopologyHint
  28. }
  29. func (m *mockAffinityStore) GetAffinity(podUID string, containerName string) topologymanager.TopologyHint {
  30. return m.hint
  31. }
  32. func makeNUMADevice(id string, numa int) pluginapi.Device {
  33. return pluginapi.Device{
  34. ID: id,
  35. Topology: &pluginapi.TopologyInfo{Nodes: []*pluginapi.NUMANode{{ID: int64(numa)}}},
  36. }
  37. }
  38. func makeSocketMask(sockets ...int) bitmask.BitMask {
  39. mask, _ := bitmask.NewBitMask(sockets...)
  40. return mask
  41. }
  42. func TestGetTopologyHints(t *testing.T) {
  43. tcases := []struct {
  44. description string
  45. podUID string
  46. containerName string
  47. request map[string]string
  48. devices map[string][]pluginapi.Device
  49. allocatedDevices map[string]map[string]map[string][]string
  50. expectedHints map[string][]topologymanager.TopologyHint
  51. }{
  52. {
  53. description: "Single Request, no alignment",
  54. podUID: "fakePod",
  55. containerName: "fakeContainer",
  56. request: map[string]string{
  57. "testdevice": "1",
  58. },
  59. devices: map[string][]pluginapi.Device{
  60. "testdevice": {
  61. {ID: "Dev1"},
  62. {ID: "Dev2"},
  63. },
  64. },
  65. expectedHints: map[string][]topologymanager.TopologyHint{
  66. "testdevice": nil,
  67. },
  68. },
  69. {
  70. description: "Single Request, only one with alignment",
  71. podUID: "fakePod",
  72. containerName: "fakeContainer",
  73. request: map[string]string{
  74. "testdevice": "1",
  75. },
  76. devices: map[string][]pluginapi.Device{
  77. "testdevice": {
  78. {ID: "Dev1"},
  79. makeNUMADevice("Dev2", 1),
  80. },
  81. },
  82. expectedHints: map[string][]topologymanager.TopologyHint{
  83. "testdevice": {
  84. {
  85. NUMANodeAffinity: makeSocketMask(1),
  86. Preferred: true,
  87. },
  88. {
  89. NUMANodeAffinity: makeSocketMask(0, 1),
  90. Preferred: false,
  91. },
  92. },
  93. },
  94. },
  95. {
  96. description: "Single Request, one device per socket",
  97. podUID: "fakePod",
  98. containerName: "fakeContainer",
  99. request: map[string]string{
  100. "testdevice": "1",
  101. },
  102. devices: map[string][]pluginapi.Device{
  103. "testdevice": {
  104. makeNUMADevice("Dev1", 0),
  105. makeNUMADevice("Dev2", 1),
  106. },
  107. },
  108. expectedHints: map[string][]topologymanager.TopologyHint{
  109. "testdevice": {
  110. {
  111. NUMANodeAffinity: makeSocketMask(0),
  112. Preferred: true,
  113. },
  114. {
  115. NUMANodeAffinity: makeSocketMask(1),
  116. Preferred: true,
  117. },
  118. {
  119. NUMANodeAffinity: makeSocketMask(0, 1),
  120. Preferred: false,
  121. },
  122. },
  123. },
  124. },
  125. {
  126. description: "Request for 2, one device per socket",
  127. podUID: "fakePod",
  128. containerName: "fakeContainer",
  129. request: map[string]string{
  130. "testdevice": "2",
  131. },
  132. devices: map[string][]pluginapi.Device{
  133. "testdevice": {
  134. makeNUMADevice("Dev1", 0),
  135. makeNUMADevice("Dev2", 1),
  136. },
  137. },
  138. expectedHints: map[string][]topologymanager.TopologyHint{
  139. "testdevice": {
  140. {
  141. NUMANodeAffinity: makeSocketMask(0, 1),
  142. Preferred: true,
  143. },
  144. },
  145. },
  146. },
  147. {
  148. description: "Request for 2, 2 devices per socket",
  149. podUID: "fakePod",
  150. containerName: "fakeContainer",
  151. request: map[string]string{
  152. "testdevice": "2",
  153. },
  154. devices: map[string][]pluginapi.Device{
  155. "testdevice": {
  156. makeNUMADevice("Dev1", 0),
  157. makeNUMADevice("Dev2", 1),
  158. makeNUMADevice("Dev3", 0),
  159. makeNUMADevice("Dev4", 1),
  160. },
  161. },
  162. expectedHints: map[string][]topologymanager.TopologyHint{
  163. "testdevice": {
  164. {
  165. NUMANodeAffinity: makeSocketMask(0),
  166. Preferred: true,
  167. },
  168. {
  169. NUMANodeAffinity: makeSocketMask(1),
  170. Preferred: true,
  171. },
  172. {
  173. NUMANodeAffinity: makeSocketMask(0, 1),
  174. Preferred: false,
  175. },
  176. },
  177. },
  178. },
  179. {
  180. description: "Request for 2, optimal on 1 NUMA node, forced cross-NUMA",
  181. podUID: "fakePod",
  182. containerName: "fakeContainer",
  183. request: map[string]string{
  184. "testdevice": "2",
  185. },
  186. devices: map[string][]pluginapi.Device{
  187. "testdevice": {
  188. makeNUMADevice("Dev1", 0),
  189. makeNUMADevice("Dev2", 1),
  190. makeNUMADevice("Dev3", 0),
  191. makeNUMADevice("Dev4", 1),
  192. },
  193. },
  194. allocatedDevices: map[string]map[string]map[string][]string{
  195. "fakePod": {
  196. "fakeOtherContainer": {
  197. "testdevice": {"Dev1", "Dev2"},
  198. },
  199. },
  200. },
  201. expectedHints: map[string][]topologymanager.TopologyHint{
  202. "testdevice": {
  203. {
  204. NUMANodeAffinity: makeSocketMask(0, 1),
  205. Preferred: false,
  206. },
  207. },
  208. },
  209. },
  210. {
  211. description: "2 device types, mixed configuration",
  212. podUID: "fakePod",
  213. containerName: "fakeContainer",
  214. request: map[string]string{
  215. "testdevice1": "2",
  216. "testdevice2": "1",
  217. },
  218. devices: map[string][]pluginapi.Device{
  219. "testdevice1": {
  220. makeNUMADevice("Dev1", 0),
  221. makeNUMADevice("Dev2", 1),
  222. makeNUMADevice("Dev3", 0),
  223. makeNUMADevice("Dev4", 1),
  224. },
  225. "testdevice2": {
  226. makeNUMADevice("Dev1", 0),
  227. },
  228. },
  229. expectedHints: map[string][]topologymanager.TopologyHint{
  230. "testdevice1": {
  231. {
  232. NUMANodeAffinity: makeSocketMask(0),
  233. Preferred: true,
  234. },
  235. {
  236. NUMANodeAffinity: makeSocketMask(1),
  237. Preferred: true,
  238. },
  239. {
  240. NUMANodeAffinity: makeSocketMask(0, 1),
  241. Preferred: false,
  242. },
  243. },
  244. "testdevice2": {
  245. {
  246. NUMANodeAffinity: makeSocketMask(0),
  247. Preferred: true,
  248. },
  249. {
  250. NUMANodeAffinity: makeSocketMask(0, 1),
  251. Preferred: false,
  252. },
  253. },
  254. },
  255. },
  256. {
  257. description: "Single device type, more requested than available",
  258. podUID: "fakePod",
  259. containerName: "fakeContainer",
  260. request: map[string]string{
  261. "testdevice": "6",
  262. },
  263. devices: map[string][]pluginapi.Device{
  264. "testdevice": {
  265. makeNUMADevice("Dev1", 0),
  266. makeNUMADevice("Dev2", 0),
  267. makeNUMADevice("Dev3", 1),
  268. makeNUMADevice("Dev4", 1),
  269. },
  270. },
  271. expectedHints: map[string][]topologymanager.TopologyHint{
  272. "testdevice": {},
  273. },
  274. },
  275. {
  276. description: "Single device type, all already allocated to container",
  277. podUID: "fakePod",
  278. containerName: "fakeContainer",
  279. request: map[string]string{
  280. "testdevice": "2",
  281. },
  282. devices: map[string][]pluginapi.Device{
  283. "testdevice": {
  284. makeNUMADevice("Dev1", 0),
  285. makeNUMADevice("Dev2", 0),
  286. },
  287. },
  288. allocatedDevices: map[string]map[string]map[string][]string{
  289. "fakePod": {
  290. "fakeContainer": {
  291. "testdevice": {"Dev1", "Dev2"},
  292. },
  293. },
  294. },
  295. expectedHints: map[string][]topologymanager.TopologyHint{
  296. "testdevice": {
  297. {
  298. NUMANodeAffinity: makeSocketMask(0),
  299. Preferred: true,
  300. },
  301. {
  302. NUMANodeAffinity: makeSocketMask(0, 1),
  303. Preferred: false,
  304. },
  305. },
  306. },
  307. },
  308. {
  309. description: "Single device type, less already allocated to container than requested",
  310. podUID: "fakePod",
  311. containerName: "fakeContainer",
  312. request: map[string]string{
  313. "testdevice": "4",
  314. },
  315. devices: map[string][]pluginapi.Device{
  316. "testdevice": {
  317. makeNUMADevice("Dev1", 0),
  318. makeNUMADevice("Dev2", 0),
  319. makeNUMADevice("Dev3", 1),
  320. makeNUMADevice("Dev4", 1),
  321. },
  322. },
  323. allocatedDevices: map[string]map[string]map[string][]string{
  324. "fakePod": {
  325. "fakeContainer": {
  326. "testdevice": {"Dev1", "Dev2"},
  327. },
  328. },
  329. },
  330. expectedHints: map[string][]topologymanager.TopologyHint{
  331. "testdevice": {},
  332. },
  333. },
  334. {
  335. description: "Single device type, more already allocated to container than requested",
  336. podUID: "fakePod",
  337. containerName: "fakeContainer",
  338. request: map[string]string{
  339. "testdevice": "2",
  340. },
  341. devices: map[string][]pluginapi.Device{
  342. "testdevice": {
  343. makeNUMADevice("Dev1", 0),
  344. makeNUMADevice("Dev2", 0),
  345. makeNUMADevice("Dev3", 1),
  346. makeNUMADevice("Dev4", 1),
  347. },
  348. },
  349. allocatedDevices: map[string]map[string]map[string][]string{
  350. "fakePod": {
  351. "fakeContainer": {
  352. "testdevice": {"Dev1", "Dev2", "Dev3", "Dev4"},
  353. },
  354. },
  355. },
  356. expectedHints: map[string][]topologymanager.TopologyHint{
  357. "testdevice": {},
  358. },
  359. },
  360. }
  361. for _, tc := range tcases {
  362. resourceList := v1.ResourceList{}
  363. for r := range tc.request {
  364. resourceList[v1.ResourceName(r)] = resource.MustParse(tc.request[r])
  365. }
  366. pod := makePod(resourceList)
  367. pod.UID = types.UID(tc.podUID)
  368. pod.Spec.Containers[0].Name = tc.containerName
  369. m := ManagerImpl{
  370. allDevices: make(map[string]map[string]pluginapi.Device),
  371. healthyDevices: make(map[string]sets.String),
  372. allocatedDevices: make(map[string]sets.String),
  373. podDevices: make(podDevices),
  374. sourcesReady: &sourcesReadyStub{},
  375. activePods: func() []*v1.Pod { return []*v1.Pod{pod} },
  376. numaNodes: []int{0, 1},
  377. }
  378. for r := range tc.devices {
  379. m.allDevices[r] = make(map[string]pluginapi.Device)
  380. m.healthyDevices[r] = sets.NewString()
  381. for _, d := range tc.devices[r] {
  382. m.allDevices[r][d.ID] = d
  383. m.healthyDevices[r].Insert(d.ID)
  384. }
  385. }
  386. for p := range tc.allocatedDevices {
  387. for c := range tc.allocatedDevices[p] {
  388. for r, devices := range tc.allocatedDevices[p][c] {
  389. m.podDevices.insert(p, c, r, sets.NewString(devices...), nil)
  390. m.allocatedDevices[r] = sets.NewString()
  391. for _, d := range devices {
  392. m.allocatedDevices[r].Insert(d)
  393. }
  394. }
  395. }
  396. }
  397. hints := m.GetTopologyHints(pod, &pod.Spec.Containers[0])
  398. for r := range tc.expectedHints {
  399. sort.SliceStable(hints[r], func(i, j int) bool {
  400. return hints[r][i].LessThan(hints[r][j])
  401. })
  402. sort.SliceStable(tc.expectedHints[r], func(i, j int) bool {
  403. return tc.expectedHints[r][i].LessThan(tc.expectedHints[r][j])
  404. })
  405. if !reflect.DeepEqual(hints[r], tc.expectedHints[r]) {
  406. t.Errorf("%v: Expected result to be %v, got %v", tc.description, tc.expectedHints[r], hints[r])
  407. }
  408. }
  409. }
  410. }
  411. func TestTopologyAlignedAllocation(t *testing.T) {
  412. tcases := []struct {
  413. description string
  414. resource string
  415. request int
  416. devices []pluginapi.Device
  417. allocatedDevices []string
  418. hint topologymanager.TopologyHint
  419. expectedAllocation int
  420. expectedAlignment map[int]int
  421. }{
  422. {
  423. description: "Single Request, no alignment",
  424. resource: "resource",
  425. request: 1,
  426. devices: []pluginapi.Device{
  427. {ID: "Dev1"},
  428. {ID: "Dev2"},
  429. },
  430. hint: topologymanager.TopologyHint{
  431. NUMANodeAffinity: makeSocketMask(0, 1),
  432. Preferred: true,
  433. },
  434. expectedAllocation: 1,
  435. expectedAlignment: map[int]int{},
  436. },
  437. {
  438. description: "Request for 1, partial alignment",
  439. resource: "resource",
  440. request: 1,
  441. devices: []pluginapi.Device{
  442. {ID: "Dev1"},
  443. makeNUMADevice("Dev2", 1),
  444. },
  445. hint: topologymanager.TopologyHint{
  446. NUMANodeAffinity: makeSocketMask(1),
  447. Preferred: true,
  448. },
  449. expectedAllocation: 1,
  450. expectedAlignment: map[int]int{1: 1},
  451. },
  452. {
  453. description: "Single Request, socket 0",
  454. resource: "resource",
  455. request: 1,
  456. devices: []pluginapi.Device{
  457. makeNUMADevice("Dev1", 0),
  458. makeNUMADevice("Dev2", 1),
  459. },
  460. hint: topologymanager.TopologyHint{
  461. NUMANodeAffinity: makeSocketMask(0),
  462. Preferred: true,
  463. },
  464. expectedAllocation: 1,
  465. expectedAlignment: map[int]int{0: 1},
  466. },
  467. {
  468. description: "Single Request, socket 1",
  469. resource: "resource",
  470. request: 1,
  471. devices: []pluginapi.Device{
  472. makeNUMADevice("Dev1", 0),
  473. makeNUMADevice("Dev2", 1),
  474. },
  475. hint: topologymanager.TopologyHint{
  476. NUMANodeAffinity: makeSocketMask(1),
  477. Preferred: true,
  478. },
  479. expectedAllocation: 1,
  480. expectedAlignment: map[int]int{1: 1},
  481. },
  482. {
  483. description: "Request for 2, socket 0",
  484. resource: "resource",
  485. request: 2,
  486. devices: []pluginapi.Device{
  487. makeNUMADevice("Dev1", 0),
  488. makeNUMADevice("Dev2", 1),
  489. makeNUMADevice("Dev3", 0),
  490. makeNUMADevice("Dev4", 1),
  491. },
  492. hint: topologymanager.TopologyHint{
  493. NUMANodeAffinity: makeSocketMask(0),
  494. Preferred: true,
  495. },
  496. expectedAllocation: 2,
  497. expectedAlignment: map[int]int{0: 2},
  498. },
  499. {
  500. description: "Request for 2, socket 1",
  501. resource: "resource",
  502. request: 2,
  503. devices: []pluginapi.Device{
  504. makeNUMADevice("Dev1", 0),
  505. makeNUMADevice("Dev2", 1),
  506. makeNUMADevice("Dev3", 0),
  507. makeNUMADevice("Dev4", 1),
  508. },
  509. hint: topologymanager.TopologyHint{
  510. NUMANodeAffinity: makeSocketMask(1),
  511. Preferred: true,
  512. },
  513. expectedAllocation: 2,
  514. expectedAlignment: map[int]int{1: 2},
  515. },
  516. {
  517. description: "Request for 4, unsatisfiable, prefer socket 0",
  518. resource: "resource",
  519. request: 4,
  520. devices: []pluginapi.Device{
  521. makeNUMADevice("Dev1", 0),
  522. makeNUMADevice("Dev2", 1),
  523. makeNUMADevice("Dev3", 0),
  524. makeNUMADevice("Dev4", 1),
  525. makeNUMADevice("Dev5", 0),
  526. makeNUMADevice("Dev6", 1),
  527. },
  528. hint: topologymanager.TopologyHint{
  529. NUMANodeAffinity: makeSocketMask(0),
  530. Preferred: true,
  531. },
  532. expectedAllocation: 4,
  533. expectedAlignment: map[int]int{0: 3, 1: 1},
  534. },
  535. {
  536. description: "Request for 4, unsatisfiable, prefer socket 1",
  537. resource: "resource",
  538. request: 4,
  539. devices: []pluginapi.Device{
  540. makeNUMADevice("Dev1", 0),
  541. makeNUMADevice("Dev2", 1),
  542. makeNUMADevice("Dev3", 0),
  543. makeNUMADevice("Dev4", 1),
  544. makeNUMADevice("Dev5", 0),
  545. makeNUMADevice("Dev6", 1),
  546. },
  547. hint: topologymanager.TopologyHint{
  548. NUMANodeAffinity: makeSocketMask(1),
  549. Preferred: true,
  550. },
  551. expectedAllocation: 4,
  552. expectedAlignment: map[int]int{0: 1, 1: 3},
  553. },
  554. {
  555. description: "Request for 4, multisocket",
  556. resource: "resource",
  557. request: 4,
  558. devices: []pluginapi.Device{
  559. makeNUMADevice("Dev1", 0),
  560. makeNUMADevice("Dev2", 1),
  561. makeNUMADevice("Dev3", 2),
  562. makeNUMADevice("Dev4", 3),
  563. makeNUMADevice("Dev5", 0),
  564. makeNUMADevice("Dev6", 1),
  565. makeNUMADevice("Dev7", 2),
  566. makeNUMADevice("Dev8", 3),
  567. },
  568. hint: topologymanager.TopologyHint{
  569. NUMANodeAffinity: makeSocketMask(1, 3),
  570. Preferred: true,
  571. },
  572. expectedAllocation: 4,
  573. expectedAlignment: map[int]int{1: 2, 3: 2},
  574. },
  575. }
  576. for _, tc := range tcases {
  577. m := ManagerImpl{
  578. allDevices: make(map[string]map[string]pluginapi.Device),
  579. healthyDevices: make(map[string]sets.String),
  580. allocatedDevices: make(map[string]sets.String),
  581. podDevices: make(podDevices),
  582. sourcesReady: &sourcesReadyStub{},
  583. activePods: func() []*v1.Pod { return []*v1.Pod{} },
  584. topologyAffinityStore: &mockAffinityStore{tc.hint},
  585. }
  586. m.allDevices[tc.resource] = make(map[string]pluginapi.Device)
  587. m.healthyDevices[tc.resource] = sets.NewString()
  588. for _, d := range tc.devices {
  589. m.allDevices[tc.resource][d.ID] = d
  590. m.healthyDevices[tc.resource].Insert(d.ID)
  591. }
  592. allocated, err := m.devicesToAllocate("podUID", "containerName", tc.resource, tc.request, sets.NewString())
  593. if err != nil {
  594. t.Errorf("Unexpected error: %v", err)
  595. continue
  596. }
  597. if len(allocated) != tc.expectedAllocation {
  598. t.Errorf("%v. expected allocation: %v but got: %v", tc.description, tc.expectedAllocation, len(allocated))
  599. }
  600. alignment := make(map[int]int)
  601. if m.deviceHasTopologyAlignment(tc.resource) {
  602. for d := range allocated {
  603. if m.allDevices[tc.resource][d].Topology != nil {
  604. alignment[int(m.allDevices[tc.resource][d].Topology.Nodes[0].ID)]++
  605. }
  606. }
  607. }
  608. if !reflect.DeepEqual(alignment, tc.expectedAlignment) {
  609. t.Errorf("%v. expected alignment: %v but got: %v", tc.description, tc.expectedAlignment, alignment)
  610. }
  611. }
  612. }