123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640 |
- /*
- Copyright 2019 The Kubernetes Authors.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
- package devicemanager
- import (
- "reflect"
- "sort"
- "testing"
- v1 "k8s.io/api/core/v1"
- "k8s.io/apimachinery/pkg/api/resource"
- "k8s.io/apimachinery/pkg/types"
- "k8s.io/apimachinery/pkg/util/sets"
- pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
- "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
- "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
- )
- type mockAffinityStore struct {
- hint topologymanager.TopologyHint
- }
- func (m *mockAffinityStore) GetAffinity(podUID string, containerName string) topologymanager.TopologyHint {
- return m.hint
- }
- func makeNUMADevice(id string, numa int) pluginapi.Device {
- return pluginapi.Device{
- ID: id,
- Topology: &pluginapi.TopologyInfo{Nodes: []*pluginapi.NUMANode{{ID: int64(numa)}}},
- }
- }
- func makeSocketMask(sockets ...int) bitmask.BitMask {
- mask, _ := bitmask.NewBitMask(sockets...)
- return mask
- }
- func TestGetTopologyHints(t *testing.T) {
- tcases := []struct {
- description string
- podUID string
- containerName string
- request map[string]string
- devices map[string][]pluginapi.Device
- allocatedDevices map[string]map[string]map[string][]string
- expectedHints map[string][]topologymanager.TopologyHint
- }{
- {
- description: "Single Request, no alignment",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "1",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- {ID: "Dev1"},
- {ID: "Dev2"},
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": nil,
- },
- },
- {
- description: "Single Request, only one with alignment",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "1",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- {ID: "Dev1"},
- makeNUMADevice("Dev2", 1),
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {
- {
- NUMANodeAffinity: makeSocketMask(1),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: false,
- },
- },
- },
- },
- {
- description: "Single Request, one device per socket",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "1",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {
- {
- NUMANodeAffinity: makeSocketMask(0),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(1),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: false,
- },
- },
- },
- },
- {
- description: "Request for 2, one device per socket",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "2",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {
- {
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: true,
- },
- },
- },
- },
- {
- description: "Request for 2, 2 devices per socket",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "2",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- makeNUMADevice("Dev3", 0),
- makeNUMADevice("Dev4", 1),
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {
- {
- NUMANodeAffinity: makeSocketMask(0),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(1),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: false,
- },
- },
- },
- },
- {
- description: "Request for 2, optimal on 1 NUMA node, forced cross-NUMA",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "2",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- makeNUMADevice("Dev3", 0),
- makeNUMADevice("Dev4", 1),
- },
- },
- allocatedDevices: map[string]map[string]map[string][]string{
- "fakePod": {
- "fakeOtherContainer": {
- "testdevice": {"Dev1", "Dev2"},
- },
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {
- {
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: false,
- },
- },
- },
- },
- {
- description: "2 device types, mixed configuration",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice1": "2",
- "testdevice2": "1",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice1": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- makeNUMADevice("Dev3", 0),
- makeNUMADevice("Dev4", 1),
- },
- "testdevice2": {
- makeNUMADevice("Dev1", 0),
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice1": {
- {
- NUMANodeAffinity: makeSocketMask(0),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(1),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: false,
- },
- },
- "testdevice2": {
- {
- NUMANodeAffinity: makeSocketMask(0),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: false,
- },
- },
- },
- },
- {
- description: "Single device type, more requested than available",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "6",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 0),
- makeNUMADevice("Dev3", 1),
- makeNUMADevice("Dev4", 1),
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {},
- },
- },
- {
- description: "Single device type, all already allocated to container",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "2",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 0),
- },
- },
- allocatedDevices: map[string]map[string]map[string][]string{
- "fakePod": {
- "fakeContainer": {
- "testdevice": {"Dev1", "Dev2"},
- },
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {
- {
- NUMANodeAffinity: makeSocketMask(0),
- Preferred: true,
- },
- {
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: false,
- },
- },
- },
- },
- {
- description: "Single device type, less already allocated to container than requested",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "4",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 0),
- makeNUMADevice("Dev3", 1),
- makeNUMADevice("Dev4", 1),
- },
- },
- allocatedDevices: map[string]map[string]map[string][]string{
- "fakePod": {
- "fakeContainer": {
- "testdevice": {"Dev1", "Dev2"},
- },
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {},
- },
- },
- {
- description: "Single device type, more already allocated to container than requested",
- podUID: "fakePod",
- containerName: "fakeContainer",
- request: map[string]string{
- "testdevice": "2",
- },
- devices: map[string][]pluginapi.Device{
- "testdevice": {
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 0),
- makeNUMADevice("Dev3", 1),
- makeNUMADevice("Dev4", 1),
- },
- },
- allocatedDevices: map[string]map[string]map[string][]string{
- "fakePod": {
- "fakeContainer": {
- "testdevice": {"Dev1", "Dev2", "Dev3", "Dev4"},
- },
- },
- },
- expectedHints: map[string][]topologymanager.TopologyHint{
- "testdevice": {},
- },
- },
- }
- for _, tc := range tcases {
- resourceList := v1.ResourceList{}
- for r := range tc.request {
- resourceList[v1.ResourceName(r)] = resource.MustParse(tc.request[r])
- }
- pod := makePod(resourceList)
- pod.UID = types.UID(tc.podUID)
- pod.Spec.Containers[0].Name = tc.containerName
- m := ManagerImpl{
- allDevices: make(map[string]map[string]pluginapi.Device),
- healthyDevices: make(map[string]sets.String),
- allocatedDevices: make(map[string]sets.String),
- podDevices: make(podDevices),
- sourcesReady: &sourcesReadyStub{},
- activePods: func() []*v1.Pod { return []*v1.Pod{pod} },
- numaNodes: []int{0, 1},
- }
- for r := range tc.devices {
- m.allDevices[r] = make(map[string]pluginapi.Device)
- m.healthyDevices[r] = sets.NewString()
- for _, d := range tc.devices[r] {
- m.allDevices[r][d.ID] = d
- m.healthyDevices[r].Insert(d.ID)
- }
- }
- for p := range tc.allocatedDevices {
- for c := range tc.allocatedDevices[p] {
- for r, devices := range tc.allocatedDevices[p][c] {
- m.podDevices.insert(p, c, r, sets.NewString(devices...), nil)
- m.allocatedDevices[r] = sets.NewString()
- for _, d := range devices {
- m.allocatedDevices[r].Insert(d)
- }
- }
- }
- }
- hints := m.GetTopologyHints(pod, &pod.Spec.Containers[0])
- for r := range tc.expectedHints {
- sort.SliceStable(hints[r], func(i, j int) bool {
- return hints[r][i].LessThan(hints[r][j])
- })
- sort.SliceStable(tc.expectedHints[r], func(i, j int) bool {
- return tc.expectedHints[r][i].LessThan(tc.expectedHints[r][j])
- })
- if !reflect.DeepEqual(hints[r], tc.expectedHints[r]) {
- t.Errorf("%v: Expected result to be %v, got %v", tc.description, tc.expectedHints[r], hints[r])
- }
- }
- }
- }
- func TestTopologyAlignedAllocation(t *testing.T) {
- tcases := []struct {
- description string
- resource string
- request int
- devices []pluginapi.Device
- allocatedDevices []string
- hint topologymanager.TopologyHint
- expectedAllocation int
- expectedAlignment map[int]int
- }{
- {
- description: "Single Request, no alignment",
- resource: "resource",
- request: 1,
- devices: []pluginapi.Device{
- {ID: "Dev1"},
- {ID: "Dev2"},
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(0, 1),
- Preferred: true,
- },
- expectedAllocation: 1,
- expectedAlignment: map[int]int{},
- },
- {
- description: "Request for 1, partial alignment",
- resource: "resource",
- request: 1,
- devices: []pluginapi.Device{
- {ID: "Dev1"},
- makeNUMADevice("Dev2", 1),
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(1),
- Preferred: true,
- },
- expectedAllocation: 1,
- expectedAlignment: map[int]int{1: 1},
- },
- {
- description: "Single Request, socket 0",
- resource: "resource",
- request: 1,
- devices: []pluginapi.Device{
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(0),
- Preferred: true,
- },
- expectedAllocation: 1,
- expectedAlignment: map[int]int{0: 1},
- },
- {
- description: "Single Request, socket 1",
- resource: "resource",
- request: 1,
- devices: []pluginapi.Device{
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(1),
- Preferred: true,
- },
- expectedAllocation: 1,
- expectedAlignment: map[int]int{1: 1},
- },
- {
- description: "Request for 2, socket 0",
- resource: "resource",
- request: 2,
- devices: []pluginapi.Device{
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- makeNUMADevice("Dev3", 0),
- makeNUMADevice("Dev4", 1),
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(0),
- Preferred: true,
- },
- expectedAllocation: 2,
- expectedAlignment: map[int]int{0: 2},
- },
- {
- description: "Request for 2, socket 1",
- resource: "resource",
- request: 2,
- devices: []pluginapi.Device{
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- makeNUMADevice("Dev3", 0),
- makeNUMADevice("Dev4", 1),
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(1),
- Preferred: true,
- },
- expectedAllocation: 2,
- expectedAlignment: map[int]int{1: 2},
- },
- {
- description: "Request for 4, unsatisfiable, prefer socket 0",
- resource: "resource",
- request: 4,
- devices: []pluginapi.Device{
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- makeNUMADevice("Dev3", 0),
- makeNUMADevice("Dev4", 1),
- makeNUMADevice("Dev5", 0),
- makeNUMADevice("Dev6", 1),
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(0),
- Preferred: true,
- },
- expectedAllocation: 4,
- expectedAlignment: map[int]int{0: 3, 1: 1},
- },
- {
- description: "Request for 4, unsatisfiable, prefer socket 1",
- resource: "resource",
- request: 4,
- devices: []pluginapi.Device{
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- makeNUMADevice("Dev3", 0),
- makeNUMADevice("Dev4", 1),
- makeNUMADevice("Dev5", 0),
- makeNUMADevice("Dev6", 1),
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(1),
- Preferred: true,
- },
- expectedAllocation: 4,
- expectedAlignment: map[int]int{0: 1, 1: 3},
- },
- {
- description: "Request for 4, multisocket",
- resource: "resource",
- request: 4,
- devices: []pluginapi.Device{
- makeNUMADevice("Dev1", 0),
- makeNUMADevice("Dev2", 1),
- makeNUMADevice("Dev3", 2),
- makeNUMADevice("Dev4", 3),
- makeNUMADevice("Dev5", 0),
- makeNUMADevice("Dev6", 1),
- makeNUMADevice("Dev7", 2),
- makeNUMADevice("Dev8", 3),
- },
- hint: topologymanager.TopologyHint{
- NUMANodeAffinity: makeSocketMask(1, 3),
- Preferred: true,
- },
- expectedAllocation: 4,
- expectedAlignment: map[int]int{1: 2, 3: 2},
- },
- }
- for _, tc := range tcases {
- m := ManagerImpl{
- allDevices: make(map[string]map[string]pluginapi.Device),
- healthyDevices: make(map[string]sets.String),
- allocatedDevices: make(map[string]sets.String),
- podDevices: make(podDevices),
- sourcesReady: &sourcesReadyStub{},
- activePods: func() []*v1.Pod { return []*v1.Pod{} },
- topologyAffinityStore: &mockAffinityStore{tc.hint},
- }
- m.allDevices[tc.resource] = make(map[string]pluginapi.Device)
- m.healthyDevices[tc.resource] = sets.NewString()
- for _, d := range tc.devices {
- m.allDevices[tc.resource][d.ID] = d
- m.healthyDevices[tc.resource].Insert(d.ID)
- }
- allocated, err := m.devicesToAllocate("podUID", "containerName", tc.resource, tc.request, sets.NewString())
- if err != nil {
- t.Errorf("Unexpected error: %v", err)
- continue
- }
- if len(allocated) != tc.expectedAllocation {
- t.Errorf("%v. expected allocation: %v but got: %v", tc.description, tc.expectedAllocation, len(allocated))
- }
- alignment := make(map[int]int)
- if m.deviceHasTopologyAlignment(tc.resource) {
- for d := range allocated {
- if m.allDevices[tc.resource][d].Topology != nil {
- alignment[int(m.allDevices[tc.resource][d].Topology.Nodes[0].ID)]++
- }
- }
- }
- if !reflect.DeepEqual(alignment, tc.expectedAlignment) {
- t.Errorf("%v. expected alignment: %v but got: %v", tc.description, tc.expectedAlignment, alignment)
- }
- }
- }
|