topology_manager.go 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
  1. /*
  2. Copyright 2019 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package topologymanager
  14. import (
  15. "fmt"
  16. "k8s.io/api/core/v1"
  17. "k8s.io/klog"
  18. cputopology "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
  19. "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
  20. "k8s.io/kubernetes/pkg/kubelet/lifecycle"
  21. )
  22. const (
  23. // maxAllowableNUMANodes specifies the maximum number of NUMA Nodes that
  24. // the TopologyManager supports on the underlying machine.
  25. //
  26. // At present, having more than this number of NUMA Nodes will result in a
  27. // state explosion when trying to enumerate possible NUMAAffinity masks and
  28. // generate hints for them. As such, if more NUMA Nodes than this are
  29. // present on a machine and the TopologyManager is enabled, an error will
  30. // be returned and the TopologyManager will not be loaded.
  31. maxAllowableNUMANodes = 8
  32. )
  33. //Manager interface provides methods for Kubelet to manage pod topology hints
  34. type Manager interface {
  35. //Manager implements pod admit handler interface
  36. lifecycle.PodAdmitHandler
  37. //Adds a hint provider to manager to indicate the hint provider
  38. //wants to be consoluted when making topology hints
  39. AddHintProvider(HintProvider)
  40. //Adds pod to Manager for tracking
  41. AddContainer(pod *v1.Pod, containerID string) error
  42. //Removes pod from Manager tracking
  43. RemoveContainer(containerID string) error
  44. //Interface for storing pod topology hints
  45. Store
  46. }
  47. type manager struct {
  48. //The list of components registered with the Manager
  49. hintProviders []HintProvider
  50. //Mapping of a Pods mapping of Containers and their TopologyHints
  51. //Indexed by PodUID to ContainerName
  52. podTopologyHints map[string]map[string]TopologyHint
  53. //Mapping of PodUID to ContainerID for Adding/Removing Pods from PodTopologyHints mapping
  54. podMap map[string]string
  55. //Topology Manager Policy
  56. policy Policy
  57. }
  58. // HintProvider is an interface for components that want to collaborate to
  59. // achieve globally optimal concrete resource alignment with respect to
  60. // NUMA locality.
  61. type HintProvider interface {
  62. // GetTopologyHints returns a map of resource names to a list of possible
  63. // concrete resource allocations in terms of NUMA locality hints. Each hint
  64. // is optionally marked "preferred" and indicates the set of NUMA nodes
  65. // involved in the hypothetical allocation. The topology manager calls
  66. // this function for each hint provider, and merges the hints to produce
  67. // a consensus "best" hint. The hint providers may subsequently query the
  68. // topology manager to influence actual resource assignment.
  69. GetTopologyHints(pod *v1.Pod, container *v1.Container) map[string][]TopologyHint
  70. }
  71. //Store interface is to allow Hint Providers to retrieve pod affinity
  72. type Store interface {
  73. GetAffinity(podUID string, containerName string) TopologyHint
  74. }
  75. //TopologyHint is a struct containing the NUMANodeAffinity for a Container
  76. type TopologyHint struct {
  77. NUMANodeAffinity bitmask.BitMask
  78. // Preferred is set to true when the NUMANodeAffinity encodes a preferred
  79. // allocation for the Container. It is set to false otherwise.
  80. Preferred bool
  81. }
  82. // IsEqual checks if TopologyHint are equal
  83. func (th *TopologyHint) IsEqual(topologyHint TopologyHint) bool {
  84. if th.Preferred == topologyHint.Preferred {
  85. if th.NUMANodeAffinity == nil || topologyHint.NUMANodeAffinity == nil {
  86. return th.NUMANodeAffinity == topologyHint.NUMANodeAffinity
  87. }
  88. return th.NUMANodeAffinity.IsEqual(topologyHint.NUMANodeAffinity)
  89. }
  90. return false
  91. }
  92. // LessThan checks if TopologyHint `a` is less than TopologyHint `b`
  93. // this means that either `a` is a preferred hint and `b` is not
  94. // or `a` NUMANodeAffinity attribute is narrower than `b` NUMANodeAffinity attribute.
  95. func (th *TopologyHint) LessThan(other TopologyHint) bool {
  96. if th.Preferred != other.Preferred {
  97. return th.Preferred == true
  98. }
  99. return th.NUMANodeAffinity.IsNarrowerThan(other.NUMANodeAffinity)
  100. }
  101. var _ Manager = &manager{}
  102. //NewManager creates a new TopologyManager based on provided policy
  103. func NewManager(numaNodeInfo cputopology.NUMANodeInfo, topologyPolicyName string) (Manager, error) {
  104. klog.Infof("[topologymanager] Creating topology manager with %s policy", topologyPolicyName)
  105. var numaNodes []int
  106. for node := range numaNodeInfo {
  107. numaNodes = append(numaNodes, node)
  108. }
  109. if topologyPolicyName != PolicyNone && len(numaNodes) > maxAllowableNUMANodes {
  110. return nil, fmt.Errorf("unsupported on machines with more than %v NUMA Nodes", maxAllowableNUMANodes)
  111. }
  112. var policy Policy
  113. switch topologyPolicyName {
  114. case PolicyNone:
  115. policy = NewNonePolicy()
  116. case PolicyBestEffort:
  117. policy = NewBestEffortPolicy(numaNodes)
  118. case PolicyRestricted:
  119. policy = NewRestrictedPolicy(numaNodes)
  120. case PolicySingleNumaNode:
  121. policy = NewSingleNumaNodePolicy(numaNodes)
  122. default:
  123. return nil, fmt.Errorf("unknown policy: \"%s\"", topologyPolicyName)
  124. }
  125. var hp []HintProvider
  126. pth := make(map[string]map[string]TopologyHint)
  127. pm := make(map[string]string)
  128. manager := &manager{
  129. hintProviders: hp,
  130. podTopologyHints: pth,
  131. podMap: pm,
  132. policy: policy,
  133. }
  134. return manager, nil
  135. }
  136. func (m *manager) GetAffinity(podUID string, containerName string) TopologyHint {
  137. return m.podTopologyHints[podUID][containerName]
  138. }
  139. func (m *manager) accumulateProvidersHints(pod *v1.Pod, container *v1.Container) (providersHints []map[string][]TopologyHint) {
  140. // Loop through all hint providers and save an accumulated list of the
  141. // hints returned by each hint provider.
  142. for _, provider := range m.hintProviders {
  143. // Get the TopologyHints from a provider.
  144. hints := provider.GetTopologyHints(pod, container)
  145. providersHints = append(providersHints, hints)
  146. klog.Infof("[topologymanager] TopologyHints for pod '%v', container '%v': %v", pod.Name, container.Name, hints)
  147. }
  148. return providersHints
  149. }
  150. // Collect Hints from hint providers and pass to policy to retrieve the best one.
  151. func (m *manager) calculateAffinity(pod *v1.Pod, container *v1.Container) (TopologyHint, bool) {
  152. providersHints := m.accumulateProvidersHints(pod, container)
  153. bestHint, admit := m.policy.Merge(providersHints)
  154. klog.Infof("[topologymanager] ContainerTopologyHint: %v", bestHint)
  155. return bestHint, admit
  156. }
  157. func (m *manager) AddHintProvider(h HintProvider) {
  158. m.hintProviders = append(m.hintProviders, h)
  159. }
  160. func (m *manager) AddContainer(pod *v1.Pod, containerID string) error {
  161. m.podMap[containerID] = string(pod.UID)
  162. return nil
  163. }
  164. func (m *manager) RemoveContainer(containerID string) error {
  165. klog.Infof("[topologymanager] RemoveContainer - Container ID: %v", containerID)
  166. podUIDString := m.podMap[containerID]
  167. delete(m.podMap, containerID)
  168. if _, exists := m.podTopologyHints[podUIDString]; exists {
  169. delete(m.podTopologyHints[podUIDString], containerID)
  170. if len(m.podTopologyHints[podUIDString]) == 0 {
  171. delete(m.podTopologyHints, podUIDString)
  172. }
  173. }
  174. return nil
  175. }
  176. func (m *manager) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAdmitResult {
  177. // Unconditionally admit the pod if we are running with the 'none' policy.
  178. if m.policy.Name() == PolicyNone {
  179. return lifecycle.PodAdmitResult{Admit: true}
  180. }
  181. klog.Infof("[topologymanager] Topology Admit Handler")
  182. pod := attrs.Pod
  183. hints := make(map[string]TopologyHint)
  184. for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) {
  185. result, admit := m.calculateAffinity(pod, &container)
  186. if !admit {
  187. return lifecycle.PodAdmitResult{
  188. Message: "Resources cannot be allocated with Topology locality",
  189. Reason: "TopologyAffinityError",
  190. Admit: false,
  191. }
  192. }
  193. hints[container.Name] = result
  194. }
  195. m.podTopologyHints[string(pod.UID)] = hints
  196. klog.Infof("[topologymanager] Topology Affinity for Pod: %v are %v", pod.UID, m.podTopologyHints[string(pod.UID)])
  197. return lifecycle.PodAdmitResult{Admit: true}
  198. }