设备调度 - 华为昇腾 NPU

背景

当前 Koordinator 支持昇腾卡在 K8s 的使用，基于 koord-device-daemon 和 koordlet 组件上报异构 GPU 资源，将异构卡信息汇总到 Device 中供调度器进行拓扑调度。当前同时支持昇腾的虚拟化模板和整卡。

使用方法

前置条件

昇腾卡的使用需要提前安装配置如下组件

1. args:
2. - --volcanoType=true
3. - --presetVirtualDevice=false
4. - --useAscendDocker=true
  • Koordinator 相关组件(koordinator >= v1.7.0)
  • 需要在 chart 的 scheduler.featureGates 参数中加入 DevicePluginAdaption=true 以启用该功能所需的特性门控

使用

  1. 确认昇腾卡已成功被 Device 识别， 示例如下，通过 kubectl get device <node-name> -o yaml 查看 Device 资源。

  2. 注意

  3. node.koordinator.sh/gpu-vendor 对应的标签值为 huawei
  4. scheduling.koordinator.sh/gpu-partitions 的内容表示这个节点的卡编号 0,1,2,3 为一组 partion , 彼此之间有 HCCS 的亲和性，卡 1，2，3，4 为另一组 partition ，多卡调度的时候需要遵循亲和性的要求, 不能跨越不同的partition，否则不同partition之间的卡无法通信。
1. apiVersion: scheduling.koordinator.sh/v1alpha1
2. kind: Device
3. metadata:
4. labels:
5. node.koordinator.sh/gpu-partition-policy: "Honor" # Enforce partition policy during scheduling.
6. node.koordinator.sh/gpu-model: Ascend-910B3 # Model of the Ascend card.
7. node.koordinator.sh/gpu-vendor: huawei # Vendor of the card.
8. annotations:
9. scheduling.koordinator.sh/gpu-partitions: |
10. {
11. "4": [
12. {
13. "minors": [
14. 0,1,2,3  # Minor numbers of cards in this partition
15. ],
16. "gpuLinkType": "HCCS", # Indicate the type of inter-card link within the partition.
17. "allocationScore": "1" # Indicate the overall allocation quality for the node after the partition has been assigned away.
18. }
19. ],
20. "4": [
21. {
22. "minors": [
23. 4,5,6,7
24. ],
25. "gpuLinkType": "HCCS",
26. "allocationScore": "1"
27. }
28. ]
29. }
30. name: node-1
31. spec:
32. devices:
33. - health: true
34. id: GPU-fd971b33-4891-fd2e-ed42-ce6adf324615 # The UUID of the Ascend card (mocked value).
35. minor: 0 # Every card has a unique minor number. Card 0.
36. resources:
37. huawei.com/npu-core: "20" # Number of aicore on the Ascend card.
38. huawei.com/npu-cpu: "7"  # Number of aicpu on the Ascend card.
39. huawei.com/npu-dvpp: "100" # There exists dvpp on the Ascend card.
40. koordinator.sh/gpu-memory: 64Gi # The total memory capacity of the Ascend card.
41. koordinator.sh/gpu-memory-ratio: "100" # The total memory capacity of the Ascend card in percentage.
42. topology:
43. busID: 0000:3b:00.0 # PCI bus ID.
44. nodeID: 0 # NUMA node ID.
45. pcieID: pci0000:3a # PCIe root ID.
46. socketID: -1
47. type: gpu
48. conditions:
49. - lastTransitionTime: "2025-04-25T10:00:00Z"
50. message: device is healthy
51. reason: DeviceHealthy
52. status: "True"
53. type: Healthy
54. - health: true
55. id: GPU-ab123c45-6789-d001-e234-f567890abcde
56. minor: 1 # Card 1.
57. resources:
58. huawei.com/npu-core: "20"
59. huawei.com/npu-cpu: "7"
60. huawei.com/npu-dvpp: "100"
61. koordinator.sh/gpu-memory: 64Gi
62. koordinator.sh/gpu-memory-ratio: "100"
63. topology:
64. busID: 0000:3b:00.1
65. nodeID: 1
66. pcieID: pci0000:3a
67. socketID: -1
68. type: gpu
69. conditions:
70. - lastTransitionTime: "2025-04-25T10:00:00Z"
71. message: device is healthy
72. reason: DeviceHealthy
73. status: "True"
74. type: Healthy
75. - health: true
76. id: GPU-bc234d56-7890-e112-f345-678901bcdefa
77. minor: 2
78. resources:
79. huawei.com/npu-core: "20"
80. huawei.com/npu-cpu: "7"
81. huawei.com/npu-dvpp: "100"
82. koordinator.sh/gpu-memory: 64Gi
83. koordinator.sh/gpu-memory-ratio: "100"
84. topology:
85. busID: 0000:3b:00.2
86. nodeID: 2
87. pcieID: pci0000:3a
88. socketID: -1
89. type: gpu
90. conditions:
91. - lastTransitionTime: "2025-04-25T10:00:00Z"
92. message: device is healthy
93. reason: DeviceHealthy
94. status: "True"
95. type: Healthy
96. - health: true
97. id: GPU-cd345e67-8901-f223-4567-890123cdefab
98. minor: 3
99. resources:
100. huawei.com/npu-core: "20"
101. huawei.com/npu-cpu: "7"
102. huawei.com/npu-dvpp: "100"
103. koordinator.sh/gpu-memory: 64Gi
104. koordinator.sh/gpu-memory-ratio: "100"
105. topology:
106. busID: 0000:3b:00.3
107. nodeID: 3
108. pcieID: pci0000:3a
109. socketID: -1
110. type: gpu
111. conditions:
112. - lastTransitionTime: "2025-04-25T10:00:00Z"
113. message: device is healthy
114. reason: DeviceHealthy
115. status: "True"
116. type: Healthy
117. - health: true
118. id: GPU-de456f78-9012-f334-5678-901234defabc
119. minor: 4
120. resources:
121. huawei.com/npu-core: "20"
122. huawei.com/npu-cpu: "7"
123. huawei.com/npu-dvpp: "100"
124. koordinator.sh/gpu-memory: 64Gi
125. koordinator.sh/gpu-memory-ratio: "100"
126. topology:
127. busID: 0000:3b:00.4
128. nodeID: 4
129. pcieID: pci0000:3b
130. socketID: -1
131. type: gpu
132. conditions:
133. - lastTransitionTime: "2025-04-25T10:00:00Z"
134. message: device is healthy
135. reason: DeviceHealthy
136. status: "True"
137. type: Healthy
138. - health: true
139. id: GPU-ef567g89-0123-f445-6789-012345efabcd
140. minor: 5
141. resources:
142. huawei.com/npu-core: "20"
143. huawei.com/npu-cpu: "7"
144. huawei.com/npu-dvpp: "100"
145. koordinator.sh/gpu-memory: 64Gi
146. koordinator.sh/gpu-memory-ratio: "100"
147. topology:
148. busID: 0000:3b:00.5
149. nodeID: 5
150. pcieID: pci0000:3b
151. socketID: -1
152. type: gpu
153. conditions:
154. - lastTransitionTime: "2025-04-25T10:00:00Z"
155. message: device is healthy
156. reason: DeviceHealthy
157. status: "True"
158. type: Healthy
159. - health: true
160. id: GPU-fg678h90-1234-f556-7890-123456fgabce
161. minor: 6
162. resources:
163. huawei.com/npu-core: "20"
164. huawei.com/npu-cpu: "7"
165. huawei.com/npu-dvpp: "100"
166. koordinator.sh/gpu-memory: 64Gi
167. koordinator.sh/gpu-memory-ratio: "100"
168. topology:
169. busID: 0000:3b:00.6
170. nodeID: 6
171. pcieID: pci0000:3b
172. socketID: -1
173. type: gpu
174. conditions:
175. - lastTransitionTime: "2025-04-25T10:00:00Z"
176. message: device is healthy
177. reason: DeviceHealthy
178. status: "True"
179. type: Healthy
180. - health: true
181. id: GPU-gh789i01-2345-f667-8901-234567ghabcd
182. minor: 7
183. resources:
184. huawei.com/npu-core: "20"
185. huawei.com/npu-cpu: "7"
186. huawei.com/npu-dvpp: "100"
187. koordinator.sh/gpu-memory: 64Gi
188. koordinator.sh/gpu-memory-ratio: "100"
189. topology:
190. busID: 0000:3b:00.7
191. nodeID: 7
192. pcieID: pci0000:3b
193. socketID: -1
194. type: gpu
195. conditions:
196. - lastTransitionTime: "2025-04-25T10:00:00Z"
197. message: device is healthy
198. reason: DeviceHealthy
199. status: "True"
200. type: Healthy
201. status: {}
  1. 确认昇腾的节点上已正确注册昇腾卡资源信息，示例如下，通过 kubectl get node <node-name> -o yaml 查看 node.status.allocatable/node.status.capacity 资源。
1. apiVersion: v1
2. kind: Node
3. metadata:
4. name: node-1
5. ...
6. status:
7. allocatable:
8. ...
9. koordinator.sh/gpu.shared: "800" # Reported by koordinator
10. koordinator.sh/gpu-memory: 512Gi # Reported by koordinator
11. koordinator.sh/gpu-memory-ratio: "800" # Reported by koordinator
12. huawei.com/npu-core: "160" # Reported by ascend device plugin
13. huawei.com/npu-cpu: "56" # Reported by Koordinator
14. huawei.com/npu-dvpp: "800" #  Reported Koordinator
15. capacity:
16. ...
17. koordinator.sh/gpu.shared: "800" # Reported by koordinator
18. koordinator.sh/gpu-memory: 512Gi # Reported by koordinator
19. koordinator.sh/gpu-memory-ratio: "800" # Reported by koordinator
20. huawei.com/npu-core: "160" # Reported by ascend device plugin
21. huawei.com/npu-cpu: "56" # Reported by Koordinator
22. huawei.com/npu-dvpp: "800" # Reported Koordinator

  1. 创建 Pod 申请使用昇腾卡，样例如下，示例中申请了整卡和虚拟卡场景。

  2. 注意

  3. 此处镜像使用了昇腾官方提供的推理基础镜像，用户可根据实际需求选择合适的镜像
  4. resources.limits 和 resources.requests 中均需要额外配置资源
    • 整卡场景，以 910B3 的整卡规格为例子。昇腾支持多卡整卡场景。
    • huawei.com/npu-core
      • 单卡包含 20 个 npucore，申请 1 卡则需要申请 20 个 npucore。number_to_fill = card_count * npucore_per_card
      • koordinator.sh/gpu-memory-ratio
      • 对于整卡场景， 1 卡对应 100 的 gpu-memory-ratio。number_to_fill = card_count * 100
    • 虚拟卡场景，以 910B3 的虚拟规格 virt10_3c_32g 为例（可查询昇腾卡虚拟规格说明）， 规格表示申请 10 个 npucore， 3 个 npucpu， 32GB 显存。昇腾不支持配置多卡虚拟化。
    • koordinator.sh/gpu.shared
      • 申请虚拟卡填写 1 即可。
    • koordinator.sh/gpu-memory
      • 申请 1 虚拟卡则需要申请 32GB 的 gpu-memory。number_to_fill = gpu-memory_in_spec。
    • huawei.com/npu-core
      • 申请 1 虚拟卡则需要申请 10 个 npucore。number_to_fill = npucore_in_spec。
    • huawei.com/npu-cpu
      • 申请 1 虚拟卡则需要申请 3 个 npucpu。number_to_fill = npucpu_in_spec。
    • huawei.com/npu-dvpp
      • 申请 1 虚拟卡则需要申请 0 的 dvpp。直接查规格表即可。 number_to_fill = dvpp_in_spec
1. apiVersion: v1
2. kind: Pod
3. metadata:
4. labels:
5. app: demo-sleep
6. name: test-ascend-full-1
7. namespace: default
8. spec:
9. containers:
10. - command:
11. - sleep
12. - infinity
13. image: swr.cn-south-1.myhuaweicloud.com/ascendhub/ascend-infer:24.0.0-ubuntu20.04
14. imagePullPolicy: IfNotPresent
15. name: demo-sleep
16. resources:
17. limits:
18. cpu: "32"
19. memory: 64Gi
20. koordinator.sh/gpu-memory-ratio: "100" # card_count * 100
21. huawei.com/npu-core: "20" # card_count * npucore_per_card
22. requests:
23. cpu: "32"
24. memory: 64Gi
25. koordinator.sh/gpu-memory-ratio: "100"
26. huawei.com/npu-core: "20"

1. apiVersion: v1
2. kind: Pod
3. metadata:
4. labels:
5. app: demo-sleep
6. name: test-ascend-partial-1
7. namespace: default
8. spec:
9. containers:
10. - command:
11. - sleep
12. - infinity
13. image: swr.cn-south-1.myhuaweicloud.com/ascendhub/ascend-infer:24.0.0-ubuntu20.04
14. imagePullPolicy: IfNotPresent
15. name: demo-sleep
16. resources:
17. limits:
18. cpu: "32"
19. memory: 64Gi
20. koordinator.sh/gpu.shared: "1"
21. koordinator.sh/gpu-memory: "32Gi" # gpu-memory_in_spec
22. huawei.com/npu-core: "10" # npucore_in_spec
23. huawei.com/npu-cpu: "3" # npucpu_in_spec
24. requests:
25. cpu: "32"
26. memory: 64Gi
27. koordinator.sh/gpu.shared: "1"
28. koordinator.sh/gpu-memory: "32Gi"
29. huawei.com/npu-core: "10"
30. huawei.com/npu-cpu: "3"
  1. 进入容器内部(kubectl exec -it {pod-name} --bash)，在容器内部执行 npu-smi 命令查看卡的使用情况。如果能够正常显示数据，表示卡已经成功分配到 Pod 中。

昇腾卡虚拟规格列表

下面列出了常见昇腾卡的虚拟规格，用户可根据实际需求选择合适的虚拟规格进行申请使用。

1. huawei-Ascend-310P:
2. vir01:
3. huawei.com/npu-core: "1"
4. huawei.com/npu-cpu: "1"
5. huawei.com/npu-dvpp: "12"
6. koordinator.sh/gpu-memory: 3Gi
7. vir02:
8. huawei.com/npu-core: "2"
9. huawei.com/npu-cpu: "2"
10. huawei.com/npu-dvpp: "25"
11. koordinator.sh/gpu-memory: 6Gi
12. vir02_1c:
13. huawei.com/npu-core: "2"
14. huawei.com/npu-cpu: "1"
15. huawei.com/npu-dvpp: "25"
16. koordinator.sh/gpu-memory: 6Gi
17. vir04:
18. huawei.com/npu-core: "4"
19. huawei.com/npu-cpu: "4"
20. huawei.com/npu-dvpp: "50"
21. koordinator.sh/gpu-memory: 12Gi
22. vir04_3c:
23. huawei.com/npu-core: "4"
24. huawei.com/npu-cpu: "3"
25. huawei.com/npu-dvpp: "50"
26. koordinator.sh/gpu-memory: 12Gi
27. vir04_3c_ndvpp:
28. huawei.com/npu-core: "4"
29. huawei.com/npu-cpu: "3"
30. koordinator.sh/gpu-memory: 12Gi
31. vir04_4c_dvpp:
32. huawei.com/npu-core: "4"
33. huawei.com/npu-cpu: "4"
34. huawei.com/npu-dvpp: "100"
35. koordinator.sh/gpu-memory: 12Gi
36. huawei-Ascend-310P3:
37. vir01:
38. huawei.com/npu-core: "1"
39. huawei.com/npu-cpu: "1"
40. huawei.com/npu-dvpp: "12"
41. koordinator.sh/gpu-memory: 3Gi
42. vir02:
43. huawei.com/npu-core: "2"
44. huawei.com/npu-cpu: "2"
45. huawei.com/npu-dvpp: "25"
46. koordinator.sh/gpu-memory: 6Gi
47. vir02_1c:
48. huawei.com/npu-core: "2"
49. huawei.com/npu-cpu: "1"
50. huawei.com/npu-dvpp: "25"
51. koordinator.sh/gpu-memory: 6Gi
52. vir04:
53. huawei.com/npu-core: "4"
54. huawei.com/npu-cpu: "4"
55. huawei.com/npu-dvpp: "50"
56. koordinator.sh/gpu-memory: 12Gi
57. vir04_3c:
58. huawei.com/npu-core: "4"
59. huawei.com/npu-cpu: "3"
60. huawei.com/npu-dvpp: "50"
61. koordinator.sh/gpu-memory: 12Gi
62. vir04_3c_ndvpp:
63. huawei.com/npu-core: "4"
64. huawei.com/npu-cpu: "3"
65. koordinator.sh/gpu-memory: 12Gi
66. vir04_4c_dvpp:
67. huawei.com/npu-core: "4"
68. huawei.com/npu-cpu: "4"
69. huawei.com/npu-dvpp: "100"
70. koordinator.sh/gpu-memory: 12Gi
71. huawei-Ascend-910:
72. vir02:
73. huawei.com/npu-core: "2"
74. huawei.com/npu-cpu: "1"
75. koordinator.sh/gpu-memory: 2Gi
76. vir04:
77. huawei.com/npu-core: "4"
78. huawei.com/npu-cpu: "1"
79. koordinator.sh/gpu-memory: 4Gi
80. vir08:
81. huawei.com/npu-core: "8"
82. huawei.com/npu-cpu: "3"
83. koordinator.sh/gpu-memory: 8Gi
84. vir16:
85. huawei.com/npu-core: "16"
86. huawei.com/npu-cpu: "7"
87. koordinator.sh/gpu-memory: 16Gi
88. huawei-Ascend-910B1:
89. vir03_1c_8g:
90. huawei.com/npu-core: "3"
91. huawei.com/npu-cpu: "1"
92. koordinator.sh/gpu-memory: 8Gi
93. vir06_1c_16g:
94. huawei.com/npu-core: "6"
95. huawei.com/npu-cpu: "1"
96. koordinator.sh/gpu-memory: 16Gi
97. vir12_3c_32g:
98. huawei.com/npu-core: "12"
99. huawei.com/npu-cpu: "3"
100. koordinator.sh/gpu-memory: 32Gi
101. huawei-Ascend-910B2:
102. vir03_1c_8g:
103. huawei.com/npu-core: "3"
104. huawei.com/npu-cpu: "1"
105. koordinator.sh/gpu-memory: 8Gi
106. vir06_1c_16g:
107. huawei.com/npu-core: "6"
108. huawei.com/npu-cpu: "1"
109. koordinator.sh/gpu-memory: 16Gi
110. vir12_3c_32g:
111. huawei.com/npu-core: "12"
112. huawei.com/npu-cpu: "3"
113. koordinator.sh/gpu-memory: 32Gi
114. huawei-Ascend-910B2C:
115. vir03_1c_8g:
116. huawei.com/npu-core: "3"
117. huawei.com/npu-cpu: "1"
118. koordinator.sh/gpu-memory: 8Gi
119. vir06_1c_16g:
120. huawei.com/npu-core: "6"
121. huawei.com/npu-cpu: "1"
122. koordinator.sh/gpu-memory: 16Gi
123. vir12_3c_32g:
124. huawei.com/npu-core: "12"
125. huawei.com/npu-cpu: "3"
126. koordinator.sh/gpu-memory: 32Gi
127. huawei-Ascend-910B3:
128. vir05_1c_16g:
129. huawei.com/npu-core: "5"
130. huawei.com/npu-cpu: "1"
131. koordinator.sh/gpu-memory: 16Gi
132. vir10_3c_32g:
133. huawei.com/npu-core: "10"
134. huawei.com/npu-cpu: "3"
135. koordinator.sh/gpu-memory: 32Gi
136. huawei-Ascend-910B4:
137. vir05_1c_8g:
138. huawei.com/npu-core: "5"
139. huawei.com/npu-cpu: "1"
140. huawei.com/npu-dvpp: "25"
141. koordinator.sh/gpu-memory: 8Gi
142. vir10_3c_16g:
143. huawei.com/npu-core: "10"
144. huawei.com/npu-cpu: "3"
145. huawei.com/npu-dvpp: "50"
146. koordinator.sh/gpu-memory: 16Gi
147. vir10_3c_16g_nm:
148. huawei.com/npu-core: "10"
149. huawei.com/npu-cpu: "3"
150. koordinator.sh/gpu-memory: 16Gi
151. vir10_4c_16g_m:
152. huawei.com/npu-core: "10"
153. huawei.com/npu-cpu: "4"
154. huawei.com/npu-dvpp: "100"
155. koordinator.sh/gpu-memory: 16Gi