{"id":601,"date":"2020-04-08T11:09:11","date_gmt":"2020-04-08T03:09:11","guid":{"rendered":"https:\/\/www.myway5.com\/?p=601"},"modified":"2023-07-05T21:28:09","modified_gmt":"2023-07-05T13:28:09","slug":"gaia-scheduler-gpu-manager","status":"publish","type":"post","link":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/","title":{"rendered":"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b"},"content":{"rendered":"<h2>\u6982\u8ff0<\/h2>\n<p>\u5728\u4e4b\u524d\u7684\u4e00\u7bc7\u6587\u7ae0\u4e3b\u8981\u662f\u5206\u6790\u4e86 <a class=\"wp-editor-md-post-content-link\" href=\"https:\/\/www.myway5.com\/index.php\/2020\/04\/01\/gpu-manager-%e5%90%af%e5%8a%a8%e6%b5%81%e7%a8%8b%e5%88%86%e6%9e%90\/\">gpu-manager \u7684\u542f\u52a8\u6d41\u7a0b<\/a>\u3002\u5173\u4e8e gpu-manager \u5e94\u8be5\u4f1a\u6709\u4e00\u7cfb\u5217\u7684\u6587\u7ae0\uff0c\u4e00\u662f\u89c9\u5f97\u8fd9\u662f\u4e00\u4e2a\u5f88\u6709\u4ef7\u503c\u7684\u9879\u76ee\uff0c\u4e8c\u662f\u4e3a\u8fd9\u4e2a\u9879\u76ee\u82b1\u4e86\u597d\u51e0\u5929\u53bb\u770b\u4ee3\u7801\uff0c\u60f3\u901a\u8fc7\u5199\u6587\u7ae0\u7684\u65b9\u5f0f\u5bf9\u5185\u5bb9\u8fdb\u884c\u68b3\u7406\u3002<\/p>\n<p>\u8fd9\u7bc7\u6587\u7ae0\u4e3b\u8981\u5206\u6790 gpu-manager \u7684\u865a\u62df gpu \u5206\u914d\u539f\u7406\uff0c\u6211\u8ba4\u4e3a\u5c06\u865a\u62df gpu \u5206\u914d\u7ed9\u5bb9\u5668\u4e3b\u8981\u6709\u4e24\u4e2a\u91cd\u70b9\uff1a<\/p>\n<ul>\n<li>gpu-manager \u4f5c\u4e3a device plugin \u7684\u5de5\u4f5c\u6d41\u7a0b<\/li>\n<li>\u865a\u62df gpu \u5206\u914d\u7684\u6700\u4f18\u65b9\u6848\uff0c\u5206\u914d\u9700\u8981\u4fdd\u8bc1\u6700\u5c11\u788e\u7247\uff0c\u540c\u65f6\u6027\u80fd\u6700\u597d<\/li>\n<\/ul>\n<h2>\u4ece pod \u8c03\u5ea6\u5230\u865a\u62df gpu \u5206\u914d<\/h2>\n<p>\u8fd9\u4e00\u90e8\u5206\u4f1a\u6d89\u53ca\u5230 <code>device plugin<\/code> \u7684\u5de5\u4f5c\u673a\u5236\uff0c\u56e0\u6b64\u4e0d\u719f\u6089\u7684\u8bdd\u53ef\u4ee5\u770b\u4e00\u4e0b\u4e4b\u524d\u7684\u4e00\u7bc7\u6587\u7ae0\uff1a<a class=\"wp-editor-md-post-content-link\" href=\"https:\/\/www.myway5.com\/index.php\/2020\/03\/24\/kubernetes%e5%bc%80%e5%8f%91%e7%9f%a5%e8%af%86-device-plugin%e7%9a%84%e5%ae%9e%e7%8e%b0\/\">Kubernetes\u5f00\u53d1\u77e5\u8bc6\u2013device-plugin\u7684\u5b9e\u73b0<\/a>\u3002\u4e0b\u9762\u9644\u4e0a\u4e00\u5f20\u8fd9\u7bc7\u6587\u7ae0\u4e2d <code>device plugin<\/code> \u7684\u5de5\u4f5c\u65f6\u5e8f\u56fe\uff1a<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg\" alt=\"device plugin\" \/><\/p>\n<p>\u5728\u4e4b\u524d\u7684\u542f\u52a8\u6d41\u7a0b\u5206\u6790\u6587\u7ae0\u4e2d\uff0c\u8bf4\u5230 gpu-manager \u5411 kubelet \u6ce8\u518c\u3002\u5728\u8fd9\u4e4b\u540e\uff0c gpu-manager \u5c31\u6b63\u5f0f\u4f5c\u4e3a\u4e00\u4e2a device plugin \u6765\u5de5\u4f5c\u4e86\u3002\u8fd9\u4e2a\u65f6\u5019\uff0c\u6211\u4eec\u53ef\u4ee5\u521b\u5efa\u5982\u4e0b\u7684 pod:<\/p>\n<pre><code class=\"language-yaml line-numbers\">apiVersion: v1\nkind: Pod\nmetadata:\n  name: tf-training-example-10\n  namespace: test\n  labels:\n    name: tf-training-example\nspec:\n  restartPolicy: Never\n  containers:\n  - name: tf-training-example\n    image: joyme\/tf_training_example:1.5\n    resources:\n      requests:\n        tencent.com\/vcuda-core: 20\n        tencent.com\/vcuda-memory: 15\n      limits:\n        tencent.com\/vcuda-core: 20\n        tencent.com\/vcuda-memory: 15\n<\/code><\/pre>\n<p>\u8fd9\u4e2a\u521b\u5efa pod \u7684\u8bf7\u6c42\u4f1a\u5230\u8fbe kubernetes \u7684 API Server\uff0c\u7136\u540e\u7531 kube-scheduler \u8fdb\u884c\u8c03\u5ea6\u3002kube-scheduler \u7684\u8c03\u5ea6\u7ecf\u8fc7\u9884\u9009\u548c\u4f18\u9009\u4e24\u4e2a\u9636\u6bb5\uff0c\u786e\u5b9a\u4e86\u6700\u4f73\u7684\u76ee\u6807\u8282\u70b9\u3002\u8fd9\u65f6\u5019 kubelet \u5c31\u4e0a\u573a\u4e86\u3002\u56e0\u4e3a\u6211\u4eec\u7684 pod \u4e2d\u7684\u5bb9\u5668\u8bf7\u6c42\u4e86 <code>vcuda-core<\/code> \u548c <code>vcuda-memory<\/code> \u8fd9\u4e24\u4e2a\u8d44\u6e90\uff0c\u4f46\u662f kubelet \u5e76\u6ca1\u6709\u80fd\u529b\u53bb\u7ed9\u5bb9\u5668\u5206\u914d\u8fd9\u4e9b\u8d44\u6e90\uff0c\u4e8e\u662f\u5b83\u5c31\u627e\u662f\u8c01\u6ce8\u518c\u4e86\u8fd9\u4e9b\u8d44\u6e90\u7c7b\u578b\uff0c\u7136\u540e\u53d1\u73b0\u662f vcore \u548c vmemory \u8fd9\u4e24\u4e2a\u670d\u52a1\u6ce8\u518c\u7684\uff0c\u4e8e\u662f\u4f7f\u7528 grpc \u548c <code>\/var\/lib\/kubelet\/device-plugins\/vcore.sock<\/code> \u4ee5\u53ca <code>\/var\/lib\/kubelet\/device-plugins\/vmemory.sock<\/code> \u901a\u8fc7 unix socket \u901a\u8baf\u3002<\/p>\n<p>vcore \u548c vmemory \u662f\u4e24\u79cd\u8d44\u6e90\uff0c\u56e0\u6b64\u8fd9\u91cc\u5176\u5b9e\u76f8\u5f53\u4e8e\u6ce8\u518c\u4e86\u4e24\u4e2a device plugin\u3002<\/p>\n<p>\u5bf9\u4e8e vcuda-memory\uff0ckubelet \u8c03\u7528\u7684 <code>Allocate<\/code> \u65b9\u6cd5\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">\/** device plugin interface *\/\nfunc (vr *vmemoryResourceServer) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {\n    glog.V(2).Infof(\"%+v allocation request for vmemory\", reqs)\n    fakeData := make([]*pluginapi.ContainerAllocateResponse, 0)\n    fakeData = append(fakeData, &amp;pluginapi.ContainerAllocateResponse{})\n\n    return &amp;pluginapi.AllocateResponse{\n        ContainerResponses: fakeData,\n    }, nil\n}\n<\/code><\/pre>\n<p>\u8fd9\u91cc\u5176\u5b9e\u5e76\u6ca1\u6709\u505a\u4efb\u4f55\u5b9e\u9645\u5206\u914d\u64cd\u4f5c\uff0c\u6211\u4eec\u53ef\u4ee5\u8ba4\u4e3a vcuda-core \u548c vcuda-memory \u5fc5\u7136\u662f\u540c\u65f6\u7533\u8bf7\u5206\u914d\u7684\uff0c\u56e0\u6b64\u6211\u4eec\u53ea\u9700\u8981\u5904\u7406\u4e8c\u8005\u4e4b\u4e00\u5373\u53ef\u3002<\/p>\n<p>\u5bf9\u4e8e vcuda-core\uff0ckubelet \u4f1a\u8c03\u7528\u7684 <code>Allocate<\/code> \u65b9\u6cd5\u4ee3\u7801\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">func (vr *vcoreResourceServer) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {\n    glog.V(2).Infof(\"%+v allocation request for vcore\", reqs)\n    return vr.mgr.Allocate(ctx, reqs)\n}\n<\/code><\/pre>\n<p>\u6700\u7ec8\u4f1a\u8d70\u5230 <code>pkg\/services\/allocator\/nvidia\/allocator.go<\/code> \u7684 Allcate \u65b9\u6cd5\u4e2d\u3002\u4e0b\u9762\u5c31\u6765\u5230\u8fd9\u7bc7\u6587\u7ae0\u6700\u590d\u6742\u7684\u90e8\u5206\u4e86\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">func (ta *NvidiaTopoAllocator) Allocate(_ context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {\n}\n<\/code><\/pre>\n<p>\u6211\u4eec\u5148\u770b\u4e00\u4e0b\u51fd\u6570\u539f\u578b\uff0c<code>reqs *pluginapi.AllocateRequest<\/code> \u8fd9\u4e2a\u53c2\u6570\u662f\u5206\u914d\u8bf7\u6c42\uff0c\u7136\u540e\u8fd4\u56de\u4e86\u4e00\u4e2a\u5206\u914d\u54cd\u5e94 <code>*pluginapi.AllocateResponse<\/code>\u3002\u8fd9\u91cc\u770b\u4e00\u4e0b <code>AllocateRequest<\/code>:<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ - Allocate is expected to be called during pod creation since allocation\n\/\/   failures for any container would result in pod startup failure.\n\/\/ - Allocate allows kubelet to exposes additional artifacts in a pod's\n\/\/   environment as directed by the plugin.\n\/\/ - Allocate allows Device Plugin to run device specific operations on\n\/\/   the Devices requested\ntype AllocateRequest struct {\n    ContainerRequests []*ContainerAllocateRequest `protobuf:\"bytes,1,rep,name=container_requests,json=containerRequests\" json:\"container_requests,omitempty\"`\n}\n\ntype ContainerAllocateRequest struct {\n    DevicesIDs []string `protobuf:\"bytes,1,rep,name=devicesIDs\" json:\"devicesIDs,omitempty\"`\n}\n<\/code><\/pre>\n<p>\u5f88\u660e\u663e\uff0c\u8bf7\u6c42\u91cc\u5305\u542b\u4e86\u6bcf\u4e2a\u5bb9\u5668\u9700\u8981\u7684\u8bbe\u5907\u6570\u7ec4\u3002\u540c\u65f6\u901a\u8fc7 <code>AllocateRequest<\/code> \u4e0a\u7684\u6ce8\u91ca\u53ef\u4ee5\u5f97\u51fa\u4ee5\u4e0b\u4fe1\u606f\uff1a<\/p>\n<ul>\n<li>Allocate \u662f\u5728 pod \u521b\u5efa\u65f6\u88ab\u8c03\u7528\u7684\uff0c\u56e0\u6b64\u4efb\u4f55\u5bb9\u5668\u5206\u914d\u5931\u8d25\u90fd\u4f1a\u9020\u6210pod\u542f\u52a8\u5931\u8d25\u3002<\/li>\n<li>Allocate \u5141\u8bb8 kubelet \u5728 pod \u73af\u5883\u4e2d\u5f15\u5165\u66f4\u591a\u7684 artifacts\uff0c\u8fd9\u90e8\u5206\u5de5\u4f5c\u7531\u6211\u4eec\u7684 device plugin \u4e3b\u5bfc\u3002\u5bf9\u4e8e gpu manager \u6765\u8bf4\u5c31\u662f\uff0c\u8986\u76d6\u5bb9\u5668\u5185\u7684 LD_LIBRARY_PATH\uff0c\u6302\u8f7d cuda \u5e93\u6587\u4ef6\u7b49\u7b49\u3002<\/li>\n<li>Allocate \u5141\u8bb8 device plugin \u5728\u8bbe\u5907\u4e0a\u8fd0\u884c\u7279\u5b9a\u7684\u64cd\u4f5c\u3002<\/li>\n<\/ul>\n<p>\u7136\u540e\u518d\u6765\u770b\u4e00\u4e0b <code>AllocateResponse<\/code>:<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ AllocateResponse includes the artifacts that needs to be injected into\n\/\/ a container for accessing 'deviceIDs' that were mentioned as part of\n\/\/ 'AllocateRequest'.\n\/\/ Failure Handling:\n\/\/ if Kubelet sends an allocation request for dev1 and dev2.\n\/\/ Allocation on dev1 succeeds but allocation on dev2 fails.\n\/\/ The Device plugin should send a ListAndWatch update and fail the\n\/\/ Allocation request\ntype AllocateResponse struct {\n    ContainerResponses []*ContainerAllocateResponse `protobuf:\"bytes,1,rep,name=container_responses,json=containerResponses\" json:\"container_responses,omitempty\"`\n}\n\ntype ContainerAllocateResponse struct {\n    \/\/ List of environment variable to be set in the container to access one of more devices.\n    Envs map[string]string `protobuf:\"bytes,1,rep,name=envs\" json:\"envs,omitempty\" protobuf_key:\"bytes,1,opt,name=key,proto3\" protobuf_val:\"bytes,2,opt,name=value,proto3\"`\n    \/\/ Mounts for the container.\n    Mounts []*Mount `protobuf:\"bytes,2,rep,name=mounts\" json:\"mounts,omitempty\"`\n    \/\/ Devices for the container.\n    Devices []*DeviceSpec `protobuf:\"bytes,3,rep,name=devices\" json:\"devices,omitempty\"`\n    \/\/ Container annotations to pass to the container runtime\n    Annotations map[string]string `protobuf:\"bytes,4,rep,name=annotations\" json:\"annotations,omitempty\" protobuf_key:\"bytes,1,opt,name=key,proto3\" protobuf_val:\"bytes,2,opt,name=value,proto3\"`\n}\n<\/code><\/pre>\n<p>\u8fd9\u91cc\u6211\u4eec\u53c8\u53ef\u4ee5\u770b\u5230\u4e00\u4e9b\u5173\u952e\u4fe1\u606f\uff0c<code>AllocateResponse<\/code> \u4e3a\u6bcf\u4e2a\u5bb9\u5668\u8fd4\u56de\u4e86 <code>ContainerAllocateResponse<\/code>\uff0c\u5305\u62ec\u5bb9\u5668\u7684\u73af\u5883\u53d8\u91cf\uff0c\u5bb9\u5668\u7684\u6302\u8f7d\uff0c\u5bb9\u5668\u7684\u8bbe\u5907\u4fe1\u606f\uff0c\u5bb9\u5668\u7684 annotations \u4fe1\u606f\u3002\u5176\u4e2d\uff0c\u5bb9\u5668\u7684\u8bbe\u5907\u4fe1\u606f\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ DeviceSpec specifies a host device to mount into a container.\ntype DeviceSpec struct {\n    \/\/ Path of the device within the container.\n    ContainerPath string `protobuf:\"bytes,1,opt,name=container_path,json=containerPath,proto3\" json:\"container_path,omitempty\"`\n    \/\/ Path of the device on the host.\n    HostPath string `protobuf:\"bytes,2,opt,name=host_path,json=hostPath,proto3\" json:\"host_path,omitempty\"`\n    \/\/ Cgroups permissions of the device, candidates are one or more of\n    \/\/ * r - allows container to read from the specified device.\n    \/\/ * w - allows container to write to the specified device.\n    \/\/ * m - allows container to create device files that do not yet exist.\n    Permissions string `protobuf:\"bytes,3,opt,name=permissions,proto3\" json:\"permissions,omitempty\"`\n}\n<\/code><\/pre>\n<p>\u5373\u5728\u5bb9\u5668\u4e2d\u6302\u8f7d\u8bbe\u5907\u9700\u8981\uff1a<\/p>\n<ul>\n<li>\u8bbe\u5907\u76f8\u5bf9\u4e8e\u5bb9\u5668\u7684\u5730\u5740<\/li>\n<li>\u8bbe\u5907\u5728\u5bbf\u4e3b\u673a\u4e0a\u7684\u5730\u5740<\/li>\n<li>\u8bbe\u5907\u7684 Cgroups \u4fe1\u606f<\/li>\n<\/ul>\n<p>\u8fd9\u65f6\u5019\u6211\u4eec\u518d\u6765\u91cd\u65b0\u601d\u8003 gpu-manager \u7684 gpu \u865a\u62df\u5316\u539f\u7406\u3002\u5982\u679c\u4f60\u770b\u8fc7\u817e\u8baf\u5173\u4e8e Gaia Scheduler \u7684\u8bba\u6587\uff0c\u5c31\u4f1a\u77e5\u9053 gpu-manager \u9700\u8981\u505a\u4ee5\u4e0b\u5de5\u4f5c\uff1a<\/p>\n<ul>\n<li>\u4e3a\u5bb9\u5668\u6302\u8f7d cuda \u76f8\u5173\u7684\u5e93\uff0c\u5305\u62ec vcuda-control \u8fd9\u4e2a\u9879\u76ee\u7684\u62e6\u622a\u5e93<\/li>\n<li>\u901a\u8fc7\u8986\u76d6\u5bb9\u5668\u4e2d\u7684 LD_LIBRARY_PATH \u6765\u5c06 cuda \u8c03\u7528\u6307\u5411 libcuda-control.so \u8fd9\u4e2a\u5e93\uff0c\u8fd9\u4e2a\u5e93\u91cc\u9762\u5bf9\u663e\u5b58\u548c\u8ba1\u7b97 api \u505a\u4e86\u62e6\u622a\u3002<\/li>\n<li>\u4e3a\u5bb9\u5668\u6302\u8f7d vcuda.sock\uff0c\u5728\u5bb9\u5668\u8c03\u7528\u7279\u5b9a\u7684 cuda api \u65f6\uff0c\u4f1a\u89e6\u53d1 grpc \u8c03\u7528\uff0c\u901a\u8fc7 vcuda.sock \u548c virtual manager \u901a\u4fe1\uff0cvirtual manager \u4e0b\u53d1\u5bb9\u5668\u914d\u7f6e\u3002\u8fd9\u6837\u62e6\u622a\u5e93\u5c31\u77e5\u9053\u81ea\u5df1\u5e94\u8be5\u600e\u4e48\u9650\u5236\u5bb9\u5668\u4e86\u3002\u8fd9\u91cc\u7559\u4e00\u4e2a\u95ee\u9898 A\uff0c\u4e3a\u4ec0\u4e48\u8981\u5927\u8d39\u5468\u7ae0\u7684\u901a\u8fc7 grpc\uff0c\u76f4\u63a5\u6302\u8f7d\u5bb9\u5668\u914d\u7f6e\u6587\u4ef6\u53ef\u884c\u5417\uff1f<\/li>\n<\/ul>\n<p>\u8fd9\u4e9b gpu-manager \u8981\u505a\u7684\u5de5\u4f5c\u90fd\u662f device plugin \u7684 Allocate \u8c03\u7528\u63d0\u4f9b\u7684\u80fd\u529b\u3002\u6240\u4ee5 gpu-manager \u9700\u8981\u5728 Allocate \u671f\u95f4\u5b8c\u6210\u8fd9\u4e48\u591a\u7684\u5de5\u4f5c\u3002\u8fd9\u4e5f\u662f\u8fd9\u90e8\u5206\u6bd4\u8f83\u590d\u6742\u7684\u539f\u56e0\u3002\u4e0b\u9762\u6211\u4eec\u5e26\u7740\u8fd9\u4e9b\u4fe1\u606f\u53bb\u770b\u4ee3\u7801\uff0c\u4f1a\u66f4\u5bb9\u6613\u61c2\u4e00\u4e9b\u3002\u4e0b\u9762\u7684\u4ee3\u7801\u90fd\u662f\u6765\u81ea\u4e8e <code>pkg\/services\/allocator\/nvidia\/allocator.go<\/code> \u4e2d\u7684 Allocate \u65b9\u6cd5\uff0c\u4f46\u662f\u56e0\u4e3a\u5f88\u957f\uff0c\u6240\u4ee5\u6211\u4f1a\u622a\u53d6\u51fa\u6765\u5206\u6790\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ k8s send allocate request for one container at a time\nreq := reqs.ContainerRequests[0]\nresps := &amp;pluginapi.AllocateResponse{}\nreqCount = uint(len(req.DevicesIDs))\n<\/code><\/pre>\n<p>\u8fd9\u90e8\u5206\u53d6\u4e86 Allocate \u4e2d\u7684\u7b2c\u4e00\u4e2a ContainerRequest\uff0c\u901a\u8fc7\u6ce8\u91ca\u77e5\u9053\uff0ck8s \u4e00\u6b21\u53ea\u4e3a\u4e00\u4e2a\u5bb9\u5668\u53d1\u9001\u5206\u914d\u8bf7\u6c42\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">    if ta.unfinishedPod != nil {\n\n    } else {\n\n    }\n<\/code><\/pre>\n<p>\u63a5\u4e0b\u6765\u6709\u4e00\u4e2a\u5bf9 <code>unfinishedPod<\/code> \u7684\u5224\u65ad\uff0c\u56e0\u4e3a k8s \u4e00\u6b21\u8bf7\u6c42\u53ea\u9488\u5bf9\u4e00\u4e2a\u5bb9\u5668\uff0c\u56e0\u6b64\u8fd9\u91cc\u7684 <code>unfinishedPod<\/code> \u6307\u7684\u662f\u53ea\u5206\u914d\u4e86\u90e8\u5206\u5bb9\u5668\uff0c\u8fd8\u6709\u5176\u4ed6\u5bb9\u5668\u6ca1\u6709\u5206\u914d\u7684 pod\u3002\u8fd9\u91cc\u6211\u4eec\u9700\u8981\u4ed4\u7ec6\u601d\u8003\u4e00\u4e0b\uff0c\u4f7f\u7528 <code>unfinishedPod<\/code> \u7684\u76ee\u7684\u662f\u4ec0\u4e48\uff1f\u770b\u5230\u8fd9\u91cc\u6211\u6709\u4e24\u4e2a\u731c\u6d4b\uff1a<\/p>\n<ol>\n<li>\u56e0\u4e3a k8s \u4e00\u6b21\u8bf7\u6c42\u53ea\u9488\u5bf9\u4e00\u4e2a\u5bb9\u5668\uff0c\u6240\u4ee5\u4e3a\u4e86\u4f18\u5148\u5206\u914d\u5b8c\u4e00\u4e2a pod\uff0c\u5c31\u9700\u8981\u6807\u8bb0 <code>unfinishedPod<\/code> \u4e86\u3002\u4f46\u662f\u4ed4\u7ec6\u60f3\u60f3\uff0c\u56e0\u4e3a Allocate \u7684\u8bf7\u6c42\u548c\u54cd\u5e94\u4e2d\u90fd\u6ca1\u6709\u5bb9\u5668\u7684\u4fe1\u606f\uff0c\u6240\u4ee5\u672c\u6b21\u8bf7\u6c42\u5206\u914d\u7684\u5bb9\u5668\u662f\u7531 kubelet \u51b3\u5b9a\u7684\u3002device plugin \u5e76\u6ca1\u6709\u80fd\u529b\u6539\u53d8\u5bb9\u5668\u7684\u5206\u914d\u987a\u5e8f\uff0c\u8fd9\u4e2a\u60f3\u6cd5\u662f\u9519\u7684\u3002<\/li>\n<li>\u4e3a\u4e86\u6027\u80fd\u8003\u8651\u3002\u56e0\u4e3a gpu-manager \u6709\u4e24\u4e2a device plugin\uff1avmemory \u548c vcore\u3002\u4f46\u662f\u4e4b\u524d\u8bf4\u5230 vmemory \u7684\u5206\u914d\u6ca1\u6709\u505a\u4efb\u4f55\u5de5\u4f5c\u3002\u6240\u4ee5\u6211\u4eec\u4e0d\u5f97\u4e0d\u5728\u5206\u914d vcore \u7684\u65f6\u5019\uff0c\u628a vmemory \u7684\u5206\u914d\u5de5\u4f5c\u4e5f\u505a\u4e86\u3002\u53ef\u662f\u6211\u600e\u4e48\u77e5\u9053\u5f53\u524d\u6b63\u5728\u7ed9\u54ea\u4e2a\u5bb9\u5668\u5206\u914d\u8d44\u6e90\uff1f\u90a3\u6211\u4e5f\u66f4\u4e0d\u77e5\u9053\u5206\u914d\u591a\u5c11 vmemory \u4e86\u3002\u4f46\u662f\u5929\u65e0\u7edd\u4eba\u4e4b\u8def\u554a\uff0c\u6211\u53ef\u4ee5\u904d\u5386\u5f53\u524d\u8282\u70b9\u4e0a\u7684\u6240\u6709 pod\uff0c\u7136\u540e\u6311\u51fa\u9700\u8981 gpu \u8d44\u6e90\u7684 pod\u3002\u7136\u540e\u518d\u4ece\u8fd9\u4e9b pod \u4e2d\u6311\u51fa\u7b26\u5408\u8fd9\u6b21\u8bf7\u6c42\u7684\u5bb9\u5668\u3002\u8fd9\u91cc\u5982\u679c\u4f7f\u7528 <code>unfinishedPod<\/code> \u5c31\u907f\u514d\u4e86\u91cd\u590d\u7684\u5927\u89c4\u6a21\u67e5\u627e\u64cd\u4f5c\u3002<\/li>\n<\/ol>\n<p>\u90a3\u4e48\uff0c\u5047\u8bbe\u73b0\u5728\u8fd8\u6709\u4e00\u4e2a\u672a\u5b8c\u6210\u7684 pod\uff0c\u4f1a\u6267\u884c\u4e0b\u9762\u7684\u4ee3\u7801<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ \u5019\u9009pod\ncandidatePod = ta.unfinishedPod\n\/\/ \u4ece\u5df2\u5206\u914d\u7684pod\u4e2d\u67e5\u627e\ncache := ta.allocatedPod.GetCache(string(candidatePod.UID))\nif cache == nil {\n    msg := fmt.Sprintf(\"failed to find pod %s in cache\", candidatePod.UID)\n    glog.Infof(msg)\n    return nil, fmt.Errorf(msg)\n}\nfor i, c := range candidatePod.Spec.Containers {\n    if _, ok := cache[c.Name]; ok {\n        continue\n    }\n\n    if !utils.IsGPURequiredContainer(&amp;c) {\n        continue\n    }\n\n    if reqCount != utils.GetGPUResourceOfContainer(&amp;candidatePod.Spec.Containers[i], types.VCoreAnnotation) {\n        msg := fmt.Sprintf(\"allocation request mismatch for pod %s, reqs %v\", candidatePod.UID, reqs)\n        glog.Infof(msg)\n        return nil, fmt.Errorf(msg)\n    }\n    \/\/ \u5019\u9009\u7684\u5bb9\u5668\uff08\u5e94\u8be5\u5c31\u662f\u5f85\u5206\u914d\u8d44\u6e90\u7684\u5bb9\u5668\uff09\n    candidateContainer = &amp;candidatePod.Spec.Containers[i]\n    found = true\n    break\n}\n<\/code><\/pre>\n<p>\u4e0a\u9762\u8fd9\u6bb5\u4ee3\u7801\u904d\u5386\u8fd9\u4e2a pod \u7684\u5bb9\u5668\u5217\u8868\uff0c\u7136\u540e\u548c\u7f13\u5b58\u4e2d\u7684\u5bb9\u5668\u5bf9\u6bd4\uff0c\u5982\u679c\u6ca1\u6709\u5206\u914d\u5e76\u4e14\u9700\u8981 gpu \u8d44\u6e90\uff0c\u5e76\u4e14\u5bb9\u5668\u8bf7\u6c42\u7684\u8d44\u6e90\u91cf\u548c\u5f53\u524d\u7684\u5206\u914d\u8bf7\u6c42\u4e00\u81f4\uff0c\u5c31\u8ba4\u5b9a\u8fd9\u4e2a\u5bb9\u5668\u662f\u6211\u4eec\u63a5\u4e0b\u6765\u8981\u4e3a\u4e4b\u5206\u914d\u7684\u5019\u9009\u4eba\u4e86\u3002\u8fd9\u91cc\u6211\u4eec\u53c8\u6709\u4e00\u4e2a\u95ee\u9898 B\uff0c\u5982\u679c\u4e00\u4e2a Pod \u4e2d\u6709\u591a\u4e2a vcore \u8bf7\u6c42\u4e00\u81f4\uff0c\u4f46\u662f vmemory \u4e0d\u540c\u7684\u5bb9\u5668\uff0c\u8fd9\u91cc\u53ea\u901a\u8fc7 vcore \u7684\u8bf7\u6c42\u91cf\u6765\u5224\u65ad\uff0c\u53ef\u4ee5\u4fdd\u8bc1\u8fd9\u4e2a\u5206\u914d\u8bf7\u6c42\u548c\u6211\u4eec\u7684\u5019\u9009\u5bb9\u5668\u80fd\u5bf9\u7684\u4e0a\u5417\uff1f\u8fd9\u4e2a\u95ee\u9898\u6211\u4eec\u53ef\u4ee5\u4ea7\u751f\u5982\u4e0b\u7684\u731c\u6d4b\uff1a<\/p>\n<ol>\n<li>AllocateRequest \u662f\u6309\u7167 Pod \u4e2d\u7684\u5bb9\u5668\u987a\u5e8f\u6765\u7684\uff0c\u8fd9\u6837\u6211\u4eec\u5728\u505a reqCount \u5bf9\u6bd4\u7684\u65f6\u5019\uff0c\u56e0\u4e3a\u987a\u5e8f\u4e00\u81f4\u5c31\u80fd\u4fdd\u8bc1\u8bf7\u6c42\u548c\u5019\u9009\u5bb9\u5668\u662f\u5bf9\u5e94\u5173\u7cfb\u4e86\u3002\u90a3\u4e48\uff0cAllocateRequest \u662f\u6309\u7167 Pod \u4e2d\u5bb9\u5668\u987a\u5e8f\u6765\u7684\u5417\uff1f\u8fd9\u662f\u4e00\u4e2a\u65b0\u7684\u95ee\u9898 C\u3002<\/li>\n<li>\u5176\u5b9e\u8bf7\u6c42\u548c\u5019\u9009\u5bb9\u5668\u4e0d\u5bf9\u5e94\u4e5f\u6ca1\u5173\u7cfb\uff0c\u56e0\u4e3a\u5bb9\u5668\u4e2d\u8fdb\u884c cuda \u8c03\u7528\u62e6\u622a\u7684\u65f6\u5019\uff0c\u624d\u4f1a\u8bf7\u6c42 virtual manager\uff0c\u62ff\u5230\u5bb9\u5668\u7684\u8d44\u6e90\u9650\u5236\u914d\u7f6e\u4fe1\u606f\u3002\u53ea\u8981\u8fd9\u4e2a\u73af\u8282\u80fd\u4fdd\u8bc1\u5bb9\u5668\u548c\u5176\u8bf7\u6c42\u7684\u8d44\u6e90\u91cf\u5bf9\u5e94\u4e0a\uff0c\u5c31\u4e0d\u4f1a\u6709\u4efb\u4f55\u95ee\u9898\uff1f\u8fd9\u4e5f\u662f\u6211\u4eec\u7684\u95ee\u9898 E\uff1acuda \u8c03\u7528\u62e6\u622a\u7684\u65f6\u5019\uff0c\u5982\u4f55\u4fdd\u8bc1\u5bb9\u5668\u548c\u914d\u7f6e\u7684\u5bf9\u5e94\u5173\u7cfb\u3002\u8fd9\u4e5f\u548c\u95ee\u9898 A \u76f8\u547c\u5e94\uff0c\u5982\u679c\u8fd9\u4e2a\u731c\u6d4b\u6210\u7acb\uff0c\u90a3\u5c31\u662f\u4e3a\u4ec0\u4e48\u95ee\u9898 A \u4e2d\u8981\u5927\u8d39\u5468\u6298\u7684\u4f7f\u7528 grpc \u8c03\u7528\u4e0b\u53d1\u914d\u7f6e\uff0c\u800c\u4e0d\u662f\u76f4\u63a5\u628a\u914d\u7f6e\u4fe1\u606f\u6302\u8f7d\u6216\u5199\u5230\u5bb9\u5668\u7684\u53d8\u91cf\u4e2d\u3002<\/li>\n<\/ol>\n<p>\u63a5\u4e0b\u6765\u6211\u4eec\u7ee7\u7eed\u770b\uff0c\u5982\u679c\u6ca1\u6709\u672a\u5b8c\u6210\u7684\u5bb9\u5668\uff0c\u5c31\u6267\u884c\u4ee5\u4e0b\u4ee3\u7801\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ \u83b7\u53d6\u5019\u9009\u7684pod,\u5019\u9009\u7684pod\u662f\u5f53\u524d\u8282\u70b9\u4e0a\u7684\u9700\u8981GPU,\u6ca1\u6709\u5206\u914d\u5e76\u4e14\u4e0d\u5e94\u8be5\u5220\u9664\u7684pod\npods, err := getCandidatePods(ta.k8sClient, ta.config.Hostname)\nif err != nil {\n    msg := fmt.Sprintf(\"Failed to find candidate pods due to %v\", err)\n    glog.Infof(msg)\n    return nil, fmt.Errorf(msg)\n}\n\nfor _, pod := range pods {\n    if found {\n        break\n    }\n    for i, c := range pod.Spec.Containers {\n        if !utils.IsGPURequiredContainer(&amp;c) {\n            continue\n        }\n        podCache := ta.allocatedPod.GetCache(string(pod.UID))\n        if podCache != nil {\n            if _, ok := podCache[c.Name]; ok {\n                glog.Infof(\"container %s of pod %s has been allocate, continue to next\", c.Name, pod.UID)\n                continue\n            }\n        }\n        if utils.GetGPUResourceOfContainer(&amp;pod.Spec.Containers[i], types.VCoreAnnotation) == reqCount {\n            glog.Infof(\"Found candidate Pod %s(%s) with device count %d\", pod.UID, c.Name, reqCount)\n            candidatePod = pod\n            candidateContainer = &amp;pod.Spec.Containers[i]\n            found = true\n            break\n        }\n    }\n}\n<\/code><\/pre>\n<p>\u548c\u4e0a\u9762\u7684\u4e0d\u540c\u4e4b\u5904\uff0c\u5c31\u662f\u5728\u83b7\u53d6\u5019\u9009 pod \u8fd9\u91cc\u3002\u83b7\u53d6\u5019\u9009 pod \u7684\u4ee3\u7801\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">    candidatePods := []*v1.Pod{}\n    allPods, err := getPodsOnNode(client, hostname, string(v1.PodPending))\n\n    for _, pod := range allPods {\n        current := pod\n        if utils.IsGPURequiredPod(\u00a4t) &amp;&amp; !utils.IsGPUAssignedPod(\u00a4t) &amp;&amp; !utils.ShouldDelete(\u00a4t) {\n            candidatePods = append(candidatePods, \u00a4t)\n        }\n    }\n\n    return OrderPodsdByPredicateTime(candidatePods), nil\n<\/code><\/pre>\n<p>\u5148\u662f\u83b7\u53d6\u8282\u70b9\u4e0a\u7684\u6240\u6709 pod\uff0c\u7136\u540e\u4ece\u8282\u70b9\u4e0a\u7684 pod \u4e2d\u9009\u53d6\u9700\u8981 GPU\uff0c\u5e76\u4e14\u6ca1\u6709\u5206\u914d GPU\uff0c\u5e76\u4e14\u4e0d\u5e94\u8be5\u5220\u9664\u7684 pod\u3002\u6700\u540e\u5f97\u5230\u4e00\u4e2a\u5019\u9009 pod \u5217\u8868\u3002\u6700\u540e\u5bf9\u8fd9\u4e2a\u5217\u8868\u6839\u636e\u65f6\u95f4\u6392\u5e8f\u3002\u8fd9\u6837\u5c31\u53ef\u4ee5\u62ff\u5230\u6700\u5148\u88ab\u8c03\u5ea6\u7684 pod \u4e86\u3002\u8fd9\u91cc\u5176\u5b9e\u4e5f\u9ed8\u8ba4\u4e86\u4e00\u4e2a\u524d\u63d0\uff0c\u6700\u5148\u8c03\u5ea6\u7684 pod \u4f1a\u6700\u5148\u53d1\u51fa\u5206\u914d\u8bf7\u6c42\u3002\u8fd9\u91cc\u8fd8\u6709\u4e00\u4e2a\u9700\u8981\u6ce8\u610f\u7684\u5730\u65b9\uff0c\u6392\u5e8f\u4f9d\u636e\u7684\u65f6\u95f4\u6709\u4e24\u4e2a\u9009\u62e9\uff1a\u9884\u9009\u65f6\u95f4\u6216\u521b\u5efa\u65f6\u95f4\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">    if predicateTimeStr, ok := pod.ObjectMeta.Annotations[types.PredicateTimeAnnotation]; ok {\n        u64, err := strconv.ParseUint(predicateTimeStr, 10, 64)\n        if err != nil {\n            glog.Warningf(\"Failed to parse predicate Timestamp %s due to %v\", predicateTimeStr, err)\n        } else {\n            predicateTime = u64\n        }\n    } else {\n        \/\/ If predicate time not found, use createionTimestamp instead\n        predicateTime = uint64(pod.ObjectMeta.CreationTimestamp.UnixNano())\n    }\n\n    return predicateTime\n<\/code><\/pre>\n<p>\u5176\u4e2d\uff0c\u9884\u9009\u65f6\u95f4\u5e76\u4e0d\u662f kube-scheduler \u6dfb\u52a0\u7684\uff0c\u800c\u662f\u548c gpu-manager \u914d\u5408\u4f7f\u7528\u7684 gpu-admission \u8fd9\u4e2a\u9879\u76ee\u3002\u5982\u679c\u6ca1\u6709\u9884\u9009\u65f6\u95f4\uff0c\u5c31\u4f1a\u4f7f\u7528 pod \u7684\u521b\u5efa\u65f6\u95f4\u3002\u8fd9\u4e5f\u5c31\u662f\u8bf4\uff0c\u6211\u4eec\u4e0d\u4f7f\u7528 gpu-admission \u8fd9\u4e2a\u9879\u76ee\uff0c\u4e5f\u53ef\u4ee5\u6b63\u5e38\u4f7f\u7528 gpu-manager\u3002\u5176\u5b9e\u8fd9\u91cc\u8fd8\u6709\u4e00\u4e2a\u95ee\u9898 D\uff0c\u6211\u600e\u4e48\u4fdd\u8bc1\u6311\u51fa\u6765\u7684\u5bb9\u5668\u5c31\u662f\u8fd9\u6b21\u5206\u914d\u8bf7\u6c42\u7684\u5462\uff1f\u8fd9\u4e2a\u95ee\u9898\u8fd8\u8981\u7559\u5728\u540e\u9762\u7684\u5206\u6790\u4e2d\u3002<\/p>\n<p>\u73b0\u5728\u6211\u4eec\u62ff\u5230\u4e86\u5019\u9009\u5bb9\u5668\uff0c\u5c31\u9700\u8981\u8fdb\u884c\u771f\u6b63\u7684\u5206\u914d\u5de5\u4f5c\u4e86\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ get vmemory info from container spec\nvmemory := utils.GetGPUResourceOfContainer(candidateContainer, types.VMemoryAnnotation)\nfor i := 0; i &lt; int(vmemory); i++ {\n    req.DevicesIDs = append(req.DevicesIDs, types.VMemoryAnnotation)\n}\n\nresp, err := ta.allocateOne(candidatePod, candidateContainer, req)\nif err != nil {\n    glog.Errorf(err.Error())\n    return nil, err\n}\nresps.ContainerResponses = append(resps.ContainerResponses, resp)\n<\/code><\/pre>\n<p>\u8fd9\u6bb5\u4ee3\u7801\u4e2d\uff0c\u6211\u4eec\u62ff\u5230\u5bb9\u5668\u7684 vmemory \u4fe1\u606f\u3002\u56e0\u4e3a vmemory \u662f\u6839\u636e\u6570\u91cf\u5212\u5206\u7684\u30021 \u4e2a vmemory \u76f8\u5f53\u4e8e 256M \u7684 memory\uff0c\u4e5f\u5c31\u662f\u4e00\u4e2a deviceID\u3002\u8fd9\u91cc\u8bf7\u6c42\u591a\u5c11\u7684 vmemory\uff0c\u5c31\u5b58\u591a\u5c11\u4e2a deviceID\u3002\u7136\u540e\u8c03\u7528 <code>allocateOne<\/code> \u4e3a\u5355\u4e2a\u5bb9\u5668\u8fdb\u884c\u771f\u6b63\u7684\u5206\u914d\u5de5\u4f5c\u3002\u4e0b\u9762\u6211\u4eec\u5f00\u59cb\u5206\u6790 <code>allocateOne<\/code> \u7684\u5206\u914d\u903b\u8f91\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">var (\n    nodes                       []*nvtree.NvidiaNode\n    needCores, needMemoryBlocks int64\n    predicateMissed             bool\n    allocated                   bool\n)\n\n\/\/ \u662f\u5426\u662f gpu \u9884\u9009 pod\npredicateMissed = !utils.IsGPUPredicatedPod(pod)\n\/\/ \u5355\u8282\u70b9\u7684\u603b\u5185\u5b58\nsingleNodeMemory := int64(ta.tree.Leaves()[0].Meta.TotalMemory)\nfor _, v := range req.DevicesIDs {\n    if strings.HasPrefix(v, types.VCoreAnnotation) {\n        \/\/ \u8bf7\u6c42 core\n        needCores++\n    } else if strings.HasPrefix(v, types.VMemoryAnnotation) {\n        \/\/ \u8bf7\u6c42 memory\n        needMemoryBlocks++\n    }\n}\n<\/code><\/pre>\n<p>\u9996\u5148\u5c31\u662f\u6839\u636e deviceID \u6765\u8ba1\u7b97\u9700\u8981\u591a\u5c11 core \u548c memory\u3002\u63a5\u4e0b\u6765\u4f1a\u8c03\u7528 <code>ta.recycle()<\/code> \u56de\u6536\u8d44\u6e90\u3002\u56de\u6536\u7684\u903b\u8f91\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">func (ta *NvidiaTopoAllocator) recycle() {\n    activePods := watchdog.GetActivePods()\n\n    lastActivePodUids := sets.NewString()\n    activePodUids := sets.NewString()\n    for _, uid := range ta.allocatedPod.Pods() {\n        lastActivePodUids.Insert(uid)\n    }\n    for uid := range activePods {\n        activePodUids.Insert(uid)\n    }\n\n    \/\/ difference \u51fa\u6765\u7684\u5c31\u662f\u5df2\u7ecf\u8fd0\u884c\u7ed3\u675f\u7684pod\uff0c\u53ef\u4ee5\u56de\u6536\u5206\u914d\u7684gpu\u8d44\u6e90\n    podsToBeRemoved := lastActivePodUids.Difference(activePodUids)\n\n    glog.V(5).Infof(\"Pods to be removed: %v\", podsToBeRemoved.List())\n\n    \/\/ \u91ca\u653e\u8d44\u6e90\n    ta.freeGPU(podsToBeRemoved.List())\n}\n<\/code><\/pre>\n<p>\u5bf9\u5df2\u5206\u914d\u7684 pod \u548c \u6b63\u5728\u8fd0\u884c\u7684 pod \u96c6\u5408\u53d6\u5dee\u96c6\uff0c\u5dee\u96c6\u5c31\u662f\u5206\u914d\u4e86\u8d44\u6e90\u4f46\u662f\u5df2\u7ecf\u505c\u6b62\u8fd0\u884c\u7684 pod \u3002\u7136\u540e\u5bf9\u8fd9\u90e8\u5206 pod \u91ca\u653e GPU \u8d44\u6e90\u3002\u5177\u4f53\u7684\u91ca\u653e\u903b\u8f91\u653e\u5728\u540e\u9762\u5206\u6790\u3002\u73b0\u5728\u7ee7\u7eed\u5411\u4e0b\u770b\uff0c\u8fd9\u91cc\u6211\u4eec\u76f4\u63a5\u8df3\u5230\u5c1d\u8bd5\u5206\u914d\u8d44\u6e90\u7684\u903b\u8f91\u4e0a\u3002\u5206\u914d gpu \u8d44\u6e90\u5206\u4e3a\u4e09\u79cd\u60c5\u51b5\uff1a<\/p>\n<ol>\n<li>\u5982\u679c\u9700\u8981\u7684\u6838\u5fc3\u6570\u5927\u4e8e 100\uff0c\u4e5f\u5c31\u662f\u8bf4\u8d85\u8fc7\u4e00\u4e2a\u7269\u7406 GPU\uff0c\u5c31\u4f7f\u7528 link \u8bc4\u4f30\u5668\u6765\u9009\u51fa GPU \u8282\u70b9<\/li>\n<li>\u5982\u679c\u6b63\u597d\u662f\u4e00\u4e2a 100 \u6838\u5fc3\uff0c\u5219\u4f7f\u7528 fragment \u8bc4\u4f30\u5668<\/li>\n<li>\u5982\u679c\u5c0f\u4e8e 100 \u6838\u5fc3\uff0c\u5219\u4f7f\u7528 share \u8bc4\u4f30\u5668\u3002<\/li>\n<\/ol>\n<p>\u60c5\u51b5 1 \u7684\u4ee3\u7801\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">eval, ok := ta.evaluators[\"link\"]\nif !ok {\n    return nil, fmt.Errorf(\"can not find evaluator link\")\n}\nif needCores%nvtree.HundredCore &gt; 0 {\n    return nil, fmt.Errorf(\"cores are greater than %d, must be multiple of %d\", nvtree.HundredCore, nvtree.HundredCore)\n}\nnodes = eval.Evaluate(needCores, 0)\n<\/code><\/pre>\n<p>\u6ce8\u610f\u5230\u8fd9\u91cc\u8fd8\u8981\u6c42\u8bf7\u6c42\u7684\u6838\u5fc3\u6570\u5fc5\u987b\u662f 100 \u7684\u6574\u6570\uff0c\u4e5f\u5c31\u662f\u8bf4\u5fc5\u987b\u662f\u6574\u6570\u4e2a\u7269\u7406 GPU\uff0c\u4f60\u4e0d\u80fd\u8bf7\u6c42 1.5 \u4e2a \u7269\u7406 GPU \u8fd9\u79cd\u3002<\/p>\n<p>\u60c5\u51b5 2 \u7684\u4ee3\u7801\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">eval, ok := ta.evaluators[\"fragment\"]\nif !ok {\n    return nil, fmt.Errorf(\"can not find evaluator fragment\")\n}\nnodes = eval.Evaluate(needCores, 0)\n<\/code><\/pre>\n<p>\u60c5\u51b5 3 \u7684\u4ee3\u7801\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ EnableShare \u662f\u5728\u542f\u52a8\u65f6\u6307\u5b9a\u7684\u53c2\u6570\uff0c\u4ee3\u8868\u662f\u5426\u5141\u8bb8\u591a\u4e2a\u5bb9\u5668\u5171\u4eab\u4e00\u4e2agpu\nif !ta.config.EnableShare {\n    return nil, fmt.Errorf(\"share mode is not enabled\")\n}\nif needCores == 0 || needMemory == 0 {\n    return nil, fmt.Errorf(\"that cores or memory is zero is not permitted in share mode\")\n}\n\n\/\/ evaluate in share mode\nshareMode = true\n\/\/ \u4f7f\u7528 share \u8bc4\u4f30\neval, ok := ta.evaluators[\"share\"]\nif !ok {\n    return nil, fmt.Errorf(\"can not find evaluator share\")\n}\n\/\/ \u8bc4\u4f30\u51fa\u6765\u7684\u5408\u9002\u7684 nvidia gpu \u8282\u70b9\nnodes = eval.Evaluate(needCores, needMemory)\nif len(nodes) == 0 {\n    if shareMode &amp;&amp; needMemory &gt; singleNodeMemory {\n        return nil, fmt.Errorf(\"request memory %d is larger than %d\", needMemory, singleNodeMemory)\n    }\n\n    return nil, fmt.Errorf(\"no free node\")\n}\n<\/code><\/pre>\n<p>\u5728\u8bc4\u4f30\u51fa\u6765\u8282\u70b9\u4e4b\u540e\uff0c\u4f1a\u5148\u5224\u65ad\u8fd9\u4e2a\u8fd9\u4e2a pod \u662f\u5426\u771f\u7684\u7ecf\u8fc7\u9884\u9009\u9636\u6bb5\uff1f\u5224\u65ad\u65b9\u6cd5\u5982\u4e0b\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">func IsGPUPredicatedPod(pod *v1.Pod) (predicated bool) {\n    glog.V(4).Infof(\"Determine if the pod %s needs GPU resource\", pod.Name)\n    var ok bool\n\n    \/\/ Check if pod request for GPU resource\n    if GetGPUResourceOfPod(pod, types.VCoreAnnotation) &lt;= 0 || GetGPUResourceOfPod(pod, types.VMemoryAnnotation) &lt;= 0 {\n        glog.V(4).Infof(\"Pod %s in namespace %s does not Request for GPU resource\",\n            pod.Name,\n            pod.Namespace)\n        return predicated\n    }\n\n    \/\/ Check if pod already has predicate time\n    \/\/ tencent.com\/predicate-time \u662f gpu-admission \u4e2d\u6dfb\u52a0\u7684\u3002\n    if _, ok = pod.ObjectMeta.Annotations[types.PredicateTimeAnnotation]; !ok {\n        glog.V(4).Infof(\"No predicate time for pod %s in namespace %s\",\n            pod.Name,\n            pod.Namespace)\n        return predicated\n    }\n\n    \/\/ Check if pod has already been assigned\n    if assigned, ok := pod.ObjectMeta.Annotations[types.GPUAssigned]; !ok {\n        glog.V(4).Infof(\"No assigned flag for pod %s in namespace %s\",\n            pod.Name,\n            pod.Namespace)\n        return predicated\n    } else if assigned == \"true\" {\n        glog.V(4).Infof(\"pod %s in namespace %s has already been assigned\",\n            pod.Name,\n            pod.Namespace)\n        return predicated\n    }\n    predicated = true\n    return predicated\n}\n<\/code><\/pre>\n<p>\u5171\u6709\u4e09\u4e2a\u8981\u6c42\u624d\u7b97\u7ecf\u8fc7\u4e86\u9884\u9009\uff1a<\/p>\n<ul>\n<li>resource \u5b57\u6bb5\u8bf7\u6c42\u4e86 vcore \u548c vgpu\uff0c\u5e76\u4e14\u5927\u4e8e 0\u3002<\/li>\n<li>\u5fc5\u987b\u6709 <code>tencent.com\/predicate-time<\/code> \u5b57\u6bb5\u3002\u8fd9\u70b9\u8981\u6c42\u5fc5\u987b\u7ecf\u8fc7 gpu-admission \u7684\u9884\u9009\u9636\u6bb5\u3002<\/li>\n<li>\u6ca1\u6709\u88ab\u5206\u914d gpu \u8d44\u6e90<\/li>\n<\/ul>\n<p>\u5982\u679c\u7ecf\u8fc7\u9884\u9009\u7684\u8bdd\uff0c\u5c31\u9700\u8981\u6267\u884c\u4ee5\u4e0b\u7684\u903b\u8f91\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ get predicate node by annotation\ncontainerIndex, err := utils.GetContainerIndexByName(pod, container.Name)\nif err != nil {\n    return nil, err\n}\nvar devStr string\nif idxStr, ok := pod.ObjectMeta.Annotations[types.PredicateGPUIndexPrefix+strconv.Itoa(containerIndex)]; ok {\n    if _, err := strconv.Atoi(idxStr); err != nil {\n        return nil, fmt.Errorf(\"predicate idx %s invalid for pod %s \", idxStr, pod.UID)\n    }\n    devStr = types.NvidiaDevicePrefix + idxStr\n    if !utils.IsValidGPUPath(devStr) {\n        return nil, fmt.Errorf(\"predicate idx %s invalid\", devStr)\n    }\n} else {\n    return nil, fmt.Errorf(\"failed to find predicate idx for pod %s\", pod.UID)\n}\n\npredicateNode := ta.tree.Query(devStr)\nif predicateNode == nil {\n    return nil, fmt.Errorf(\"failed to get predicate node %s\", devStr)\n}\n\n\/\/ check if we choose the same node as scheduler\nif predicateNode.MinorName() != nodes[0].MinorName() {\n    return nil, fmt.Errorf(\"Nvidia node mismatch for pod %s(%s), pick up:%s  predicate: %s\",\n        pod.Name, container.Name, nodes[0].MinorName(), predicateNode.MinorName())\n}\n<\/code><\/pre>\n<p>\u4e5f\u5c31\u662f\u8bf4\uff0c\u7ecf\u8fc7\u9884\u9009\u9636\u6bb5\u7684 Pod \u90fd\u4f1a\u6839\u636e\u5bb9\u5668\u7684\u987a\u5e8f\u5728 <code>Annotations<\/code> \u4e3a\u8be5\u5bb9\u5668\u5199\u4e0a\u914d\u7f6e\u4fe1\u606f\u3002\u8fd9\u8bf4\u660e\u5728 <code>gpu-admission<\/code> \u8fd9\u4e2a\u9879\u76ee\u4e2d\u4f1a\u4e3a\u5bb9\u5668\u5206\u914d gpu \u8bbe\u5907\u3002\u6700\u540e\u8fd8\u8981\u68c0\u67e5\u4e00\u4e0b\u5728 <code>gpu-manager<\/code> \u4e2d\u5206\u914d\u7684 gpu \u8bbe\u5907\u548c <code>gpu-admission<\/code> \u4e2d\u662f\u5426\u4e00\u81f4\uff0c\u4e0d\u4e00\u81f4\u7684\u8bdd\u4e5f\u4f1a\u8fd4\u56de\u5206\u914d\u5931\u8d25\u3002<\/p>\n<p>\u73b0\u5728\u6211\u4eec\u5df2\u7ecf\u77e5\u9053\u8981\u4e3a\u5f53\u524d\u8bf7\u6c42\u7684\u5bb9\u5668\u5206\u914d\u54ea\u4e2a gpu \u8bbe\u5907\uff0c\u4ee5\u53ca\u5206\u914d\u7684\u8d44\u6e90\u6570\u91cf\u3002\u8fd9\u6837\u5c31\u53ef\u4ee5\u6784\u5efa <code>ContainerAllocateResponse<\/code> \u4e86\u3002\u5148\u628a\u5df2\u5206\u914d\u7684\u8bbe\u5907\u653e\u5230\u54cd\u5e94\u4e2d\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">    for _, n := range nodes {\n        name := n.MinorName()\n        glog.V(2).Infof(\"Allocate %s for %s(%s), Meta (%d:%d)\", name, pod.UID, container.Name, n.Meta.ID, n.Meta.MinorID)\n\n        ctntResp.Annotations[types.VCoreAnnotation] = fmt.Sprintf(\"%d\", needCores)\n        ctntResp.Annotations[types.VMemoryAnnotation] = fmt.Sprintf(\"%d\", needMemory)\n\n        ctntResp.Devices = append(ctntResp.Devices, &amp;pluginapi.DeviceSpec{\n            ContainerPath: name,\n            HostPath:      name,\n            Permissions:   \"rwm\",\n        })\n        deviceList = append(deviceList, n.Meta.UUID)\n\n        if !allocated {\n            \/\/ \u5728 gpu tree \u4e2d\u6807\u8bb0\u8bbe\u5907\u5df2\u5360\u7528\n            ta.tree.MarkOccupied(n, needCores, needMemory)\n        }\n        allocatedDevices.Insert(name)\n    }\n<\/code><\/pre>\n<p>\u66f4\u6539\u54cd\u5e94\u7684 <code>Annotations<\/code>:<\/p>\n<pre><code class=\"language-go line-numbers\">ctntResp.Annotations[types.VDeviceAnnotation] = vDeviceAnnotationStr(nodes)\n<\/code><\/pre>\n<p>\u68c0\u67e5 pod \u7684\u6240\u6709\u5bb9\u5668\u662f\u5426\u90fd\u5b8c\u6210\u4e86\u5206\u914d\uff0c\u5e76\u628a\u65b0\u7684\u5206\u914d\u4fe1\u606f\u5199\u5165\u5230 checkpoint:<\/p>\n<pre><code class=\"language-go line-numbers\">unfinished := false\nfor _, c := range pod.Spec.Containers {\n    if !utils.IsGPURequiredContainer(&amp;c) {\n        continue\n    }\n    podCache := ta.allocatedPod.GetCache(string(pod.UID))\n    if podCache != nil {\n        if _, ok := podCache[c.Name]; !ok {\n            unfinished = true\n            break\n        }\n    }\n}\nif unfinished {\n    ta.unfinishedPod = pod\n} else {\n    ta.unfinishedPod = nil\n}\nta.writeCheckpoint()\n<\/code><\/pre>\n<p>\u5728\u54cd\u5e94\u4e2d\u4e3a\u5bb9\u5668\u6dfb\u52a0 <code>\/dev\/nvidiactl<\/code> \u548c <code>\/dev\/nvidia-uvm<\/code>\uff0c\u5982\u679c\u914d\u7f6e\u4e86 extraConfig\uff0c\u8fd8\u4f1a\u628a\u91cc\u9762\u8981\u9ed8\u8ba4\u6dfb\u52a0\u7684\u8bbe\u5907\u52a0\u8fdb\u53bb\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ Append control device\nctntResp.Devices = append(ctntResp.Devices, &amp;pluginapi.DeviceSpec{\n    ContainerPath: types.NvidiaCtlDevice,\n    HostPath:      types.NvidiaCtlDevice,\n    Permissions:   \"rwm\",\n})\n\nctntResp.Devices = append(ctntResp.Devices, &amp;pluginapi.DeviceSpec{\n    ContainerPath: types.NvidiaUVMDevice,\n    HostPath:      types.NvidiaUVMDevice,\n    Permissions:   \"rwm\",\n})\n\n\/\/ Append default device\nif cfg, found := ta.extraConfig[\"default\"]; found {\n    for _, dev := range cfg.Devices {\n        ctntResp.Devices = append(ctntResp.Devices, &amp;pluginapi.DeviceSpec{\n            ContainerPath: dev,\n            HostPath:      dev,\n            Permissions:   \"rwm\",\n        })\n    }\n}\n<\/code><\/pre>\n<p>\u6b64\u65f6\uff0c\u54cd\u5e94\u4e2d\u7684\u8bbe\u5907\u4fe1\u606f\u5df2\u7ecf\u5904\u7406\u7ed3\u675f\uff0c\u63a5\u4e0b\u6765\u5904\u7406\u5bb9\u5668\u4e2d\u7684\u73af\u5883\u53d8\u91cf\uff0cgpu manager \u9700\u8981\u901a\u8fc7\u4fee\u6539 <code>LD_LIBRARY_PATH<\/code> \u6765\u52ab\u6301\u7a0b\u5e8f\u5bf9 cuda \u7684\u8c03\u7528\uff0c\u7136\u540e\u901a\u8fc7 <code>NVIDIA_VISIBLE_DEVICES<\/code> \u6765\u8ba9\u6302\u8f7d\u7684\u8bbe\u5907\u53ef\u89c1\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ LD_LIBRARY_PATH\nctntResp.Envs[\"LD_LIBRARY_PATH\"] = \"\/usr\/local\/nvidia\/lib64\"\nfor _, env := range container.Env {\n    if env.Name == \"compat32\" &amp;&amp; strings.ToLower(env.Value) == \"true\" {\n        ctntResp.Envs[\"LD_LIBRARY_PATH\"] = \"\/usr\/local\/nvidia\/lib\"\n    }\n}\n\n\/\/ NVIDIA_VISIBLE_DEVICES\nctntResp.Envs[\"NVIDIA_VISIBLE_DEVICES\"] = strings.Join(deviceList, \",\")\n<\/code><\/pre>\n<p>\u63a5\u7740\u6839\u636e\u662f\u5426\u5904\u4e8e <code>shareMode<\/code>\uff0c\u4e5f\u5c31\u662f\u5355\u4e2a gpu \u80fd\u5426\u88ab\u5171\u4eab\u6765\u6302\u8f7d\u4e0d\u540c\u7684 host \u76ee\u5f55\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">if shareMode {\n    \/\/ nvidia \u662f\u52ab\u6301\u7684\u5e93\uff0c\u7528\u5728shareMode\u8fd9\u79cd\u60c5\u51b5\n    ctntResp.Mounts = append(ctntResp.Mounts, &amp;pluginapi.Mount{\n        ContainerPath: \"\/usr\/local\/nvidia\",\n        HostPath:      types.DriverLibraryPath,\n        ReadOnly:      true,\n    })\n} else {\n    \/\/ \u975eshareMode\u7528\u6b63\u5e38\u7684\u5e93\u5373\u53ef\n    ctntResp.Mounts = append(ctntResp.Mounts, &amp;pluginapi.Mount{\n        ContainerPath: \"\/usr\/local\/nvidia\",\n        HostPath:      types.DriverOriginLibraryPath,\n        ReadOnly:      true,\n    })\n}\n<\/code><\/pre>\n<p><code>shareMode<\/code> \u4e0b\uff0c\u4f1a\u6302\u8f7d\u7684 host \u76ee\u5f55\u662f <code>\/etc\/gpu-manager\/vdriver\/nvidia<\/code>\uff0c\u8fd9\u91cc\u9762\u662f\u88ab\u52ab\u6301\u7684\u5e93\u3002\u5426\u5219\u6302\u8f7d <code>\/etc\/gpu-manager\/vdriver\/origin<\/code>\uff0c\u91cc\u9762\u662f\u539f\u59cb\u7684 CUDA \u5e93\u3002<\/p>\n<p>\u7d27\u63a5\u7740\uff0c\u5c06 host \u4e0a\u7684 <code>\/etc\/gpu-manager\/vm\/{podUID}<\/code> \u6302\u8f7d\u5230\u5bb9\u5668\u4e2d\uff0c\u8fd9\u4e2a\u662f\u4e3a\u4e86\u5bb9\u5668\u5185\u53ef\u4ee5\u901a\u8fc7 <code>vcuda.sock<\/code> \u548c <code>virtual-manager<\/code> \u901a\u4fe1\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">\/\/ \u5c06host\u4e0a\u7684\/etc\/gpu-manager\/vm\/podUID\u6302\u8f7d\u8fdb\u53bb(vcuda.sock)\uff0c\u8fd9\u4e2a\u76ee\u5f55\u662f\u5728PreStartContainer\u671f\u95f4\u7531VirtualManager\u521b\u5efa\u7684\nctntResp.Mounts = append(ctntResp.Mounts, &amp;pluginapi.Mount{\n    ContainerPath: types.VCUDA_MOUNTPOINT,\n    HostPath:      filepath.Join(ta.config.VirtualManagerPath, string(pod.UID)),\n    ReadOnly:      true,\n})\n<\/code><\/pre>\n<p>\u5982\u679c\u5f53\u524d\u8bf7\u6c42\u7684\u5bb9\u5668\u6240\u5c5e Pod \u6ca1\u6709\u7ecf\u8fc7 <code>gpu-admission<\/code>\uff0c\u8fd8\u4f1a\u88ab\u653e\u5230\u4e00\u4e2a\u5904\u7406\u961f\u5217\u4e2d\uff1a<\/p>\n<pre><code class=\"language-go line-numbers\">if predicateMissed {\n    ar := &amp;allocateResult{\n        pod:     pod,\n        result:  PREDICATE_MISSING,\n        resChan: make(chan struct{}),\n    }\n\n    \/\/ \u8fd9\u4e2a queue \u7684\u5904\u7406\u662f\u5728virtualmanager\u91cc\u9762\u7684process\u65b9\u6cd5\n    ta.queue.AddRateLimited(ar)\n    &lt;-ar.resChan\n}\n<\/code><\/pre>\n<p>\u8fd9\u4e2a\u961f\u5217\u4f1a\u5728 <code>pkg\/service\/allocator\/nvidia\/allocator.go<\/code> \u7684 <code>proccessResult<\/code> \u4e2d\u5904\u7406\u3002<\/p>\n<p>\u8fd9\u6837\uff0ckubelet \u8c03\u7528 <code>Allocate<\/code> \u65b9\u6cd5\u5c31\u7ed3\u675f\u4e86\u3002\u8fd9\u91cc\u518d\u6765\u56de\u987e\u4e00\u4e0b\u4e0a\u9762\u9057\u7559\u7684\u95ee\u9898\u548c\u76f8\u5173\u903b\u8f91\uff1a<\/p>\n<p>\u95ee\u9898\uff1a<\/p>\n<ul>\n<li>\u95ee\u9898 A: \u4e3a\u4ec0\u4e48\u8981\u5927\u8d39\u5468\u7ae0\u7684\u901a\u8fc7 grpc\uff0c\u76f4\u63a5\u6302\u8f7d\u5bb9\u5668\u914d\u7f6e\u6587\u4ef6\u53ef\u884c\u5417\uff1f\n<p>\u603b\u7ed3\uff1a \u8fd9\u4e00\u70b9\u5728\u4e0a\u9762\u7684\u9605\u8bfb\u4e2d\u53ef\u4ee5\u53d1\u73b0\uff0c\u8fd9\u65f6\u5019\u5bb9\u5668\u672c\u8eab\u5bf9\u81ea\u5df1\u5e94\u8be5\u9650\u5236\u591a\u5c11\u7684 gpu \u8d44\u6e90\u8c03\u7528\u5e76\u4e0d\u77e5\u9053\u3002\u8fd9\u4e2a\u95ee\u9898\u5f97\u548c B\/C\/D \u95ee\u9898\u7ed3\u5408\u6765\u770b\u3002\u56e0\u4e3a\u505a\u3000Allocate \u8c03\u7528\u65f6\uff0ckubelet \u5e76\u6ca1\u6709\u544a\u77e5\u6b64\u65f6\u5728\u4e3a\u54ea\u4e2a\u5bb9\u5668\u8bf7\u6c42\u914d\u7f6e\u3002\u56e0\u6b64\u53ea\u80fd\u6839\u636e\u8bf7\u6c42\u7684\u8d44\u6e90\u91cf\u4ee5\u53ca Pod \u7684 predicateTime \u6216 createTime \u6765\u5224\u65ad\u3002\u8fd9\u4e2a\u662f\u65e0\u6cd5\u4fdd\u8bc1\u4e00\u5b9a\u51c6\u786e\u7684\uff0c\u56e0\u6b64\u6b64\u65f6\u5bb9\u5668\u7684\u5177\u4f53\u8d44\u6e90\u914d\u7f6e\u4e5f\u65e0\u6cd5\u786e\u5b9a\u3002\u53ef\u80fd\u8fd9\u5c31\u662f\u8981\u901a\u8fc7 grpc \u800c\u4e0d\u662f\u6302\u8f7d\u5bb9\u5668\u914d\u7f6e\u6587\u4ef6\u7684\u539f\u56e0\u5427\u3002<\/p>\n<\/li>\n<li>\n<p>\u95ee\u9898 B: \u5982\u679c\u4e00\u4e2a Pod \u4e2d\u6709\u591a\u4e2a vcore \u8bf7\u6c42\u4e00\u81f4\uff0c\u4f46\u662f vmemory \u4e0d\u540c\u7684\u5bb9\u5668\uff0c\u8fd9\u91cc\u53ea\u901a\u8fc7 vcore \u7684\u8bf7\u6c42\u91cf\u6765\u5224\u65ad\uff0c\u53ef\u4ee5\u4fdd\u8bc1\u8fd9\u4e2a\u5206\u914d\u8bf7\u6c42\u548c\u6211\u4eec\u7684\u5019\u9009\u5bb9\u5668\u80fd\u5bf9\u7684\u4e0a\u5417\uff1f<br \/>\n\u603b\u7ed3\uff1a\u95ee\u9898 B \u662f\u5728 unfinisedPod \u4e2d\u67e5\u627e\u5f53\u524d\u8bf7\u6c42\u7684\u5bb9\u5668\u3002\u53ea\u8981\u80fd\u4fdd\u8bc1 unfinishedPod \u662f\u6b63\u786e\u7684\uff08\u95ee\u9898 D \u8bf4\u660e\u4e0d\u80fd\u4fdd\u8bc1\uff09\uff0c\u90a3\u4e48\u5c31\u53ef\u4ee5\u4fdd\u8bc1\u5bb9\u5668\u662f\u5bf9\u7684\u4e0a\u7684\uff08\u95ee\u9898 C \u4fdd\u8bc1\u4e86\u8fd9\u4e2a\u7ed3\u8bba\uff09\u3002<\/p>\n<\/li>\n<li>\n<p>\u95ee\u9898 C: AllocateRequest \u662f\u6309\u7167 Pod \u4e2d\u7684\u5bb9\u5668\u987a\u5e8f\u6765\u7684?<br \/>\n\u603b\u7ed3\uff1a\u5bf9\u4e8e\u8fd9\u4e2a\u95ee\u9898\uff0c\u6700\u597d\u7684\u56de\u7b54\u65b9\u5f0f\u662f\u53bb\u770b <code>kubelet<\/code> \u7684\u6e90\u4ee3\u7801\u3002<\/p>\n<pre><code class=\"language-go line-numbers\">for _, container := range pod.Spec.Containers {\n    if err := m.allocateContainerResources(pod, &amp;container, devicesToReuse); err != nil {\n        return err\n    }\n    m.podDevices.removeContainerAllocatedResources(string(pod.UID), container.Name, devicesToReuse)\n}\n<\/code><\/pre>\n<p>\u8fd9\u8fb9\u505a <code>Allocate<\/code> \u7684\u65f6\u5019\uff0c\u662f\u987a\u5e8f\u904d\u5386 Pod \u4e2d\u7684\u5bb9\u5668\uff0c\u56e0\u6b64\u8fd9\u4e2a\u95ee\u9898\u7684\u7b54\u6848\u662f\u80af\u5b9a\u7684\u3002<\/p>\n<\/li>\n<li>\n<p>\u95ee\u9898 D: \u904d\u5386\u5f53\u524d\u8282\u70b9\u4e0a\u7684\u6240\u6709 pod\uff0c\u7136\u540e\u6311\u51fa\u9700\u8981 gpu \u8d44\u6e90\u7684 pod\uff0c\u6839\u636e <code>predicatedTime<\/code> \u6216 <code>createTime<\/code> \u6392\u5e8f\u3002\u7136\u540e\u518d\u4ece\u8fd9\u4e9b pod \u4e2d\uff0c \u6309\u987a\u5e8f\u6311\u51fa\u7b26\u5408\u8fd9\u6b21\u8bf7\u6c42\u7684\u5bb9\u5668\uff0c\u600e\u4e48\u4fdd\u8bc1\u6311\u51fa\u6765\u7684\u5bb9\u5668\u5c31\u662f\u8fd9\u6b21\u5206\u914d\u8bf7\u6c42\u7684\u5462\uff1f<\/p>\n<p>\u603b\u7ed3\uff1a\u6211\u89c9\u5f97\u56de\u7b54\u8fd9\u4e2a\u95ee\u9898\uff0c\u9700\u8981\u786e\u5b9a\u4e24\u4e2a\u5927\u524d\u63d0\uff0c\u4e00\u662f Pod \u4ece\u521b\u5efa\u5230\u53d1\u8d77 Allocate \u7684\u8fc7\u7a0b\uff0c\u90fd\u662f\u987a\u5e8f\u7684\u3002\u8fd9\u6837\u5c31\u80fd\u4fdd\u8bc1\u5f53\u8c03\u7528 Allocate \u5bf9\u5e94\u7684 Pod \u6c38\u8fdc\u662f\u5c1a\u672a\u5206\u914d\u5230\u8d44\u6e90\u7684\u7b2c\u4e00\u4e2a\u3002\u4e8c\u662f\u5728\u4e00\u4e2a Pod \u4e2d\uff0c\u4e3a\u6bcf\u4e2a\u5bb9\u5668 Allocate \u65f6\uff0c\u4e5f\u662f\u987a\u5e8f\u7684\uff0c\u8fd9\u4e00\u70b9\u5728\u95ee\u9898 <code>C<\/code> \u4e2d\u5f97\u5230\u786e\u8ba4\u3002<\/p>\n<p>\u4f46\u662f\u5b9e\u9645\u4e0a\uff0c\u7b2c\u4e00\u4e2a\u524d\u63d0\u662f\u4e0d\u80fd\u4fdd\u8bc1\u7684\uff0c\u5728 Pod <code>bind<\/code> \u5230\u8282\u70b9\u65f6\uff0c\u8fd9\u4e2a\u662f\u5e76\u53d1\u6267\u884c\u7684\u3002\u56e0\u6b64\u53ef\u4ee5\u5f97\u51fa\u4e00\u4e2a\u7ed3\u8bba\uff1a\u5728\u8fd9\u4e2a\u9636\u6bb5\u65e0\u6cd5\u4fdd\u8bc1 Allocate \u8bf7\u6c42\u548c\u6211\u4eec\u7684\u5019\u9009\u5bb9\u5668\u662f\u5bf9\u5e94\u5173\u7cfb\u3002\u5173\u4e8e\u8fd9\u4e00\u70b9\u6211\u4e5f\u63d0\u4e86\u4e2a issue\uff1a<a class=\"wp-editor-md-post-content-link\" href=\"https:\/\/github.com\/tkestack\/gpu-manager\/issues\/17\">a question about Allocate for a container?<\/a>\u3002\u5b98\u65b9\u4e5f\u7ed9\u4e86\u56de\u7b54\uff0c\u56e0\u4e3a\u8fd9\u4e2a\u539f\u56e0 gpu manager \u6709\u65f6\u5019\u4f1a\u62a5 <code>UnexpectedAdmissionError<\/code> \u9519\u8bef\u3002<\/p>\n<p>\u6240\u4ee5\u6839\u636e\u95ee\u9898 4\uff0c\u6211\u4eec\u8fd8\u8981\u4f7f\u7528 <code>gpu-admission<\/code> \u8fd9\u4e2a\u9879\u76ee\uff0c\u6765\u4fdd\u8bc1\u8be5\u9636\u6bb5\u7684\u6b63\u786e\u6027\uff0c\u5177\u4f53\u673a\u5236\u8fd8\u5f97\u7b49\u5230\u770b <code>gpu-admission<\/code> \u7684\u65f6\u5019\u624d\u80fd\u77e5\u9053\u4e86\u3002<\/p>\n<\/li>\n<\/ul>\n<p>\u5176\u5b9e\u4ee5\u4e0a\u56db\u4e2a\u95ee\u9898\u90fd\u662f\u56e0\u4e3a kubelet \u7684 Allocate \u8bf7\u6c42\u4e0d\u4f1a\u5e26\u4e0a\u6b63\u5728\u5206\u914d\u7684\u5bb9\u5668\u3002\u6240\u4ee5\u9700\u8981\u4e00\u7cfb\u5217\u7684\u67e5\u627e\u65b9\u5f0f\u6765\u786e\u5b9a\u5177\u4f53\u7684\u5bb9\u5668\u3002<\/p>\n<p>\u56e0\u4e3a\u7bc7\u5e45\u95ee\u9898\uff0c\u5173\u4e8e gpu \u7684\u6700\u4f73\u5206\u914d\u7b56\u7565\u4f1a\u4f5c\u4e3a\u4e0b\u4e00\u7bc7\u6587\u7ae0\u7684\u5185\u5bb9\u3002<\/p>\n<h2>\u53c2\u8003\u8d44\u6599<\/h2>\n<p><a class=\"wp-editor-md-post-content-link\" href=\"https:\/\/www.infoq.cn\/article\/or7CRphTDlX1IVhsFNgk\">\u4ece\u96f6\u5f00\u59cb\u5165\u95e8 K8s\uff1a\u8c03\u5ea6\u5668\u7684\u8c03\u5ea6\u6d41\u7a0b\u548c\u7b97\u6cd5\u4ecb\u7ecd<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6982\u8ff0 \u5728\u4e4b\u524d\u7684\u4e00\u7bc7\u6587\u7ae0\u4e3b\u8981\u662f\u5206\u6790\u4e86 gpu-manager \u7684\u542f\u52a8\u6d41\u7a0b\u3002\u5173\u4e8e gpu-manager \u5e94\u8be5\u4f1a\u6709 &hellip; <a href=\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/\" class=\"more-link\">\u7ee7\u7eed\u9605\u8bfb<span class=\"screen-reader-text\">[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[89],"tags":[],"class_list":["post-601","post","type-post","status-publish","format-standard","hentry","category-k8s"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v25.4 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b - \u4e00\u53ea\u5b89\u9759\u7684\u732b<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b - \u4e00\u53ea\u5b89\u9759\u7684\u732b\" \/>\n<meta property=\"og:description\" content=\"\u6982\u8ff0 \u5728\u4e4b\u524d\u7684\u4e00\u7bc7\u6587\u7ae0\u4e3b\u8981\u662f\u5206\u6790\u4e86 gpu-manager \u7684\u542f\u52a8\u6d41\u7a0b\u3002\u5173\u4e8e gpu-manager \u5e94\u8be5\u4f1a\u6709 &hellip; \u7ee7\u7eed\u9605\u8bfb[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/\" \/>\n<meta property=\"og:site_name\" content=\"\u4e00\u53ea\u5b89\u9759\u7684\u732b\" \/>\n<meta property=\"article:published_time\" content=\"2020-04-08T03:09:11+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2023-07-05T13:28:09+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg\" \/>\n<meta name=\"author\" content=\"jiangpengfei\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"jiangpengfei\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"7 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#article\",\"isPartOf\":{\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/\"},\"author\":{\"name\":\"jiangpengfei\",\"@id\":\"https:\/\/www.myway5.com\/#\/schema\/person\/b19267e8b106343431e163ec96950685\"},\"headline\":\"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b\",\"datePublished\":\"2020-04-08T03:09:11+00:00\",\"dateModified\":\"2023-07-05T13:28:09+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/\"},\"wordCount\":269,\"commentCount\":5,\"publisher\":{\"@id\":\"https:\/\/www.myway5.com\/#\/schema\/person\/b19267e8b106343431e163ec96950685\"},\"image\":{\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#primaryimage\"},\"thumbnailUrl\":\"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg\",\"articleSection\":[\"k8s\"],\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"CommentAction\",\"name\":\"Comment\",\"target\":[\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#respond\"]}]},{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/\",\"url\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/\",\"name\":\"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b - \u4e00\u53ea\u5b89\u9759\u7684\u732b\",\"isPartOf\":{\"@id\":\"https:\/\/www.myway5.com\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#primaryimage\"},\"image\":{\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#primaryimage\"},\"thumbnailUrl\":\"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg\",\"datePublished\":\"2020-04-08T03:09:11+00:00\",\"dateModified\":\"2023-07-05T13:28:09+00:00\",\"breadcrumb\":{\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#primaryimage\",\"url\":\"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg\",\"contentUrl\":\"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.myway5.com\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.myway5.com\/#website\",\"url\":\"https:\/\/www.myway5.com\/\",\"name\":\"\u4e00\u53ea\u5b89\u9759\u7684\u732b\",\"description\":\"\u60f3\u5565\u5462\",\"publisher\":{\"@id\":\"https:\/\/www.myway5.com\/#\/schema\/person\/b19267e8b106343431e163ec96950685\"},\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.myway5.com\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"zh-Hans\"},{\"@type\":[\"Person\",\"Organization\"],\"@id\":\"https:\/\/www.myway5.com\/#\/schema\/person\/b19267e8b106343431e163ec96950685\",\"name\":\"jiangpengfei\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.myway5.com\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/f8c7de757f6e0247412bcfd31b7c2271?s=96&d=monsterid&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/f8c7de757f6e0247412bcfd31b7c2271?s=96&d=monsterid&r=g\",\"caption\":\"jiangpengfei\"},\"logo\":{\"@id\":\"https:\/\/www.myway5.com\/#\/schema\/person\/image\/\"},\"url\":\"https:\/\/www.myway5.com\/index.php\/author\/joyme\/\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b - \u4e00\u53ea\u5b89\u9759\u7684\u732b","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/","og_locale":"zh_CN","og_type":"article","og_title":"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b - \u4e00\u53ea\u5b89\u9759\u7684\u732b","og_description":"\u6982\u8ff0 \u5728\u4e4b\u524d\u7684\u4e00\u7bc7\u6587\u7ae0\u4e3b\u8981\u662f\u5206\u6790\u4e86 gpu-manager \u7684\u542f\u52a8\u6d41\u7a0b\u3002\u5173\u4e8e gpu-manager \u5e94\u8be5\u4f1a\u6709 &hellip; \u7ee7\u7eed\u9605\u8bfb[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b","og_url":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/","og_site_name":"\u4e00\u53ea\u5b89\u9759\u7684\u732b","article_published_time":"2020-04-08T03:09:11+00:00","article_modified_time":"2023-07-05T13:28:09+00:00","og_image":[{"url":"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg","type":"","width":"","height":""}],"author":"jiangpengfei","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"jiangpengfei","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"7 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#article","isPartOf":{"@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/"},"author":{"name":"jiangpengfei","@id":"https:\/\/www.myway5.com\/#\/schema\/person\/b19267e8b106343431e163ec96950685"},"headline":"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b","datePublished":"2020-04-08T03:09:11+00:00","dateModified":"2023-07-05T13:28:09+00:00","mainEntityOfPage":{"@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/"},"wordCount":269,"commentCount":5,"publisher":{"@id":"https:\/\/www.myway5.com\/#\/schema\/person\/b19267e8b106343431e163ec96950685"},"image":{"@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#primaryimage"},"thumbnailUrl":"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg","articleSection":["k8s"],"inLanguage":"zh-Hans","potentialAction":[{"@type":"CommentAction","name":"Comment","target":["https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#respond"]}]},{"@type":"WebPage","@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/","url":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/","name":"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b - \u4e00\u53ea\u5b89\u9759\u7684\u732b","isPartOf":{"@id":"https:\/\/www.myway5.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#primaryimage"},"image":{"@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#primaryimage"},"thumbnailUrl":"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg","datePublished":"2020-04-08T03:09:11+00:00","dateModified":"2023-07-05T13:28:09+00:00","breadcrumb":{"@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/"]}]},{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#primaryimage","url":"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg","contentUrl":"https:\/\/www.myway5.com\/wp-content\/uploads\/2020\/03\/device-plugins.svg"},{"@type":"BreadcrumbList","@id":"https:\/\/www.myway5.com\/index.php\/2020\/04\/08\/gaia-scheduler-gpu-manager\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.myway5.com\/"},{"@type":"ListItem","position":2,"name":"[Gaia Scheduler] gpu-manager \u7684\u865a\u62df\u5316 gpu \u5206\u914d\u6d41\u7a0b"}]},{"@type":"WebSite","@id":"https:\/\/www.myway5.com\/#website","url":"https:\/\/www.myway5.com\/","name":"\u4e00\u53ea\u5b89\u9759\u7684\u732b","description":"\u60f3\u5565\u5462","publisher":{"@id":"https:\/\/www.myway5.com\/#\/schema\/person\/b19267e8b106343431e163ec96950685"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.myway5.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"zh-Hans"},{"@type":["Person","Organization"],"@id":"https:\/\/www.myway5.com\/#\/schema\/person\/b19267e8b106343431e163ec96950685","name":"jiangpengfei","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.myway5.com\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/f8c7de757f6e0247412bcfd31b7c2271?s=96&d=monsterid&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/f8c7de757f6e0247412bcfd31b7c2271?s=96&d=monsterid&r=g","caption":"jiangpengfei"},"logo":{"@id":"https:\/\/www.myway5.com\/#\/schema\/person\/image\/"},"url":"https:\/\/www.myway5.com\/index.php\/author\/joyme\/"}]}},"views":11062,"_links":{"self":[{"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/posts\/601","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/comments?post=601"}],"version-history":[{"count":2,"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/posts\/601\/revisions"}],"predecessor-version":[{"id":1547,"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/posts\/601\/revisions\/1547"}],"wp:attachment":[{"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/media?parent=601"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/categories?post=601"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.myway5.com\/index.php\/wp-json\/wp\/v2\/tags?post=601"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}