mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-06 12:03:39 +09:00
Aleksei Sviridkin
8f4036c0da
GPUs become a catalog of hoster offers, tenant claims against them and a per-node inventory, so a tenant can hold a GPU while its VM is off and an order that does not fit is refused at create time. The CRDs enforce what the apiserver alone can: a class name is a DNS label other than total because it becomes part of a quota key, the source is a union with dra reserved for a later member, a class's mode never changes because a pool that names the class takes its worker template from it, a claim's spec is immutable and orders at least one GPU, and bound claims are listable per node through a field selector on status.nodeName. A class resource name must have the shape of an extended resource name and lie outside gpu.cozystack.io, where it would read as a GPU budget key. The CRDs are kept on chart uninstall, since they ship as plain templates and deleting GPUClaim would drop every hold and usage record. A class carries the tolerations of its VMs. A claim is bound when it is ordered, before any pod exists, so a dedicated taint on GPU nodes can only be checked against the class; without them such a node would serve no class at all. The tolerations are copied onto launcher pods, so the class takes only what pod validation accepts on every cluster: Exists and Equal, since Gt and Lt are refused where their feature gate is off, and the key, value, effect and tolerationSeconds rules of a pod toleration. They are a type of their own rather than the core one, because the core type carries no schema to put length limits on, and without them the checks would not fit the CEL cost budget. A claim records the nodes that refused its set's pinned launcher pod, each with an expiry, in status, so a leader change does not forget them and the controller does not pin the set to the same node again on its next pass. maxCount carries no schema default: it defaults to 1 for vgpu classes and to no limit for passthrough, which a structural default cannot express, so readers resolve it through the shared model. A GPUNode device reports how many device-plugin devices kubelet counts as allocatable for it, because an mdev parent serves several from one PCI function and one healthy entry per function would undercount the node. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
301 lines
13 KiB
YAML
301 lines
13 KiB
YAML
---
|
|
apiVersion: apiextensions.k8s.io/v1
|
|
kind: CustomResourceDefinition
|
|
metadata:
|
|
annotations:
|
|
controller-gen.kubebuilder.io/version: v0.16.4
|
|
helm.sh/resource-policy: keep
|
|
name: gpuclasses.gpu.cozystack.io
|
|
spec:
|
|
group: gpu.cozystack.io
|
|
names:
|
|
kind: GPUClass
|
|
listKind: GPUClassList
|
|
plural: gpuclasses
|
|
singular: gpuclass
|
|
scope: Cluster
|
|
versions:
|
|
- additionalPrinterColumns:
|
|
- jsonPath: .spec.mode
|
|
name: Mode
|
|
type: string
|
|
- jsonPath: .spec.model
|
|
name: Model
|
|
type: string
|
|
- jsonPath: .spec.profile
|
|
name: Profile
|
|
type: string
|
|
- jsonPath: .status.free
|
|
name: Free
|
|
type: integer
|
|
- jsonPath: .metadata.creationTimestamp
|
|
name: Age
|
|
type: date
|
|
name: v1alpha1
|
|
schema:
|
|
openAPIV3Schema:
|
|
description: GPUClass is one GPU offer of the hoster's catalog.
|
|
properties:
|
|
apiVersion:
|
|
description: |-
|
|
APIVersion defines the versioned schema of this representation of an object.
|
|
Servers should convert recognized schemas to the latest internal value, and
|
|
may reject unrecognized values.
|
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
|
type: string
|
|
kind:
|
|
description: |-
|
|
Kind is a string value representing the REST resource this object represents.
|
|
Servers may infer this from the endpoint the client submits requests to.
|
|
Cannot be updated.
|
|
In CamelCase.
|
|
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
|
type: string
|
|
metadata:
|
|
type: object
|
|
spec:
|
|
description: GPUClassSpec defines one GPU offer of the hoster's catalog.
|
|
properties:
|
|
displayName:
|
|
description: DisplayName is the human-readable name of the offer.
|
|
maxLength: 256
|
|
type: string
|
|
maxCount:
|
|
description: |-
|
|
MaxCount is how many GPUs of this class one VM may take. Unset means 1
|
|
for mode vgpu and, for mode passthrough, no limit beyond the GPUs of
|
|
one node.
|
|
format: int32
|
|
minimum: 1
|
|
type: integer
|
|
memory:
|
|
anyOf:
|
|
- type: integer
|
|
- type: string
|
|
description: Memory is the GPU memory of one unit of the class.
|
|
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
|
x-kubernetes-int-or-string: true
|
|
mode:
|
|
description: |-
|
|
Mode cannot change: a pool that names the class takes its worker
|
|
template from it.
|
|
enum:
|
|
- passthrough
|
|
- vgpu
|
|
type: string
|
|
x-kubernetes-validations:
|
|
- message: mode is immutable; create a new GPUClass instead
|
|
rule: self == oldSelf
|
|
model:
|
|
maxLength: 63
|
|
minLength: 1
|
|
type: string
|
|
nodeSelector:
|
|
additionalProperties:
|
|
type: string
|
|
description: NodeSelector selects the nodes that serve this class.
|
|
type: object
|
|
onCordon:
|
|
default: Move
|
|
description: OnCordon decides whether an unattached claim leaves a
|
|
cordoned node.
|
|
enum:
|
|
- Move
|
|
- Stay
|
|
type: string
|
|
profile:
|
|
description: Profile is the vGPU profile, such as L40S-24C. Set only
|
|
for mode vgpu.
|
|
maxLength: 63
|
|
minLength: 1
|
|
type: string
|
|
source:
|
|
description: |-
|
|
GPUClassSource is the backend that serves a class. Exactly one member is
|
|
set. The name "dra" is reserved for a DRA-backed member.
|
|
properties:
|
|
devicePlugin:
|
|
description: DevicePluginSource serves a class through a Kubernetes
|
|
device plugin.
|
|
properties:
|
|
resourceName:
|
|
description: |-
|
|
ResourceName is the extended resource the device plugin advertises for
|
|
this class's GPUs. Two classes may share it only when their node
|
|
selectors do not overlap.
|
|
maxLength: 317
|
|
minLength: 3
|
|
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?$
|
|
type: string
|
|
x-kubernetes-validations:
|
|
- message: resourceName must not be in gpu.cozystack.io, whose
|
|
keys are GPU budgets
|
|
rule: '!self.startsWith(''gpu.cozystack.io/'')'
|
|
required:
|
|
- resourceName
|
|
type: object
|
|
type: object
|
|
x-kubernetes-validations:
|
|
- message: exactly one source must be set; devicePlugin is the only
|
|
source supported, dra is reserved
|
|
rule: '[has(self.devicePlugin)].filter(x, x).size() == 1'
|
|
tolerations:
|
|
description: |-
|
|
Tolerations are the taints of the class's nodes that its VMs
|
|
tolerate, in the shape of a pod's, operators Exists and Equal only. A
|
|
node with a NoSchedule or NoExecute taint they do not tolerate serves
|
|
no claim of the class, and the virt-launcher pod of a VMI gets the
|
|
tolerations of its claims' classes.
|
|
items:
|
|
description: |-
|
|
GPUClassToleration is a pod toleration that pod validation accepts on
|
|
every cluster: the class's tolerations are copied onto launcher pods, so
|
|
one the API server refuses there would break every start of the class.
|
|
Gt and Lt are left out because pod validation refuses them where their
|
|
feature gate is off.
|
|
properties:
|
|
effect:
|
|
description: Effect matches every effect when unset.
|
|
enum:
|
|
- NoSchedule
|
|
- PreferNoSchedule
|
|
- NoExecute
|
|
type: string
|
|
key:
|
|
maxLength: 317
|
|
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?$
|
|
type: string
|
|
operator:
|
|
description: Operator is Equal when unset.
|
|
enum:
|
|
- Exists
|
|
- Equal
|
|
type: string
|
|
tolerationSeconds:
|
|
format: int64
|
|
type: integer
|
|
value:
|
|
maxLength: 63
|
|
pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$
|
|
type: string
|
|
type: object
|
|
x-kubernetes-validations:
|
|
- message: a toleration with an empty key needs operator Exists
|
|
rule: (has(self.key) && size(self.key) > 0) || (has(self.operator)
|
|
&& self.operator == 'Exists')
|
|
- message: a toleration with operator Exists takes no value
|
|
rule: '!has(self.operator) || self.operator != ''Exists'' || !has(self.value)
|
|
|| size(self.value) == 0'
|
|
- message: tolerationSeconds needs effect NoExecute
|
|
rule: '!has(self.tolerationSeconds) || (has(self.effect) && self.effect
|
|
== ''NoExecute'')'
|
|
- message: toleration key prefix must be at most 253 characters
|
|
rule: '!has(self.key) || self.key.indexOf(''/'') <= 253'
|
|
maxItems: 32
|
|
type: array
|
|
x-kubernetes-list-type: atomic
|
|
vendor:
|
|
maxLength: 63
|
|
minLength: 1
|
|
type: string
|
|
required:
|
|
- mode
|
|
- model
|
|
- source
|
|
- vendor
|
|
type: object
|
|
x-kubernetes-validations:
|
|
- message: profile is required for mode vgpu and not allowed for mode
|
|
passthrough
|
|
rule: 'self.mode == ''vgpu'' ? has(self.profile) : !has(self.profile)'
|
|
status:
|
|
description: |-
|
|
GPUClassStatus is readable by every authenticated user, so it carries no
|
|
node names and no counts beyond free capacity.
|
|
properties:
|
|
conditions:
|
|
items:
|
|
description: Condition contains details for one aspect of the current
|
|
state of this API Resource.
|
|
properties:
|
|
lastTransitionTime:
|
|
description: |-
|
|
lastTransitionTime is the last time the condition transitioned from one status to another.
|
|
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
|
|
format: date-time
|
|
type: string
|
|
message:
|
|
description: |-
|
|
message is a human readable message indicating details about the transition.
|
|
This may be an empty string.
|
|
maxLength: 32768
|
|
type: string
|
|
observedGeneration:
|
|
description: |-
|
|
observedGeneration represents the .metadata.generation that the condition was set based upon.
|
|
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
|
|
with respect to the current state of the instance.
|
|
format: int64
|
|
minimum: 0
|
|
type: integer
|
|
reason:
|
|
description: |-
|
|
reason contains a programmatic identifier indicating the reason for the condition's last transition.
|
|
Producers of specific condition types may define expected values and meanings for this field,
|
|
and whether the values are considered a guaranteed API.
|
|
The value should be a CamelCase string.
|
|
This field may not be empty.
|
|
maxLength: 1024
|
|
minLength: 1
|
|
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
|
|
type: string
|
|
status:
|
|
description: status of the condition, one of True, False, Unknown.
|
|
enum:
|
|
- "True"
|
|
- "False"
|
|
- Unknown
|
|
type: string
|
|
type:
|
|
description: type of condition in CamelCase or in foo.example.com/CamelCase.
|
|
maxLength: 316
|
|
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
|
|
type: string
|
|
required:
|
|
- lastTransitionTime
|
|
- message
|
|
- reason
|
|
- status
|
|
- type
|
|
type: object
|
|
type: array
|
|
x-kubernetes-list-map-keys:
|
|
- type
|
|
x-kubernetes-list-type: map
|
|
free:
|
|
description: |-
|
|
Free is the number of free units of the class on eligible nodes.
|
|
Absent until the controller has computed it.
|
|
format: int32
|
|
type: integer
|
|
maxFreePerNode:
|
|
description: MaxFreePerNode is the largest number of free units on
|
|
one node.
|
|
format: int32
|
|
type: integer
|
|
type: object
|
|
required:
|
|
- spec
|
|
type: object
|
|
x-kubernetes-validations:
|
|
- message: GPUClass name must be a DNS label, since it becomes part of a quota
|
|
key
|
|
rule: size(self.metadata.name) <= 63 && self.metadata.name.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')
|
|
- message: GPUClass name total is reserved for the gpu.cozystack.io/total
|
|
quota key
|
|
rule: self.metadata.name != 'total'
|
|
served: true
|
|
storage: true
|
|
subresources:
|
|
status: {}
|