Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

nfd-master: Add status for NodeFeatureRule CRD #1901

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

25 changes: 25 additions & 0 deletions api/nfd/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ type NodeFeatureRuleList struct {
// customization of node objects, such as node labeling.
// +kubebuilder:object:root=true
// +kubebuilder:resource:scope=Cluster,shortName=nfr
// +kubebuilder:subresource:status
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +genclient
// +genclient:nonNamespaced
Expand All @@ -131,6 +132,8 @@ type NodeFeatureRule struct {

// Spec defines the rules to be evaluated.
Spec NodeFeatureRuleSpec `json:"spec"`
// +optional
Status NodeFeatureRuleStatus `json:"status,omitempty"`
}

// NodeFeatureRuleSpec describes a NodeFeatureRule.
Expand All @@ -139,6 +142,28 @@ type NodeFeatureRuleSpec struct {
Rules []Rule `json:"rules"`
}

// NodeFeatureRuleStatus represents the status of a NodeFeatureRule
type NodeFeatureRuleStatus struct {
// +optional
Rules []RuleStatus `json:"rules,omitempty"`
}

// RuleStatus contains information on matched rules and nodes
type RuleStatus struct {
Name string `json:"name"`
MatchedNodes []string `json:"matchedNodes"`
}

// NodeFeatureRuleStatusList contains a list of NodeFeatureRuleStatus objects.
// +kubebuilder:object:root=true
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type NodeFeatureRuleStatusList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata"`

Items []NodeFeatureRuleStatus `json:"items"`
}

// NodeFeatureGroup resource holds Node pools by featureGroup
// +kubebuilder:object:root=true
// +kubebuilder:resource:scope=Namespaced,shortName=nfg
Expand Down
78 changes: 78 additions & 0 deletions api/nfd/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 22 additions & 0 deletions deployment/base/nfd-crds/nfd-api-crds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -703,8 +703,30 @@ spec:
required:
- rules
type: object
status:
description: NodeFeatureRuleStatus represents the status of a NodeFeatureRule
properties:
rules:
items:
description: RuleStatus contains information on matched rules and
nodes
properties:
matchedNodes:
items:
type: string
type: array
name:
type: string
required:
- matchedNodes
- name
type: object
type: array
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
22 changes: 22 additions & 0 deletions deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -703,8 +703,30 @@ spec:
required:
- rules
type: object
status:
description: NodeFeatureRuleStatus represents the status of a NodeFeatureRule
properties:
rules:
items:
description: RuleStatus contains information on matched rules and
nodes
properties:
matchedNodes:
items:
type: string
type: array
name:
type: string
required:
- matchedNodes
- name
type: object
type: array
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
2 changes: 2 additions & 0 deletions pkg/apis/nfd/nodefeaturerule/rule.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type RuleOutput struct {
Annotations map[string]string
Vars map[string]string
Taints []corev1.Taint
Matched bool
}

// Execute the rule against a set of input features.
Expand Down Expand Up @@ -103,6 +104,7 @@ func Execute(r *nfdv1alpha1.Rule, features *nfdv1alpha1.Features) (RuleOutput, e
Annotations: maps.Clone(r.Annotations),
ExtendedResources: maps.Clone(r.ExtendedResources),
Taints: slices.Clone(r.Taints),
Matched: true,
}
klog.V(2).InfoS("rule matched", "ruleName", r.Name, "ruleOutput", utils.DelayedDumper(ret))
return ret, nil
Expand Down
95 changes: 90 additions & 5 deletions pkg/nfd-master/nfd-master.go
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,12 @@ func (m *nfdMaster) nfdAPIUpdateAllNodes() error {
m.updaterPool.addNode(node.Name)
}

err = m.updateRuleStatus()
if err != nil {
klog.ErrorS(err, "failed to update rule status")
return err
}

return nil
}

Expand Down Expand Up @@ -878,7 +884,7 @@ func (m *nfdMaster) refreshNodeFeatures(cli k8sclient.Interface, node *corev1.No
labels = make(map[string]string)
}

crLabels, crAnnotations, crExtendedResources, crTaints := m.processNodeFeatureRule(node.Name, features)
crLabels, crAnnotations, crExtendedResources, crTaints, _ := m.processNodeFeatureRule(node.Name, features)

// Labels
maps.Copy(labels, crLabels)
Expand Down Expand Up @@ -989,9 +995,9 @@ func (m *nfdMaster) setTaints(cli k8sclient.Interface, taints []corev1.Taint, no
return nil
}

func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha1.Features) (Labels, Annotations, ExtendedResources, []corev1.Taint) {
func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha1.Features) (Labels, Annotations, ExtendedResources, []corev1.Taint, []*nfdv1alpha1.NodeFeatureRule) {
if m.nfdController == nil {
return nil, nil, nil, nil
return nil, nil, nil, nil, nil
}

extendedResources := ExtendedResources{}
Expand All @@ -1005,12 +1011,13 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha

if err != nil {
klog.ErrorS(err, "failed to list NodeFeatureRule resources")
return nil, nil, nil, nil
return nil, nil, nil, nil, nil
}

// Process all rule CRs
processStart := time.Now()
for _, spec := range ruleSpecs {
spec.Status.Rules = []nfdv1alpha1.RuleStatus{}
t := time.Now()
switch {
case klog.V(3).Enabled():
Expand Down Expand Up @@ -1042,13 +1049,91 @@ func (m *nfdMaster) processNodeFeatureRule(nodeName string, features *nfdv1alpha
// Feed back rule output to features map for subsequent rules to match
features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Vars)

if ruleOut.Matched {
r := nfdv1alpha1.RuleStatus{
Name: rule.Name,
MatchedNodes: []string{
nodeName,
ozhuraki marked this conversation as resolved.
Show resolved Hide resolved
},
}
spec.Status.Rules = append(spec.Status.Rules, r)
}
}
nfrProcessingTime.WithLabelValues(spec.Name, nodeName).Observe(time.Since(t).Seconds())
}
processingTime := time.Since(processStart)
klog.V(2).InfoS("processed NodeFeatureRule objects", "nodeName", nodeName, "objectCount", len(ruleSpecs), "duration", processingTime)

return labels, annotations, extendedResources, taints
return labels, annotations, extendedResources, taints, ruleSpecs
}

func findRuleByName(ruleSpecs []*nfdv1alpha1.NodeFeatureRule, name string) *nfdv1alpha1.NodeFeatureRule {
var spec *nfdv1alpha1.NodeFeatureRule
for _, r := range ruleSpecs {
if r.Name == name {
spec = r
break
}
}
return spec
}

func findStatusRuleByName(status *[]nfdv1alpha1.RuleStatus, name string) *nfdv1alpha1.RuleStatus {
var rule *nfdv1alpha1.RuleStatus
for _, r := range *status {
if r.Name == name {
rule = &r
break
}
}
return rule
}

func (m *nfdMaster) updateRuleStatus() error {
nodes, err := getNodes(m.k8sClient)
if err != nil {
return err
}

var ruleSpecs []*nfdv1alpha1.NodeFeatureRule
var outSpecs []*nfdv1alpha1.NodeFeatureRule

for _, node := range nodes.Items {
nodeFeatures, err := m.getAndMergeNodeFeatures(node.Name)
if err != nil {
return fmt.Errorf("failed to merge NodeFeature objects for node %q: %w", node.Name, err)
}

_, _, _, _, ruleSpecs = m.processNodeFeatureRule(node.Name, &nodeFeatures.Spec.Features)
}

for _, spec := range ruleSpecs {
if len(spec.Status.Rules) > 0 {
s := findRuleByName(outSpecs, spec.Name)
if s != nil {
for _, r := range spec.Status.Rules {
s.Status.Rules = append(s.Status.Rules, r)

sr := findStatusRuleByName(&s.Status.Rules, r.Name)
if sr != nil {
sr.MatchedNodes = append(sr.MatchedNodes, r.MatchedNodes...)
}
}
} else {
outSpecs = append(outSpecs, spec)
}
}
}

for _, spec := range outSpecs {
_, err = m.nfdClient.NfdV1alpha1().NodeFeatureRules().Update(context.TODO(), spec, metav1.UpdateOptions{})
if err != nil {
klog.ErrorS(err, "failed to update rule status", "nodefeaturerule", klog.KObj(spec))
}
}

return nil
}

// updateNodeObject ensures the Kubernetes node object is up to date,
Expand Down