Skip to content

Commit 2799666

Browse files
committed
Support for Load/Attach Split with Initial Cluster-Scoped XDP Programs
This commit introduces the foundation for the load/attach split, including: - Updates to the `BpfApplication` CRD to support a separate list of optional attach points for programs. This allows programs to be loaded before attachments are made and enables dynamic attachment updates. - An initial version of the `BpfApplicationNode` CRD to manage per-node information for a single `BpfApplication`. - Proof of concept and initial implementation for cluster-scoped XDP programs, with working unit tests in the app-agent. Additionally: - Updated existing controllers to work with the new CRD format, though currently limited to XDP programs. - Initial changes in the operator to support the load/attach split, with further cleanup and support for additional program types planned. TODO: - Generalize the agent code to support multiple program types. - Support for namespace-scoped CRDs and other program types beyond XDP. - More clean-up, complete testing, and address remaining edge cases. Signed-off-by: Andre Fredette <[email protected]>
1 parent b7fc471 commit 2799666

File tree

133 files changed

+13579
-5034
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

133 files changed

+13579
-5034
lines changed

Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,8 @@ COMMON_FLAGS ?= ${VERIFY_FLAG} --go-header-file $(shell pwd)/hack/boilerplate.go
208208
.PHONY: manifests
209209
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
210210
$(CONTROLLER_GEN) crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
211-
$(CONTROLLER_GEN) rbac:roleName=agent-role paths="./controllers/bpfman-agent/..." output:rbac:artifacts:config=config/rbac/bpfman-agent
212-
$(CONTROLLER_GEN) rbac:roleName=operator-role paths="./controllers/bpfman-operator" output:rbac:artifacts:config=config/rbac/bpfman-operator
211+
$(CONTROLLER_GEN) rbac:roleName=agent-role paths="./controllers/bpfman-agent/...;./controllers/app-agent/..." output:rbac:artifacts:config=config/rbac/bpfman-agent
212+
$(CONTROLLER_GEN) rbac:roleName=operator-role paths="./controllers/bpfman-operator;./controllers/app-operator" output:rbac:artifacts:config=config/rbac/bpfman-operator
213213

214214
.PHONY: generate
215215
generate: manifests generate-register generate-deepcopy generate-typed-clients generate-typed-listers generate-typed-informers ## Generate ALL auto-generated code.

PROJECT

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,4 +129,11 @@ resources:
129129
kind: BpfNsApplication
130130
path: github.com/bpfman/bpfman-operator/apis/v1alpha1
131131
version: v1alpha1
132+
- api:
133+
crdVersion: v1
134+
controller: true
135+
domain: bpfman.io
136+
kind: BpfApplicationNode
137+
path: github.com/bpfman/bpfman-operator/apis/v1alpha1
138+
version: v1alpha1
132139
version: "3"
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
/*
2+
Copyright 2023 The bpfman Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package v1alpha1
18+
19+
import (
20+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21+
metav1types "k8s.io/apimachinery/pkg/types"
22+
"sigs.k8s.io/controller-runtime/pkg/client"
23+
)
24+
25+
// BpfApplicationProgramNode defines the desired state of BpfApplication
26+
// +union
27+
// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'XDP' ? has(self.xdp) : !has(self.xdp)",message="xdp configuration is required when type is XDP, and forbidden otherwise"
28+
// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'TC' ? has(self.tc) : !has(self.tc)",message="tc configuration is required when type is TC, and forbidden otherwise"
29+
// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'TCX' ? has(self.tcx) : !has(self.tcx)",message="tcx configuration is required when type is TCX, and forbidden otherwise"
30+
// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Fentry' ? has(self.fentry) : !has(self.fentry)",message="fentry configuration is required when type is Fentry, and forbidden otherwise"
31+
// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Fexit' ? has(self.fexit) : !has(self.fexit)",message="fexit configuration is required when type is Fexit, and forbidden otherwise"
32+
// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Kprobe' ? has(self.kprobe) : !has(self.kprobe)",message="kprobe configuration is required when type is Kprobe, and forbidden otherwise"
33+
// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Kretprobe' ? has(self.kretprobe) : !has(self.kretprobe)",message="kretprobe configuration is required when type is Kretprobe, and forbidden otherwise"
34+
// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Uprobe' ? has(self.uprobe) : !has(self.uprobe)",message="uprobe configuration is required when type is Uprobe, and forbidden otherwise"
35+
// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Uretprobe' ? has(self.uretprobe) : !has(self.uretprobe)",message="uretprobe configuration is required when type is Uretprobe, and forbidden otherwise"
36+
// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Tracepoint' ? has(self.tracepoint) : !has(self.tracepoint)",message="tracepoint configuration is required when type is Tracepoint, and forbidden otherwise"
37+
type BpfApplicationProgramNode struct {
38+
// ProgramAttachStatus records whether the program should be loaded and whether
39+
// the program is loaded.
40+
ProgramAttachStatus BpfProgramConditionType `json:"programattachstatus"`
41+
42+
// ProgramId is the id of the program in the kernel. Not set until the
43+
// program is loaded.
44+
// +optional
45+
ProgramId *uint32 `json:"program_id"`
46+
47+
// Type specifies the bpf program type
48+
// +unionDiscriminator
49+
// +kubebuilder:validation:Required
50+
// +kubebuilder:validation:Enum:="XDP";"TC";"TCX";"Fentry";"Fexit";"Kprobe";"Kretprobe";"Uprobe";"Uretprobe";"Tracepoint"
51+
Type EBPFProgType `json:"type,omitempty"`
52+
53+
// xdp defines the desired state of the application's XdpPrograms.
54+
// +unionMember
55+
// +optional
56+
XDP *XdpProgramInfoNode `json:"xdp,omitempty"`
57+
58+
// tc defines the desired state of the application's TcPrograms.
59+
// +unionMember
60+
// +optional
61+
TC *TcProgramInfoNode `json:"tc,omitempty"`
62+
63+
// // tcx defines the desired state of the application's TcxPrograms.
64+
// // +unionMember
65+
// // +optional
66+
// TCX *TcxProgramInfoNode `json:"tcx,omitempty"`
67+
68+
// fentry defines the desired state of the application's FentryPrograms.
69+
// +unionMember
70+
// +optional
71+
Fentry *FentryProgramInfoNode `json:"fentry,omitempty"`
72+
73+
// // fexit defines the desired state of the application's FexitPrograms.
74+
// // +unionMember
75+
// // +optional
76+
// Fexit *FexitProgramInfoNode `json:"fexit,omitempty"`
77+
78+
// // kprobe defines the desired state of the application's KprobePrograms.
79+
// // +unionMember
80+
// // +optional
81+
// Kprobe *KprobeProgramInfoNode `json:"kprobe,omitempty"`
82+
83+
// // kretprobe defines the desired state of the application's KretprobePrograms.
84+
// // +unionMember
85+
// // +optional
86+
// Kretprobe *KprobeProgramInfoNode `json:"kretprobe,omitempty"`
87+
88+
// // uprobe defines the desired state of the application's UprobePrograms.
89+
// // +unionMember
90+
// // +optional
91+
// Uprobe *UprobeProgramInfoNode `json:"uprobe,omitempty"`
92+
93+
// // uretprobe defines the desired state of the application's UretprobePrograms.
94+
// // +unionMember
95+
// // +optional
96+
// Uretprobe *UprobeProgramInfoNode `json:"uretprobe,omitempty"`
97+
98+
// // tracepoint defines the desired state of the application's TracepointPrograms.
99+
// // +unionMember
100+
// // +optional
101+
// Tracepoint *TracepointProgramInfoNode `json:"tracepoint,omitempty"`
102+
}
103+
104+
// BpfApplicationSpec defines the desired state of BpfApplication
105+
type BpfApplicationNodeSpec struct {
106+
// The number of times the BpfApplicationNode has been updated. Set to 1
107+
// when the object is created, then it is incremented prior to each update.
108+
// This allows us to verify that the API server has the updated object prior
109+
// to starting a new Reconcile operation.
110+
UpdateCount int64 `json:"updatecount"`
111+
// AppLoadStatus reflects the status of loading the bpf application on the
112+
// given node.
113+
AppLoadStatus BpfProgramConditionType `json:"apploadstatus"`
114+
// Programs is a list of bpf programs contained in the parent application.
115+
// It is a map from the bpf program name to BpfApplicationProgramNode
116+
// elements.
117+
Programs map[string]BpfApplicationProgramNode `json:"programs,omitempty"`
118+
}
119+
120+
// +genclient
121+
// +genclient:nonNamespaced
122+
// +kubebuilder:object:root=true
123+
// +kubebuilder:subresource:status
124+
// +kubebuilder:resource:scope=Cluster
125+
126+
// BpfApplicationNode is the Schema for the bpfapplications API
127+
// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=".metadata.labels['kubernetes.io/hostname']"
128+
// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[0].reason`
129+
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
130+
type BpfApplicationNode struct {
131+
metav1.TypeMeta `json:",inline"`
132+
metav1.ObjectMeta `json:"metadata,omitempty"`
133+
134+
Spec BpfApplicationNodeSpec `json:"spec,omitempty"`
135+
Status BpfAppStatus `json:"status,omitempty"`
136+
}
137+
138+
// +kubebuilder:object:root=true
139+
// BpfApplicationList contains a list of BpfApplications
140+
type BpfApplicationNodeList struct {
141+
metav1.TypeMeta `json:",inline"`
142+
metav1.ListMeta `json:"metadata,omitempty"`
143+
Items []BpfApplicationNode `json:"items"`
144+
}
145+
146+
func (an BpfApplicationNode) GetName() string {
147+
return an.Name
148+
}
149+
150+
func (an BpfApplicationNode) GetUID() metav1types.UID {
151+
return an.UID
152+
}
153+
154+
func (an BpfApplicationNode) GetAnnotations() map[string]string {
155+
return an.Annotations
156+
}
157+
158+
func (an BpfApplicationNode) GetLabels() map[string]string {
159+
return an.Labels
160+
}
161+
162+
func (an BpfApplicationNode) GetStatus() *BpfAppStatus {
163+
return &an.Status
164+
}
165+
166+
func (an BpfApplicationNode) GetClientObject() client.Object {
167+
return &an
168+
}
169+
170+
func (anl BpfApplicationNodeList) GetItems() []BpfApplicationNode {
171+
return anl.Items
172+
}

apis/v1alpha1/bpfApplication_types.go

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -129,16 +129,11 @@ type BpfApplicationProgram struct {
129129
type BpfApplicationSpec struct {
130130
BpfAppCommon `json:",inline"`
131131

132-
// Programs is a list of bpf programs supported for a specific application.
133-
// It's possible that the application can selectively choose which program(s)
134-
// to run from this list.
135-
// +kubebuilder:validation:MinItems:=1
136-
Programs []BpfApplicationProgram `json:"programs,omitempty"`
137-
}
138-
139-
// BpfApplicationStatus defines the observed state of BpfApplication
140-
type BpfApplicationStatus struct {
141-
BpfProgramStatusCommon `json:",inline"`
132+
// Programs is the list of bpf programs in the BpfApplication that should be
133+
// loaded. The application can selectively choose which program(s) to run
134+
// from this list based on the optional attach points provided. The list is
135+
// implemented as a map from the bpf function name to BpfApplicationProgram.
136+
Programs map[string]BpfApplicationProgram `json:"programs,omitempty"`
142137
}
143138

144139
// +genclient
@@ -155,8 +150,8 @@ type BpfApplication struct {
155150
metav1.TypeMeta `json:",inline"`
156151
metav1.ObjectMeta `json:"metadata,omitempty"`
157152

158-
Spec BpfApplicationSpec `json:"spec,omitempty"`
159-
Status BpfApplicationStatus `json:"status,omitempty"`
153+
Spec BpfApplicationSpec `json:"spec,omitempty"`
154+
Status BpfAppStatus `json:"status,omitempty"`
160155
}
161156

162157
// +kubebuilder:object:root=true

apis/v1alpha1/bpfNsApplication_types.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ type BpfNsApplication struct {
8585
metav1.ObjectMeta `json:"metadata,omitempty"`
8686

8787
Spec BpfNsApplicationSpec `json:"spec,omitempty"`
88-
Status BpfApplicationStatus `json:"status,omitempty"`
88+
Status BpfAppStatus `json:"status,omitempty"`
8989
}
9090

9191
// +kubebuilder:object:root=true

apis/v1alpha1/fentryProgram_types.go

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,8 @@ type FentryProgram struct {
3737
metav1.TypeMeta `json:",inline"`
3838
metav1.ObjectMeta `json:"metadata,omitempty"`
3939

40-
Spec FentryProgramSpec `json:"spec"`
41-
// +optional
42-
Status FentryProgramStatus `json:"status,omitempty"`
40+
Spec FentryProgramSpec `json:"spec"`
41+
Status BpfAppStatus `json:"status,omitempty"`
4342
}
4443

4544
// FentryProgramSpec defines the desired state of FentryProgram
@@ -52,13 +51,19 @@ type FentryProgramSpec struct {
5251
// FentryProgramInfo defines the Fentry program details
5352
type FentryProgramInfo struct {
5453
BpfProgramCommon `json:",inline"`
55-
// Function to attach the fentry to.
56-
FunctionName string `json:"func_name"`
54+
FentryLoadInfo `json:",inline"`
55+
// Whether the program should be attached to the function.
56+
// This may be updated after the program has been loaded.
57+
// +optional
58+
// +kubebuilder:default=false
59+
Attach bool `json:"attach"`
5760
}
5861

59-
// FentryProgramStatus defines the observed state of FentryProgram
60-
type FentryProgramStatus struct {
61-
BpfProgramStatusCommon `json:",inline"`
62+
// FentryLoadInfo contains the program-specific load information for Fentry
63+
// programs
64+
type FentryLoadInfo struct {
65+
// FunctionName is the name of the function to attach the Fentry program to.
66+
FunctionName string `json:"function_name"`
6267
}
6368

6469
// +kubebuilder:object:root=true
@@ -68,3 +73,21 @@ type FentryProgramList struct {
6873
metav1.ListMeta `json:"metadata,omitempty"`
6974
Items []FentryProgram `json:"items"`
7075
}
76+
77+
type FentryProgramInfoNode struct {
78+
// The list of points to which the program should be attached.
79+
// FentryAttachInfoNode is similar to FentryAttachInfo, but the interface and
80+
// container selectors are expanded, and we have one instance of
81+
// FentryAttachInfoNode for each unique attach point. The list is optional and
82+
// may be udated after the bpf program has been loaded.
83+
// +optional
84+
AttachPoint FentryAttachInfoNode `json:"attach_points"`
85+
}
86+
87+
type FentryAttachInfoNode struct {
88+
AttachInfoCommon `json:",inline"`
89+
// An identifier for the attach point assigned by bpfman. This field is
90+
// empty until the program is successfully attached and bpfman returns the
91+
// id.
92+
AttachId *uint32 `json:"attachid"`
93+
}

apis/v1alpha1/fexitProgram_types.go

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,8 @@ type FexitProgram struct {
3737
metav1.TypeMeta `json:",inline"`
3838
metav1.ObjectMeta `json:"metadata,omitempty"`
3939

40-
Spec FexitProgramSpec `json:"spec"`
41-
// +optional
42-
Status FexitProgramStatus `json:"status,omitempty"`
40+
Spec FexitProgramSpec `json:"spec"`
41+
Status BpfAppStatus `json:"status,omitempty"`
4342
}
4443

4544
// FexitProgramSpec defines the desired state of FexitProgram
@@ -52,13 +51,19 @@ type FexitProgramSpec struct {
5251
// FexitProgramInfo defines the Fexit program details
5352
type FexitProgramInfo struct {
5453
BpfProgramCommon `json:",inline"`
55-
// Function to attach the fexit to.
56-
FunctionName string `json:"func_name"`
54+
FexitLoadInfo `json:",inline"`
55+
// Whether the program should be attached to the function.
56+
// This may be updated after the program has been loaded.
57+
// +optional
58+
// +kubebuilder:default=false
59+
Attach bool `json:"attach"`
5760
}
5861

59-
// FexitProgramStatus defines the observed state of FexitProgram
60-
type FexitProgramStatus struct {
61-
BpfProgramStatusCommon `json:",inline"`
62+
// FexitLoadInfo contains the program-specific load information for Fexit
63+
// programs
64+
type FexitLoadInfo struct {
65+
// FunctionName is the name of the function to attach the Fexit program to.
66+
FunctionName string `json:"function_name"`
6267
}
6368

6469
// +kubebuilder:object:root=true

apis/v1alpha1/kprobeProgram_types.go

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,16 +39,14 @@ type KprobeProgram struct {
3939
metav1.TypeMeta `json:",inline"`
4040
metav1.ObjectMeta `json:"metadata,omitempty"`
4141

42-
Spec KprobeProgramSpec `json:"spec"`
43-
// +optional
44-
Status KprobeProgramStatus `json:"status,omitempty"`
42+
Spec KprobeProgramSpec `json:"spec"`
43+
Status BpfAppStatus `json:"status,omitempty"`
4544
}
4645

4746
// KprobeProgramSpec defines the desired state of KprobeProgram
4847
// +kubebuilder:printcolumn:name="FunctionName",type=string,JSONPath=`.spec.func_name`
4948
// +kubebuilder:printcolumn:name="Offset",type=integer,JSONPath=`.spec.offset`
5049
// +kubebuilder:printcolumn:name="RetProbe",type=boolean,JSONPath=`.spec.retprobe`
51-
// +kubebuilder:validation:XValidation:message="offset cannot be set for kretprobes",rule="self.retprobe == false || self.offset == 0"
5250
type KprobeProgramSpec struct {
5351
KprobeProgramInfo `json:",inline"`
5452
BpfAppCommon `json:",inline"`
@@ -57,7 +55,14 @@ type KprobeProgramSpec struct {
5755
// KprobeProgramInfo defines the common fields for KprobeProgram
5856
type KprobeProgramInfo struct {
5957
BpfProgramCommon `json:",inline"`
58+
// The list of points to which the program should be attached. The list is
59+
// optional and may be udated after the bpf program has been loaded
60+
// +optional
61+
AttachPoints []KprobeAttachInfo `json:"attach_points"`
62+
}
6063

64+
// +kubebuilder:validation:XValidation:message="offset cannot be set for kretprobes",rule="self.retprobe == false || self.offset == 0"
65+
type KprobeAttachInfo struct {
6166
// Functions to attach the kprobe to.
6267
FunctionName string `json:"func_name"`
6368

@@ -73,11 +78,6 @@ type KprobeProgramInfo struct {
7378
RetProbe bool `json:"retprobe"`
7479
}
7580

76-
// KprobeProgramStatus defines the observed state of KprobeProgram
77-
type KprobeProgramStatus struct {
78-
BpfProgramStatusCommon `json:",inline"`
79-
}
80-
8181
// +kubebuilder:object:root=true
8282
// KprobeProgramList contains a list of KprobePrograms
8383
type KprobeProgramList struct {

0 commit comments

Comments
 (0)