Skip to content

Commit 4dc695c

Browse files
yetoneyetone
committed
feat: support bento image build engines: buildkit and buildkit-rootless
1 parent 171ee6d commit 4dc695c

6 files changed

Lines changed: 118 additions & 12 deletions

File tree

‎controllers/resources/bentorequest_controller.go‎

Lines changed: 104 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"k8s.io/apimachinery/pkg/runtime"
3636
"k8s.io/apimachinery/pkg/types"
3737
"k8s.io/client-go/tools/record"
38+
"k8s.io/utils/pointer"
3839
ctrl "sigs.k8s.io/controller-runtime"
3940
"sigs.k8s.io/controller-runtime/pkg/client"
4041
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -618,6 +619,26 @@ func hash(text string) string {
618619
return hex.EncodeToString(hasher.Sum(nil))
619620
}
620621

622+
type BentoImageBuildEngine string
623+
624+
const (
625+
BentoImageBuildEngineKaniko BentoImageBuildEngine = "kaniko"
626+
BentoImageBuildEngineBuildkit BentoImageBuildEngine = "buildkit"
627+
BentoImageBuildEngineBuildkitRootless BentoImageBuildEngine = "buildkit-rootless"
628+
)
629+
630+
const (
631+
EnvBentoImageBuildEngine = "BENTO_IMAGE_BUILD_ENGINE"
632+
)
633+
634+
func getBentoImageBuildEngine() BentoImageBuildEngine {
635+
engine := os.Getenv(EnvBentoImageBuildEngine)
636+
if engine == "" {
637+
return BentoImageBuildEngineKaniko
638+
}
639+
return BentoImageBuildEngine(engine)
640+
}
641+
621642
func MakeSureDockerConfigJSONSecret(ctx context.Context, kubeCli *kubernetes.Clientset, namespace string, dockerRegistryConf *commonconfig.DockerRegistryConfig) (dockerConfigJSONSecret *corev1.Secret, err error) {
622643
if dockerRegistryConf.Username == "" {
623644
return
@@ -1097,6 +1118,10 @@ func (r *BentoRequestReconciler) generateImageBuilderPod(ctx context.Context, op
10971118
internalImages := commonconfig.GetInternalImages()
10981119
logrus.Infof("Image builder is using the images %v", *internalImages)
10991120

1121+
buildEngine := getBentoImageBuildEngine()
1122+
1123+
privileged := buildEngine != BentoImageBuildEngineBuildkitRootless
1124+
11001125
bentoDownloadCommandTemplate, err := template.New("downloadCommand").Parse(`
11011126
set -e
11021127
@@ -1114,6 +1139,10 @@ echo "Extracting bento tar file..."
11141139
tar -xvf /tmp/downloaded.tar
11151140
echo "Removing bento tar file..."
11161141
rm /tmp/downloaded.tar
1142+
{{if not .Privileged}}
1143+
echo "Changing directory permission..."
1144+
chown -R 1000:1000 /workspace
1145+
{{end}}
11171146
echo "Done"
11181147
`)
11191148

@@ -1129,6 +1158,7 @@ echo "Done"
11291158
"BentoDownloadHeader": bentoDownloadHeader,
11301159
"BentoRepositoryName": bentoRepositoryName,
11311160
"BentoVersion": bentoVersion,
1161+
"Privileged": privileged,
11321162
})
11331163
if err != nil {
11341164
err = errors.Wrap(err, "failed to execute download command template")
@@ -1272,6 +1302,10 @@ tar -xvf /tmp/downloaded.tar
12721302
echo -n '{{.ModelVersion}}' > {{.ModelRepositoryDirPath}}/latest
12731303
echo "Removing model tar file..."
12741304
rm /tmp/downloaded.tar
1305+
{{if not .Privileged}}
1306+
echo "Changing directory permission..."
1307+
chown -R 1000:1000 /workspace
1308+
{{end}}
12751309
echo "Done"
12761310
`)).Execute(&modelDownloadCommandOutput, map[string]interface{}{
12771311
"ModelDirPath": modelDirPath,
@@ -1280,6 +1314,7 @@ echo "Done"
12801314
"ModelRepositoryDirPath": modelRepositoryDirPath,
12811315
"ModelRepositoryName": modelRepositoryName,
12821316
"ModelVersion": modelVersion,
1317+
"Privileged": privileged,
12831318
})
12841319
if err != nil {
12851320
err = errors.Wrap(err, "failed to generate download command")
@@ -1400,6 +1435,14 @@ echo "Done"
14001435
})
14011436
}
14021437

1438+
if !privileged {
1439+
envs = append(envs, corev1.EnvVar{
1440+
Name: "BUILDKITD_FLAGS",
1441+
Value: "--oci-worker-no-process-sandbox",
1442+
})
1443+
}
1444+
1445+
kubeAnnotations := make(map[string]string)
14031446
var command []string
14041447
args := []string{
14051448
"--context=/workspace/buildcontext",
@@ -1408,10 +1451,60 @@ echo "Done"
14081451
fmt.Sprintf("--insecure=%v", dockerRegistryInsecure),
14091452
fmt.Sprintf("--destination=%s", inClusterImageName),
14101453
}
1454+
var builderImage string
1455+
switch buildEngine {
1456+
case BentoImageBuildEngineKaniko:
1457+
builderImage = internalImages.Kaniko
1458+
case BentoImageBuildEngineBuildkit:
1459+
builderImage = internalImages.Buildkit
1460+
case BentoImageBuildEngineBuildkitRootless:
1461+
builderImage = internalImages.BuildkitRootless
1462+
default:
1463+
err = errors.Errorf("unknown bento image build engine %s", buildEngine)
1464+
return
1465+
}
1466+
1467+
isBuildkit := buildEngine == BentoImageBuildEngineBuildkit || buildEngine == BentoImageBuildEngineBuildkitRootless
1468+
1469+
if isBuildkit {
1470+
command = []string{"buildctl-daemonless.sh"}
1471+
args = []string{
1472+
"build",
1473+
"--frontend",
1474+
"dockerfile.v0",
1475+
"--local",
1476+
"context=/workspace/buildcontext",
1477+
"--local",
1478+
fmt.Sprintf("dockerfile=%s", filepath.Dir(dockerFilePath)),
1479+
"--output",
1480+
fmt.Sprintf("type=image,name=%s,push=true,registry.insecure=%v", inClusterImageName, dockerRegistryInsecure),
1481+
}
1482+
}
1483+
1484+
var builderContainerSecurityContext *corev1.SecurityContext
1485+
1486+
if buildEngine == BentoImageBuildEngineBuildkit {
1487+
builderContainerSecurityContext = &corev1.SecurityContext{
1488+
Privileged: pointer.BoolPtr(true),
1489+
}
1490+
} else if buildEngine == BentoImageBuildEngineBuildkitRootless {
1491+
kubeAnnotations["container.apparmor.security.beta.kubernetes.io/builder"] = "unconfined"
1492+
builderContainerSecurityContext = &corev1.SecurityContext{
1493+
SeccompProfile: &corev1.SeccompProfile{
1494+
Type: corev1.SeccompProfileTypeUnconfined,
1495+
},
1496+
RunAsUser: pointer.Int64Ptr(1000),
1497+
RunAsGroup: pointer.Int64Ptr(1000),
1498+
}
1499+
}
14111500

14121501
// add build args to pass via --build-arg
14131502
for _, buildArg := range buildArgs {
1414-
args = append(args, fmt.Sprintf("--build-arg=%s", buildArg))
1503+
if isBuildkit {
1504+
args = append(args, "--opt", fmt.Sprintf("build-arg:%s", buildArg))
1505+
} else {
1506+
args = append(args, fmt.Sprintf("--build-arg=%s", buildArg))
1507+
}
14151508
}
14161509
// add other arguments to builder
14171510
args = append(args, builderArgs...)
@@ -1480,14 +1573,16 @@ echo "Done"
14801573
},
14811574
})
14821575

1483-
args = append(args, fmt.Sprintf("--build-arg=%s=$(%s)", key, envName))
1576+
if isBuildkit {
1577+
args = append(args, "--opt", fmt.Sprintf("build-arg:%s=$(%s)", key, envName))
1578+
} else {
1579+
args = append(args, fmt.Sprintf("--build-arg=%s=$(%s)", key, envName))
1580+
}
14841581
}
14851582
} else {
14861583
r.Recorder.Eventf(opt.BentoRequest, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Secret %s is not found in namespace %s", buildArgsSecretName, configNamespace)
14871584
}
14881585

1489-
builderImage := internalImages.Kaniko
1490-
14911586
container := corev1.Container{
14921587
Name: "builder",
14931588
Image: builderImage,
@@ -1498,6 +1593,7 @@ echo "Done"
14981593
Env: envs,
14991594
TTY: true,
15001595
Stdin: true,
1596+
SecurityContext: builderContainerSecurityContext,
15011597
}
15021598

15031599
if opt.BentoRequest.Spec.ImageBuilderContainerResources != nil {
@@ -1506,9 +1602,10 @@ echo "Done"
15061602

15071603
pod = &corev1.Pod{
15081604
ObjectMeta: metav1.ObjectMeta{
1509-
Name: kubeName,
1510-
Namespace: opt.BentoRequest.Namespace,
1511-
Labels: kubeLabels,
1605+
Name: kubeName,
1606+
Namespace: opt.BentoRequest.Namespace,
1607+
Labels: kubeLabels,
1608+
Annotations: kubeAnnotations,
15121609
},
15131610
Spec: corev1.PodSpec{
15141611
RestartPolicy: corev1.RestartPolicyNever,

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.18
44

55
require (
66
github.com/aws/aws-sdk-go v1.44.152
7-
github.com/bentoml/yatai-common v0.0.0-20221204065908-975ef2e3714e
7+
github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1
88
github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451
99
github.com/huandu/xstrings v1.3.2
1010
github.com/iancoleman/strcase v0.2.0
@@ -98,7 +98,7 @@ require (
9898
k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect
9999
k8s.io/klog/v2 v2.70.1 // indirect
100100
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
101-
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
101+
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
102102
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
103103
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
104104
sigs.k8s.io/yaml v1.3.0 // indirect

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ github.com/aws/aws-sdk-go v1.44.152 h1:L9aaepO8wHB67gwuGD8VgIYH/cmQDxieCt7FeLa0+
7777
github.com/aws/aws-sdk-go v1.44.152/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
7878
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
7979
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
80-
github.com/bentoml/yatai-common v0.0.0-20221204065908-975ef2e3714e h1:et8vRFRG2CgnFUElqpXVrerB1DkV2DcpmFWpzAGAvqo=
81-
github.com/bentoml/yatai-common v0.0.0-20221204065908-975ef2e3714e/go.mod h1:pox0XYk/bVUwKkadn0XwWHEbJmxSEeN3+HwGA4a8uOQ=
80+
github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1 h1:VgN2DLopHoMaEVOiD8J4bZO0L1BGXLRSmNSs7GkfbHo=
81+
github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1/go.mod h1:pox0XYk/bVUwKkadn0XwWHEbJmxSEeN3+HwGA4a8uOQ=
8282
github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451 h1:FNxCbN61Ev8ea6BXzlfmRUT5CYNmqlOv8zDRGs8ufVE=
8383
github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451/go.mod h1:q7tt064G8YIiAwQabKyVaKEdSIHYDQA9Oyt+kyCsflU=
8484
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

‎helm/yatai-image-builder/templates/secret-env.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,12 @@ stringData:
3030

3131
INTERNAL_IMAGES_BENTO_DOWNLOADER: {{ .Values.internalImages.bentoDownloader | quote }}
3232
INTERNAL_IMAGES_KANIKO: {{ .Values.internalImages.kaniko | quote }}
33+
INTERNAL_IMAGES_BUILDKIT: {{ .Values.internalImages.buildkit | quote }}
34+
INTERNAL_IMAGES_BUILDKIT_ROOTLESS: {{ .Values.internalImages.buildkitRootless | quote }}
3335

3436
{{- if .Values.dockerRegistry.useAWSECRWithIAMRole }}
3537
AWS_ECR_WITH_IAM_ROLE: "true"
3638
AWS_ECR_REGION: {{ .Values.dockerRegistry.awsECRRegion | quote }}
3739
{{- end }}
40+
41+
BENTO_IMAGE_BUILD_ENGINE: {{ .Values.bentoImageBuildEngine | quote }}

‎helm/yatai-image-builder/values.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,7 @@ aws:
9898
internalImages:
9999
bentoDownloader: quay.io/bentoml/bento-downloader:0.0.1
100100
kaniko: quay.io/bentoml/kaniko:1.9.1
101+
buildkit: quay.io/bentoml/buildkit:master
102+
buildkitRootless: quay.io/bentoml/buildkit:master-rootless
103+
104+
bentoImageBuildEngine: kaniko # options: kaniko, buildkit, buildkit-rootless

‎scripts/quick-install-yatai-image-builder.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,8 @@ if [ "${USE_LOCAL_HELM_CHART}" = "true" ]; then
225225
--set dockerRegistry.bentoRepositoryName=${DOCKER_REGISTRY_BENTO_REPOSITORY_NAME} \
226226
--set aws.accessKeyID=${AWS_ACCESS_KEY_ID} \
227227
--set aws.secretAccessKeyExistingSecretName=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME} \
228-
--set aws.secretAccessKeyExistingSecretKey=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY}
228+
--set aws.secretAccessKeyExistingSecretKey=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY} \
229+
--set bentoImageBuildEngine=buildkit-rootless
229230
else
230231
helm_repo_name=bentoml
231232
helm_repo_url=https://bentoml.github.io/helm-charts

0 commit comments

Comments
 (0)