Kubernetes Operator for Nessie

Author: adutra

gradle/libs.versions.toml

@@ -149,6 +149,10 @@ quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quark
 quarkus-cassandra-bom = { module = "io.quarkus.platform:quarkus-cassandra-bom", version.ref = "quarkus" }
 quarkus-google-cloud-services-bom = { module = "io.quarkus.platform:quarkus-google-cloud-services-bom", version.ref = "quarkus" }
 quarkus-logging-sentry = { module = "io.quarkiverse.loggingsentry:quarkus-logging-sentry", version = "2.0.5" }
+# references io.quarkiverse.operatorsdk:quarkus-operator-sdk:6.4.0, too old
+# TODO switch when quarkus 3.7 is released
+# quarkus-operator-sdk-bom = { module = "io.quarkus.platform:quarkus-operator-sdk-bom", version.ref = "quarkus" }
+quarkus-operator-sdk-bom = { module = "io.quarkiverse.operatorsdk:quarkus-operator-sdk-bom", version = "6.6.0" }
 rest-assured = { module = "io.rest-assured:rest-assured", version = "5.4.0" }
 rocksdb-jni = { module = "org.rocksdb:rocksdbjni", version = "8.10.0" }
 scala-library-v212 = { module = "org.scala-lang:scala-library", version = { strictly = "[2.12, 2.13[", prefer = "2.12.18" }}

gradle/projects.main.properties

@@ -21,6 +21,7 @@ nessie-jaxrs-testextension=servers/jax-rs-testextension
 nessie-jaxrs-tests=servers/jax-rs-tests
 nessie-keycloak-testcontainer=testing/keycloak-container
 nessie-nessie-testcontainer=testing/nessie-container
+nessie-operator=operator
 nessie-quarkus-auth=servers/quarkus-auth
 nessie-quarkus-cli=servers/quarkus-cli
 nessie-quarkus-common=servers/quarkus-common

operator/Makefile

@@ -0,0 +1,100 @@
+
+VERSION ?= 0.0.1
+
+# CHANNELS define the bundle channels used in the bundle.
+# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
+# To re-generate a bundle for other specific channels without changing the standard setup, you can:
+# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable)
+# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable")
+ifneq ($(origin CHANNELS), undefined)
+BUNDLE_CHANNELS := --channels=$(CHANNELS)
+endif
+
+# DEFAULT_CHANNEL defines the default channel used in the bundle.
+# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable")
+# To re-generate a bundle for any other default channel without changing the default setup, you can:
+# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable)
+# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable")
+ifneq ($(origin DEFAULT_CHANNEL), undefined)
+BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
+endif
+BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
+
+# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images.
+# This variable is used to construct full image tags for bundle and catalog images.
+#
+# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both
+# projectnessie.org/nessie-operator-bundle:$VERSION and projectnessie.org/nessie-operator-catalog:$VERSION.
+IMAGE_TAG_BASE ?= ghcr.io/projectnessie/nessie-operator
+
+# BUNDLE_IMG defines the image:tag used for the bundle.
+# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
+BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
+
+# Image URL to use all building/pushing image targets
+IMG ?= $(IMAGE_TAG_BASE):latest
+
+PULL_POLICY ?= Always
+PLATFORM ?= linux/amd64
+
+all: docker-build
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Build
+
+docker-build: ## Build docker image with the manager.
+	../gradlew --no-build-cache :nessie-operator:spotlessApply :nessie-operator:clean :nessie-operator:build -x check \
+		-Dquarkus.container-image.image=${IMG} \
+		-Dquarkus.jib.platforms=${PLATFORM} \
+		-Dquarkus.kubernetes.image-pull-policy=${PULL_POLICY}
+
+docker-push: ## Build and push docker image with the manager.
+	../gradlew --no-build-cache :nessie-operator:spotlessApply :nessie-operator:clean :nessie-operator:build -x check \
+		-Dquarkus.container-image.push=true \
+		-Dquarkus.container-image.image=${IMG} \
+		-Dquarkus.jib.platforms=${PLATFORM} \
+		-Dquarkus.kubernetes.image-pull-policy=${PULL_POLICY}
+
+##@ Deployment
+
+install: ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	@$(foreach file, $(wildcard build/kubernetes/*-v1.yml), kubectl apply -f $(file);)
+
+uninstall: ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
+	@$(foreach file, $(wildcard build/kubernetes/*-v1.yml), kubectl delete -f $(file);)
+
+deploy: ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	kubectl apply -f build/kubernetes/kubernetes.yml
+
+undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
+	kubectl delete -f build/kubernetes/kubernetes.yml
+
+##@Bundle
+.PHONY: bundle
+bundle:  ## Generate bundle manifests and metadata, then validate generated files.
+## marker
+	cat build/kubernetes/nessies.nessie.projectnessie.org-v1alpha1.yml target/kubernetes/kubernetes.yml | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)
+	operator-sdk bundle validate ./bundle
+
+.PHONY: bundle-build
+bundle-build: ## Build the bundle image.
+	docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) .
+
+.PHONY: bundle-push
+bundle-push: ## Push the bundle image.
+	docker push $(BUNDLE_IMG)

operator/PROJECT

@@ -0,0 +1,16 @@
+# Code generated by tool. DO NOT EDIT.
+# This file is used to track the info used to scaffold your project
+# and allow the plugins properly work.
+# More info: https://book.kubebuilder.io/reference/project-config.html
+domain: projectnessie.org
+layout:
+- quarkus.javaoperatorsdk.io/v1-alpha
+projectName: nessie-operator
+resources:
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: projectnessie.org
+  kind: Nessie
+  version: v1alpha1
+version: "3"

operator/README.md

@@ -0,0 +1,110 @@
+# Kubernetes Operator for Nessie
+
+## Overview
+
+This module is a Kubernetes Operator for Nessie.
+
+**WARNING: This is a work in progress and is not ready for production use.**
+
+## Usage
+
+TODO describe typical deployment scenarios.
+
+## Development
+
+### TODOS
+
+- [ ] CRD spec: move some configs to a config map ?
+- [ ] CRD status: conditions, etc.
+- [ ] CRD deletion and cleanup
+- [ ] CRD validation
+  - [ ] Admission webhook ? https://github.com/operator-framework/josdk-webhooks
+- [ ] Operator release
+  - [ ] Operator Lifecycle Manager (OLM)
+  - [ ] Helm chart
+- [ ] Enable operator remote debugging
+- [ ] Tests
+  - [ ] Unit tests
+    - [ ] Smoke test
+    - [ ] Missing / invalid configs
+  - [ ] Integration tests: 
+    - [ ] All version stores
+    - [ ] https://github.com/java-operator-sdk/jenvtest ?
+    - Examples: https://github.com/operator-framework/java-operator-sdk/tree/main/operator-framework/src/test/java/io/javaoperatorsdk/operator
+
+### Known issues
+
+- [ ] Including a PVC in the reconcile workflow keeps triggering the reconcile loop
+- [ ] Check generated RBAC rules
+- [ ] Helm chart has incorrect kubernetes.yml template when watching all namespaces
+  - https://github.com/quarkiverse/quarkus-operator-sdk/pull/812
+
+### Prerequisites
+
+- Operator SDK: https://sdk.operatorframework.io/docs/installation/
+- Optional: 
+  - yq: https://github.com/mikefarah/yq
+
+### Scaffolding the project
+
+The initial scaffold was generated using [Operator SDK], with the Quarkus plugin:
+
+```bash
+operator-sdk init --plugins=quarkus --domain=projectnessie.org --project-name=nessie-operator
+operator-sdk create api --plugins=quarkus --version=v1alpha1 --kind=Nessie
+```
+
+[Operator SDK]:https://sdk.operatorframework.io/docs/cli/operator-sdk/
+
+### Building the operator image
+
+```bash
+make docker-build
+```
+
+### Adhoc testing with Minikube
+
+Install [minikube](https://minikube.sigs.k8s.io/docs/start/).
+
+If you need ingress, install the ingress addon:
+
+```bash
+minikube addons enable ingress
+minikube tunnel
+```
+
+Create the nessie-operator and nessie-ns namespaces (only needed once):
+
+```bash
+kubectl create namespace nessie-operator
+kubectl create namespace nessie-ns
+```
+
+Grant admin rights to the nessie-operator service account (only needed once):
+
+```bash
+kubectl apply -f examples/nessie-operator-rbac.yaml
+```
+
+Build the operator docker image _inside_ minikube to facilitate testing (doesn't need a registry):
+
+```bash
+eval $(minikube docker-env)
+make docker-build PULL_POLICY=IfNotPresent
+```
+
+Note: the `PULL_POLICY=IfNotPresent` is required to avoid pulling the image from a registry.
+
+Install the CRDs and deploy the operator in the nessie-operator namespace:
+
+```bash
+make install deploy
+```
+
+Create a Nessie resource in the nessie-ns namespace:
+
+```bash
+kubectl apply -n nessie-ns -f examples/nessie-simple.yaml
+```
+
+You should see 1 pod, 1 deployment and 1 service running.

operator/build.gradle.kts

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.tools.ant.filters.ReplaceTokens
+
+plugins {
+  alias(libs.plugins.quarkus)
+  id("nessie-conventions-quarkus")
+  id("nessie-jacoco")
+}
+
+extra["maven.name"] = "Nessie - Kubernetes Operator"
+
+dependencies {
+  implementation(enforcedPlatform(libs.quarkus.bom))
+  implementation(enforcedPlatform(libs.quarkus.operator.sdk.bom))
+
+  implementation("io.quarkiverse.operatorsdk:quarkus-operator-sdk")
+  implementation("io.quarkiverse.operatorsdk:quarkus-operator-sdk-bundle-generator")
+  implementation("io.quarkus:quarkus-micrometer-registry-prometheus")
+  implementation("io.quarkus:quarkus-container-image-jib")
+
+  compileOnly("io.sundr:builder-annotations:0.103.0")
+  compileOnly("io.fabric8:generator-annotations:6.10.0")
+
+  annotationProcessor(enforcedPlatform(libs.quarkus.bom))
+  annotationProcessor("io.sundr:builder-annotations:0.103.0")
+  // see https://github.com/sundrio/sundrio/issues/104
+  annotationProcessor("io.fabric8:kubernetes-client")
+
+  testImplementation(enforcedPlatform(libs.quarkus.bom))
+  testImplementation("io.quarkus:quarkus-junit5")
+  testImplementation("io.quarkus:quarkus-junit5-mockito")
+  testImplementation("io.quarkus:quarkus-test-kubernetes-client")
+  testImplementation("org.bouncycastle:bcpkix-jdk18on")
+  testImplementation(platform(libs.junit.bom))
+  testImplementation(libs.bundles.junit.testing)
+  testImplementation(libs.awaitility)
+}
+
+listOf("javadoc", "sourcesJar").forEach { name ->
+  tasks.named(name).configure { dependsOn(tasks.named("compileQuarkusGeneratedSourcesJava")) }
+}
+
+listOf("checkstyleTest", "compileTestJava").forEach { name ->
+  tasks.named(name).configure { dependsOn(tasks.named("compileQuarkusTestGeneratedSourcesJava")) }
+}
+
+tasks.named<ProcessResources>("processTestResources").configure {
+  inputs.property("projectVersion", project.version)
+  filter(ReplaceTokens::class, mapOf("tokens" to mapOf("projectVersion" to project.version)))
+}

operator/examples/nessie-autoscaling.yaml

@@ -0,0 +1,19 @@
+apiVersion: nessie.projectnessie.org/v1alpha1
+kind: Nessie
+metadata:
+  name: nessie-autoscaling
+spec:
+  size: 1
+  logLevel: INFO
+  image:
+    repository: projectnessie/nessie
+    tag: 0.75.0
+  versionStore:
+    type: Jdbc
+    jdbc:
+      jdbcUrl: jdbc:h2:mem:nessie
+  autoscaling:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 3
+    targetCpuUtilizationPercentage: 50

operator/examples/nessie-inmemory.yaml

@@ -0,0 +1,43 @@
+apiVersion: nessie.projectnessie.org/v1alpha1
+kind: Nessie
+metadata:
+  name: nessie-inmemory
+spec:
+  size: 1
+  logLevel: INFO
+  image:
+    repository: projectnessie/nessie
+    tag: 0.75.0
+  serviceAccount:
+    create: true
+    name: nessie-inmemory-sa
+  versionStore:
+    type: InMemory
+  authentication:
+    enabled: true
+    oidcAuthServerUrl: http://localhost:8080/auth/realms/nessie
+    oidcClientId: quarkus-app
+  authorization:
+    enabled: true
+    rules:
+      allowViewingBranch: op=='VIEW_REFERENCE' && role.startsWith('test_user') && ref.startsWith('allowedBranch')
+  telemetry:
+    enabled: true
+    endpoint: http://localhost:14268/api/traces
+    sample: "1.0"
+    attributes:
+        foo: "bar"
+  extraEnv:
+    - name: NESSIE_QUARKUS_PROFILE
+      value: "prod"
+  advancedConfig:
+    nessie.version.store.persist.cache-capacity-mb: 1024
+    nessie.version.store.persist.cache-capacity-fraction-of-heap: 0.7
+    nessie.version.store.persist.cache-capacity-fraction-adjust-mb: 256
+    nessie.version.store.persist.cache-capacity-fraction-min-size-mb: 64
+    nessie.server.default-branch: my-branch
+    nessie.version.store.persist.repository-id: my-repository
+    quarkus:
+      log:
+        console.format: "%d{HH:mm:ss} %s%e%n"
+        category."org.projectnessie".level: "DEBUG"