diff --git a/pkg/controllers/ingress/ingress.go b/pkg/controllers/ingress/ingress.go index ee527fe9..81441f14 100644 --- a/pkg/controllers/ingress/ingress.go +++ b/pkg/controllers/ingress/ingress.go @@ -41,6 +41,7 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" + "github.com/oracle/oci-go-sdk/v65/common" ociloadbalancer "github.com/oracle/oci-go-sdk/v65/loadbalancer" ) @@ -366,11 +367,41 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass * var listenerSslConfig *ociloadbalancer.SslConfigurationDetails artifact, artifactType := stateStore.GetTLSConfigForListener(port) + listenerSslConfig, err := GetSSLConfigForListener(ingress.Namespace, nil, artifactType, artifact, c.defaultCompartmentId, c.client) if err != nil { return err } + // listenerSslConfig.VerifyPeerCertificate + + mode, deepth, trustcacert := stateStore.GetMutualTlsPortConfigForListener(port) + mtlsPorts := stateStore.IngressGroupState.MtlsPorts + klog.Infof(" GetMutualTlsPortConfigForListener ********** mtlsPorts : %s ", util.PrettyPrint(mtlsPorts)) + + klog.Infof(" GetMutualTlsPortConfigForListener ********** before : %s ", util.PrettyPrint(listenerSslConfig)) + if mode == util.MutualTlsAuthenticationVerify { + // listenerSslConfig.VerifyDepth + listenerSslConfig.VerifyPeerCertificate = common.Bool(true) + listenerSslConfig.VerifyDepth = &deepth + + // check wethear the trustcacert is valid ca bundle + if mode == util.MutualTlsAuthenticationVerify && isTrustAuthorityCaBundle(trustcacert) { + caBundleIds := []string{trustcacert} + listenerSslConfig.TrustedCertificateAuthorityIds = caBundleIds + } else { + if mode == util.MutualTlsAuthenticationVerify { + klog.Error("verify trustcacert error, not a valid CA bundle ID: %s", util.PrettyPrint(trustcacert)) + } + listenerSslConfig.VerifyPeerCertificate = common.Bool(false) + listenerSslConfig.VerifyDepth = common.Int(1) + listenerSslConfig.TrustedCertificateAuthorityIds = nil + + } + + } + klog.Infof(" GetMutualTlsPortConfigForListener after : %s ", util.PrettyPrint(listenerSslConfig)) + protocol := stateStore.GetListenerProtocol(port) defaultBackendSet := stateStore.GetListenerDefaultBackendSet(port) err = c.client.GetLbClient().CreateListener(context.TODO(), lbId, int(port), protocol, defaultBackendSet, listenerSslConfig) @@ -482,6 +513,7 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string, needsUpdate := false artifact, artifactType := stateStore.GetTLSConfigForListener(int32(*listener.Port)) + var sslConfig *ociloadbalancer.SslConfigurationDetails if artifact != "" { sslConfig, err = GetSSLConfigForListener(namespace, &listener, artifactType, artifact, c.defaultCompartmentId, c.client) @@ -495,6 +527,41 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string, needsUpdate = true } } + + var port = int32(*listener.Port) + mode, deepth, trustcacert := stateStore.GetMutualTlsPortConfigForListener(port) + mtlsPorts := stateStore.IngressGroupState.MtlsPorts + klog.Infof(" syncListenerr mtlsPorts : %s ", util.PrettyPrint(mtlsPorts)) + + klog.Infof(" syncListener before : %s ", util.PrettyPrint(listener.SslConfiguration)) + var modeToBool bool = false + if mode == util.MutualTlsAuthenticationVerify { + modeToBool = true + } + if *listener.SslConfiguration.VerifyPeerCertificate != modeToBool || *listener.SslConfiguration.VerifyDepth != deepth { + klog.Infof(" mtls port config %d needs update mode: %s", *listener.Name, mode) + needsUpdate = true + + // check wethear the trustcacert is valid ca bundle + if mode == util.MutualTlsAuthenticationVerify && isTrustAuthorityCaBundle(trustcacert) { + listener.SslConfiguration.VerifyPeerCertificate = common.Bool(true) + listener.SslConfiguration.VerifyDepth = &deepth + caBundleIds := []string{trustcacert} + listener.SslConfiguration.TrustedCertificateAuthorityIds = caBundleIds + + } else { + if mode == util.MutualTlsAuthenticationVerify { + klog.Error("verify trustcacert error, not a valid CA bundle ID: %s", util.PrettyPrint(trustcacert)) + } + listener.SslConfiguration.VerifyPeerCertificate = common.Bool(false) + listener.SslConfiguration.VerifyDepth = common.Int(1) + listener.SslConfiguration.TrustedCertificateAuthorityIds = nil + + } + } + + klog.Infof(" syncListener after : %s ", util.PrettyPrint(listener.SslConfiguration)) + } protocol := stateStore.GetListenerProtocol(int32(*listener.Port)) diff --git a/pkg/controllers/ingress/util.go b/pkg/controllers/ingress/util.go index b6cff8cf..61297068 100644 --- a/pkg/controllers/ingress/util.go +++ b/pkg/controllers/ingress/util.go @@ -187,6 +187,8 @@ func CreateCaBundle(certificateName string, compartmentId string, certificatesCl CreateCaBundleDetails: caBundleDetails, OpcRetryToken: &certificateName, } + klog.Infof(" createCaBundleRequest ********** certificateName %s createCaBundleRequest : %s ", certificateName, util.PrettyPrint(createCaBundleRequest)) + createCaBundle, err := certificatesClient.CreateCaBundle(context.TODO(), createCaBundleRequest) if err != nil { return nil, err @@ -373,7 +375,9 @@ func GetSSLConfigForListener(namespace string, listener *ociloadbalancer.Listene } newCertificateId = *cId } - + // //TODO add VerifyPeerCertificate here + // https://github.com/oracle/oci-go-sdk/blob/master/example/example_loadbalancer_test.go + // listenerSslConfig.VerifyPeerCertificate(common.Bool(true)) if newCertificateId != "" { certificateIds := []string{newCertificateId} listenerSslConfig = &ociloadbalancer.SslConfigurationDetails{CertificateIds: certificateIds} diff --git a/pkg/loadbalancer/loadbalancer.go b/pkg/loadbalancer/loadbalancer.go index 6b062b61..bba6bd8a 100644 --- a/pkg/loadbalancer/loadbalancer.go +++ b/pkg/loadbalancer/loadbalancer.go @@ -584,6 +584,10 @@ func (lbc *LoadBalancerClient) UpdateListener(ctx context.Context, lbId *string, if sslConfigurationDetails == nil && l.SslConfiguration != nil { sslConfigurationDetails = &loadbalancer.SslConfigurationDetails{ CertificateIds: l.SslConfiguration.CertificateIds, + //add for mtls verify + VerifyDepth: l.SslConfiguration.VerifyDepth, + VerifyPeerCertificate: l.SslConfiguration.VerifyPeerCertificate, + TrustedCertificateAuthorityIds: l.SslConfiguration.TrustedCertificateAuthorityIds, } } diff --git a/pkg/state/ingressstate.go b/pkg/state/ingressstate.go index 67f5630d..3c77db0f 100644 --- a/pkg/state/ingressstate.go +++ b/pkg/state/ingressstate.go @@ -10,6 +10,7 @@ package state import ( + "encoding/json" "fmt" "reflect" @@ -40,7 +41,12 @@ type TlsConfig struct { Artifact string Type string } - +type MutualTlsPortConfig struct { + Port int32 `json:"port"` + Mode string `json:"mode"` + Depth int `json:"depth,omitempty"` + TrustCACert string `json:"trustcacert"` +} type StateStore struct { IngressClassLister networkinglisters.IngressClassLister IngressLister networkinglisters.IngressLister @@ -58,6 +64,7 @@ type IngressClassState struct { Listeners sets.Int32 ListenerProtocolMap map[int32]string ListenerTLSConfigMap map[int32]TlsConfig + MtlsPorts map[int32]MutualTlsPortConfig ListenerDefaultBsMap map[int32]string } @@ -108,6 +115,12 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { allBackendSets := sets.NewString(util.DefaultBackendSetName) allListeners := sets.NewInt32() + // add mtls verify configmap + mutualTlsPortConfigMap := make(map[int32]MutualTlsPortConfig) + + bsHealthCheckerMap[DefaultIngressName] = util.GetDefaultHeathChecker() + bsPolicyMap[DefaultIngressName] = util.DefaultBackendSetRoutingPolicy + bsHealthCheckerMap[util.DefaultBackendSetName] = util.GetDefaultHeathChecker() bsPolicyMap[util.DefaultBackendSetName] = util.DefaultBackendSetRoutingPolicy @@ -115,6 +128,21 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { hostSecretMap := make(map[string]string) tlsConfiguredHosts := sets.NewString() desiredPorts := sets.NewInt32() + + validateMtlsPortAnnotationJson, err := ParseMutualTlsAnnotationJSON(ing) + klog.Infof("Ingress name: %s, validateMtlsPortAnnotationJson %s", util.PrettyPrint(ing.Name), util.PrettyPrint(validateMtlsPortAnnotationJson)) + + if err != nil { + klog.Infof("Error parsing validateMtlsPortAnnotationJson JSON:", err) + return nil + } + + for _, configPort := range validateMtlsPortAnnotationJson { + // add new mtls port to map + mutualTlsPortConfigMap[configPort.Port] = configPort + } + klog.Infof(" mutualTlsPortConfigMap %s, mutualTlsPortConfigMap %s", util.PrettyPrint(ing.Name), util.PrettyPrint(mutualTlsPortConfigMap)) + // we always expect the default_ingress backendset desiredBackendSets := sets.NewString(util.DefaultBackendSetName) @@ -139,6 +167,10 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { return errors.Wrap(err, "error finding service and port") } + + desiredPorts.Insert(servicePort) + allListeners.Insert(servicePort) + listenerPort, err := util.DetermineListenerPort(ing, &tlsConfiguredHosts, host, servicePort) if err != nil { return errors.Wrap(err, "error determining listener port") @@ -192,6 +224,8 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { Listeners: allListeners, ListenerProtocolMap: listenerProtocolMap, ListenerTLSConfigMap: listenerTLSConfigMap, + //add mutual tls port configmap + MtlsPorts: mutualTlsPortConfigMap, ListenerDefaultBsMap: listenerDefaultBsMap, } @@ -353,6 +387,22 @@ func (s *StateStore) GetTLSConfigForListener(port int32) (string, string) { return "", "" } +// func (s *StateStore) GetMutualTlsPortConfigForListener(port int32) MutualTlsPortConfig { +// portMtlsConfig, ok := s.IngressGroupState.MtlsPorts[port] +// if ok { +// return portMtlsConfig +// } +// return MutualTlsPortConfig{} +// } + +func (s *StateStore) GetMutualTlsPortConfigForListener(port int32) (string, int, string) { + portMtlsConfig, ok := s.IngressGroupState.MtlsPorts[port] + if ok { + return portMtlsConfig.Mode, portMtlsConfig.Depth, portMtlsConfig.TrustCACert + } + return "", 0, "" +} + func (s *StateStore) GetTLSConfigForBackendSet(bsName string) (string, string) { bsTLSConfig, ok := s.IngressGroupState.BackendSetTLSConfigMap[bsName] if ok { @@ -380,3 +430,42 @@ func validatePortInUse(listenerTLSConfig TlsConfig, secretName string, certifica } return nil } + +func ParseMutualTlsAnnotationJSON(ing *networkingv1.Ingress) ([]MutualTlsPortConfig, error) { + + mtlsPortsAnnotation := util.GetMutualTlsVerifyAnnotation(ing) + klog.Infof("Ingress name ParseMutualTlsAnnotationJSON %s, mtlsPortsAnnotation %s", util.PrettyPrint(ing.Name), util.PrettyPrint(mtlsPortsAnnotation)) + + s := mtlsPortsAnnotation + //check if s is empty + if s == "" { + return []MutualTlsPortConfig{}, nil + } + // Check if the string conforms to JSON format + if !isValidJSON(s) { + return nil, fmt.Errorf("Original string does not conform to JSON format") + } + + // Parse JSON string + var mtls []MutualTlsPortConfig + err := json.Unmarshal([]byte(s), &mtls) + if err != nil { + return nil, err + } + + // Remove sub-objects without the 'port' field + var validMtls []MutualTlsPortConfig + for _, config := range mtls { + if config.Port != 0 { + validMtls = append(validMtls, config) + } + } + + return validMtls, nil +} + +// Check if the string conforms to JSON format +func isValidJSON(s string) bool { + var js json.RawMessage + return json.Unmarshal([]byte(s), &js) == nil +} diff --git a/pkg/state/ingressstate_test.go b/pkg/state/ingressstate_test.go index dd49cf9f..f6c59b5f 100644 --- a/pkg/state/ingressstate_test.go +++ b/pkg/state/ingressstate_test.go @@ -36,6 +36,7 @@ const ( TestIngressStateWithPortNameFilePath = "test-ingress-state_withportname.yaml" TestIngressStateWithNamedClassesFilePath = "test-ingress-state_withnamedclasses.yaml" TestSslTerminationAtLb = "test-ssl-termination-lb.yaml" + TestMtlsAuthVerfify = "validate-mutual-tls-authentication.yaml" DefaultBackendSetValidationsFilePath = "validate-default-backend-set.yaml" ) @@ -451,13 +452,35 @@ func TestSslTerminationAtLB(t *testing.T) { Expect(lstTlsConfig.Type).Should(Equal(ArtifactTypeCertificate)) } -func TestValidateListenerDefaultBackendSet(t *testing.T) { + +func TestMtlsAuthVerifyPortConfig(t *testing.T) { RegisterTestingT(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ingressClassList := testutil.GetIngressClassList() + + ingressList := testutil.ReadResourceAsIngressList(TestMtlsAuthVerfify) + + testService := testutil.GetServiceListResource("default", "mtls-auth-verify-annotation", 943) + ingressClassLister, ingressLister, serviceLister := setUp(ctx, ingressClassList, ingressList, testService) + + stateStore := NewStateStore(ingressClassLister, ingressLister, serviceLister, nil) + err := stateStore.BuildState(&ingressClassList.Items[0]) + Expect(err).NotTo(HaveOccurred()) + + mode, deepth, _ := stateStore.GetMutualTlsPortConfigForListener(943) + + Expect(mode).Should(Equal(util.MutualTlsAuthenticationVerify)) + Expect(deepth).Should(Equal(1)) + +} +func TestValidateListenerDefaultBackendSet(t *testing.T) { RegisterTestingT(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() ingressClassList := testutil.GetIngressClassList() + ingressList := testutil.ReadResourceAsIngressList(DefaultBackendSetValidationsFilePath) testService := testutil.GetServiceListResource("default", "tcp-test", 8080) @@ -465,6 +488,7 @@ func TestValidateListenerDefaultBackendSet(t *testing.T) { stateStore := NewStateStore(ingressClassLister, ingressLister, serviceLister, nil) err := stateStore.BuildState(&ingressClassList.Items[0]) + Expect(err).ShouldNot(HaveOccurred()) bsName := util.GenerateBackendSetName("default", "host-es", 8080) diff --git a/pkg/state/validate-mutual-tls-authentication.yaml b/pkg/state/validate-mutual-tls-authentication.yaml new file mode 100644 index 00000000..da62eb55 --- /dev/null +++ b/pkg/state/validate-mutual-tls-authentication.yaml @@ -0,0 +1,24 @@ +# +# OCI Native Ingress Controller +# +# Copyright (c) 2023 Oracle America, Inc. and its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +# +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-mutual-authentication-annotation-one + namespace: default + annotations: + oci-native-ingress.oraclecloud.com/mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 943, "mode": "verify","depth":1 ,"trustCACert" : "ocid1.cabundle.oc1.phx.amaaaaaacuco5yqaiwnnqo54ffsumwoxjxtgwtvyyau3dv7gyeisykfavzta" }]' +spec: + rules: + - http: + paths: + - pathType: Exact + path: "/HCPath" + backend: + service: + name: mtls-auth-verify-annotation + port: + number: 443 diff --git a/pkg/util/util.go b/pkg/util/util.go index b2250e1e..a7bf7aa4 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -53,6 +53,12 @@ const ( IngressListenerTlsCertificateAnnotation = "oci-native-ingress.oraclecloud.com/certificate-ocid" IngressBackendTlsEnabledAnnotation = "oci-native-ingress.oraclecloud.com/backend-tls-enabled" + //verify client certificate , only use the lisentner bound CA certificate to verify the client certificate . + /* + mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 443, "mode": "verify","depth":1 }]' + */ + IngressListenerMutualTlsVerifyAnnotation = "oci-native-ingress.oraclecloud.com/mutual-tls-authentication" + // IngressProtocolAnntoation - HTTP only for now // HTTP, HTTP2, TCP - accepted. IngressProtocolAnnotation = "oci-native-ingress.oraclecloud.com/protocol" @@ -90,6 +96,8 @@ const ( CertificateCacheMaxAgeInMinutes = 10 LBCacheMaxAgeInMinutes = 1 WAFCacheMaxAgeInMinutes = 5 + + MutualTlsAuthenticationVerify = "verify" ) var ErrIngressClassNotReady = errors.New("ingress class not ready") @@ -642,6 +650,15 @@ func RetrievePods(endpointLister corelisters.EndpointsLister, podLister corelist return pods, nil } + +func GetMutualTlsVerifyAnnotation(i *networkingv1.Ingress) string { + mtlsVerifyPorts, ok := i.Annotations[IngressListenerMutualTlsVerifyAnnotation] + if !ok { + return "" + } + + return strings.ToLower(mtlsVerifyPorts) +} func DetermineListenerPort(ingress *networkingv1.Ingress, tlsConfiguredHosts *sets.String, host string, servicePort int32) (int32, error) { annotatedHttpPort, err := GetIngressHttpListenerPort(ingress) if err != nil {