From b774eab4141ba61a71bdb929d0c4616e07d7f3df Mon Sep 17 00:00:00 2001 From: george Date: Fri, 19 Apr 2024 19:33:20 +0800 Subject: [PATCH 1/6] mtls feature for OKE native ingress controller --- pkg/controllers/ingress/ingress.go | 29 +++++++ pkg/controllers/ingress/util.go | 4 +- pkg/state/ingressstate.go | 80 ++++++++++++++++++- .../validate-mutual-tls-authentication.yaml | 24 ++++++ pkg/util/util.go | 15 ++++ 5 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 pkg/state/validate-mutual-tls-authentication.yaml diff --git a/pkg/controllers/ingress/ingress.go b/pkg/controllers/ingress/ingress.go index 63d6f1e3..aad20097 100644 --- a/pkg/controllers/ingress/ingress.go +++ b/pkg/controllers/ingress/ingress.go @@ -41,6 +41,7 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" + "github.com/oracle/oci-go-sdk/v65/common" ociloadbalancer "github.com/oracle/oci-go-sdk/v65/loadbalancer" ) @@ -366,11 +367,34 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass * var listenerSslConfig *ociloadbalancer.SslConfigurationDetails artifact, artifactType := stateStore.GetTLSConfigForListener(port) + listenerSslConfig, err := GetSSLConfigForListener(ingress.Namespace, nil, artifactType, artifact, c.defaultCompartmentId, c.client) if err != nil { return err } + // listenerSslConfig.VerifyPeerCertificate + + mode, deepth := stateStore.GetMutualTlsPortConfigForListener(port) + mtlsPorts := stateStore.IngressGroupState.MtlsPorts + klog.Infof(" GetMutualTlsPortConfigForListener ********** GetMutualTlsPortConfigForListener mtlsPorts : %s ", mtlsPorts) + + if mode == "verify" { + // listenerSslConfig.VerifyDepth + listenerSslConfig.VerifyPeerCertificate = common.Bool(true) + listenerSslConfig.VerifyDepth = &deepth + caBundleId, _ := CreateOrGetCaBundleForBackendSet(ingress.Namespace, artifact, c.defaultCompartmentId, c.client) + if err != nil { + klog.Infof(" CreateOrGetCaBundleForBackendSet ********** CreateOrGetCaBundleForBackendSet port : %s ", port) + return err + } + if caBundleId != nil { + caBundleIds := []string{*caBundleId} + listenerSslConfig.TrustedCertificateAuthorityIds = caBundleIds + } + + } + protocol := stateStore.GetListenerProtocol(port) err = c.client.GetLbClient().CreateListener(context.TODO(), lbId, int(port), protocol, listenerSslConfig) if err != nil { @@ -480,6 +504,11 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string, needsUpdate := false artifact, artifactType := stateStore.GetTLSConfigForListener(int32(*listener.Port)) + // c. + // mode, _ := stateStore.GetMutualTlsPortConfigForListener(int32(*listener.Port)) + // if mode == "verify" { + // // 执行相关操作 + // } var sslConfig *ociloadbalancer.SslConfigurationDetails if artifact != "" { sslConfig, err = GetSSLConfigForListener(namespace, &listener, artifactType, artifact, c.defaultCompartmentId, c.client) diff --git a/pkg/controllers/ingress/util.go b/pkg/controllers/ingress/util.go index e95a0cd1..def83675 100644 --- a/pkg/controllers/ingress/util.go +++ b/pkg/controllers/ingress/util.go @@ -337,7 +337,9 @@ func GetSSLConfigForListener(namespace string, listener *ociloadbalancer.Listene } newCertificateId = *cId } - + // //TODO add VerifyPeerCertificate here + // https://github.com/oracle/oci-go-sdk/blob/master/example/example_loadbalancer_test.go + // listenerSslConfig.VerifyPeerCertificate(common.Bool(true)) if newCertificateId != "" { certificateIds := []string{newCertificateId} listenerSslConfig = &ociloadbalancer.SslConfigurationDetails{CertificateIds: certificateIds} diff --git a/pkg/state/ingressstate.go b/pkg/state/ingressstate.go index 4a48f7d3..c66d129a 100644 --- a/pkg/state/ingressstate.go +++ b/pkg/state/ingressstate.go @@ -10,6 +10,7 @@ package state import ( + "encoding/json" "fmt" "reflect" @@ -40,7 +41,11 @@ type TlsConfig struct { Artifact string Type string } - +type MutualTlsPortConfig struct { + Port int32 `json:"port"` + Mode string `json:"mode"` + Depth int `json:"depth,omitempty"` +} type StateStore struct { IngressClassLister networkinglisters.IngressClassLister IngressLister networkinglisters.IngressLister @@ -58,6 +63,7 @@ type IngressClassState struct { Listeners sets.Int32 ListenerProtocolMap map[int32]string ListenerTLSConfigMap map[int32]TlsConfig + MtlsPorts map[int32]MutualTlsPortConfig } type IngressState struct { @@ -106,12 +112,27 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { allBackendSets := sets.NewString(DefaultIngressName) allListeners := sets.NewInt32() + mutualTlsPortConfigMap := make(map[int32]MutualTlsPortConfig) + bsHealthCheckerMap[DefaultIngressName] = util.GetDefaultHeathChecker() bsPolicyMap[DefaultIngressName] = util.DefaultBackendSetRoutingPolicy for _, ing := range ingressGroup { hostSecretMap := make(map[string]string) desiredPorts := sets.NewInt32() + + // mtlsVerifyPorts := sets.NewInt32() + //stroe mutual tls port + mutualTlsPortConfigMap := make(map[int32]MutualTlsPortConfig) + // add need mutual tls verifycaiton ports here + mtlsPortsAnnotation := util.GetMutualTlsVerifyAnnotation(ing) + //parse mtls port to json + validateMtlsPortAnnotationJson, err := ParseMutualTlsAnnotationJSON(mtlsPortsAnnotation) + if err != nil { + klog.Infof("Error ******* parsing validateMtlsPortAnnotationJson JSON:", err) + return nil + } + // we always expect the default_ingress backendset desiredBackendSets := sets.NewString(DefaultIngressName) @@ -132,6 +153,15 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { desiredPorts.Insert(servicePort) allListeners.Insert(servicePort) + + //add mtls ports to state map + for _, config := range validateMtlsPortAnnotationJson { + if config.Port == servicePort { + mutualTlsPortConfigMap[servicePort] = config + break + } + } + bsName := util.GenerateBackendSetName(ing.Namespace, serviceName, servicePort) desiredBackendSets.Insert(bsName) allBackendSets.Insert(bsName) @@ -203,6 +233,8 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { Listeners: allListeners, ListenerProtocolMap: listenerProtocolMap, ListenerTLSConfigMap: listenerTLSConfigMap, + //add mutual tls port configmap + MtlsPorts: mutualTlsPortConfigMap, } klog.Infof("Ingress Group state %s, Ingress state %s", util.PrettyPrint(s.IngressGroupState), util.PrettyPrint(s.IngressState)) @@ -300,6 +332,22 @@ func (s *StateStore) GetTLSConfigForListener(port int32) (string, string) { return "", "" } +// func (s *StateStore) GetMutualTlsPortConfigForListener(port int32) MutualTlsPortConfig { +// portMtlsConfig, ok := s.IngressGroupState.MtlsPorts[port] +// if ok { +// return portMtlsConfig +// } +// return MutualTlsPortConfig{} +// } + +func (s *StateStore) GetMutualTlsPortConfigForListener(port int32) (string, int) { + portMtlsConfig, ok := s.IngressGroupState.MtlsPorts[port] + if ok { + return portMtlsConfig.Mode, portMtlsConfig.Depth + } + return "", 0 +} + func (s *StateStore) GetTLSConfigForBackendSet(bsName string) (string, string) { bsTLSConfig, ok := s.IngressGroupState.BackendSetTLSConfigMap[bsName] if ok { @@ -327,3 +375,33 @@ func validatePortInUse(listenerTLSConfig TlsConfig, secretName string, certifica } return nil } + +func ParseMutualTlsAnnotationJSON(s string) ([]MutualTlsPortConfig, error) { + // Check if the string conforms to JSON format + if !isValidJSON(s) { + return nil, fmt.Errorf("Original string does not conform to JSON format") + } + + // Parse JSON string + var mtls []MutualTlsPortConfig + err := json.Unmarshal([]byte(s), &mtls) + if err != nil { + return nil, err + } + + // Remove sub-objects without the 'port' field + var validMtls []MutualTlsPortConfig + for _, config := range mtls { + if config.Port != 0 { + validMtls = append(validMtls, config) + } + } + + return validMtls, nil +} + +// Check if the string conforms to JSON format +func isValidJSON(s string) bool { + var js json.RawMessage + return json.Unmarshal([]byte(s), &js) == nil +} diff --git a/pkg/state/validate-mutual-tls-authentication.yaml b/pkg/state/validate-mutual-tls-authentication.yaml new file mode 100644 index 00000000..8fa89e0d --- /dev/null +++ b/pkg/state/validate-mutual-tls-authentication.yaml @@ -0,0 +1,24 @@ +# +# OCI Native Ingress Controller +# +# Copyright (c) 2023 Oracle America, Inc. and its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +# +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-mutual-authentication-annotation-one + namespace: default + annotations: + oci-native-ingress.oraclecloud.com/mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 443, "mode": "verify","depth":1 }]' +spec: + rules: + - http: + paths: + - pathType: Exact + path: "/HCPath" + backend: + service: + name: test-health-checker-annotation + port: + number: 800 diff --git a/pkg/util/util.go b/pkg/util/util.go index 6ffad8c9..22a44659 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -52,6 +52,12 @@ const ( IngressListenerTlsCertificateAnnotation = "oci-native-ingress.oraclecloud.com/certificate-ocid" IngressBackendTlsEnabledAnnotation = "oci-native-ingress.oraclecloud.com/backend-tls-enabled" + //verify client certificate , only use the lisentner bound CA certificate to verify the client certificate . + /* + mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 443, "mode": "verify","depth":1 }]' + */ + IngressListenerMutualTlsVerifyAnnotation = "oci-native-ingress.oraclecloud.com/mutual-tls-authentication" + // IngressProtocolAnntoation - HTTP only for now // HTTP, HTTP2 - accepted. IngressProtocolAnnotation = "oci-native-ingress.oraclecloud.com/protocol" @@ -599,3 +605,12 @@ func RetrievePods(endpointLister corelisters.EndpointsLister, podLister corelist } return pods, nil } + +func GetMutualTlsVerifyAnnotation(i *networkingv1.Ingress) string { + mtlsVerifyPorts, ok := i.Annotations[IngressListenerMutualTlsVerifyAnnotation] + if !ok { + return "" + } + + return strings.ToLower(mtlsVerifyPorts) +} From 98863302a9c4343c3d02c6ce1cef2bdbc3ee31d7 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 22 Apr 2024 21:18:07 +0800 Subject: [PATCH 2/6] update mtls verify feature Signed-off-by: george --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index fcad8fdd..7d00f8f7 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,8 @@ build: ./main.go CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GO111MODULE=on go build -mod vendor -a -o dist/onic ./main.go image: + + docker build --platform linux/amd64 --build-arg goos=$(GOOS) --build-arg goarch=$(GOARCH) -t ${IMAGE_PATH} -f Dockerfile . docker build --build-arg goos=$(GOOS) --build-arg goarch=$(GOARCH) -t ${IMAGE_PATH} -f Dockerfile . push: From 034497d37b3d50fd167adb81d11682b314ab3083 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 24 Apr 2024 12:51:22 +0800 Subject: [PATCH 3/6] add test mtls verify method Signed-off-by: george --- pkg/controllers/ingress/ingress.go | 50 ++++++++++++++++--- pkg/controllers/ingress/util.go | 2 + pkg/loadbalancer/loadbalancer.go | 4 ++ pkg/state/ingressstate.go | 38 +++++++------- pkg/state/ingressstate_test.go | 23 +++++++++ .../validate-mutual-tls-authentication.yaml | 6 +-- pkg/util/util.go | 2 + 7 files changed, 98 insertions(+), 27 deletions(-) diff --git a/pkg/controllers/ingress/ingress.go b/pkg/controllers/ingress/ingress.go index aad20097..20e78c7c 100644 --- a/pkg/controllers/ingress/ingress.go +++ b/pkg/controllers/ingress/ingress.go @@ -377,9 +377,10 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass * mode, deepth := stateStore.GetMutualTlsPortConfigForListener(port) mtlsPorts := stateStore.IngressGroupState.MtlsPorts - klog.Infof(" GetMutualTlsPortConfigForListener ********** GetMutualTlsPortConfigForListener mtlsPorts : %s ", mtlsPorts) + klog.Infof(" GetMutualTlsPortConfigForListener ********** mtlsPorts : %s ", util.PrettyPrint(mtlsPorts)) - if mode == "verify" { + klog.Infof(" GetMutualTlsPortConfigForListener ********** before : %s ", util.PrettyPrint(listenerSslConfig)) + if mode == util.MutualTlsAuthenticationVerify { // listenerSslConfig.VerifyDepth listenerSslConfig.VerifyPeerCertificate = common.Bool(true) listenerSslConfig.VerifyDepth = &deepth @@ -394,6 +395,7 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass * } } + klog.Infof(" GetMutualTlsPortConfigForListener after : %s ", util.PrettyPrint(listenerSslConfig)) protocol := stateStore.GetListenerProtocol(port) err = c.client.GetLbClient().CreateListener(context.TODO(), lbId, int(port), protocol, listenerSslConfig) @@ -504,11 +506,7 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string, needsUpdate := false artifact, artifactType := stateStore.GetTLSConfigForListener(int32(*listener.Port)) - // c. - // mode, _ := stateStore.GetMutualTlsPortConfigForListener(int32(*listener.Port)) - // if mode == "verify" { - // // 执行相关操作 - // } + var sslConfig *ociloadbalancer.SslConfigurationDetails if artifact != "" { sslConfig, err = GetSSLConfigForListener(namespace, &listener, artifactType, artifact, c.defaultCompartmentId, c.client) @@ -522,6 +520,44 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string, needsUpdate = true } } + + var port = int32(*listener.Port) + mode, deepth := stateStore.GetMutualTlsPortConfigForListener(port) + mtlsPorts := stateStore.IngressGroupState.MtlsPorts + klog.Infof(" syncListenerr mtlsPorts : %s ", util.PrettyPrint(mtlsPorts)) + + klog.Infof(" syncListener before : %s ", util.PrettyPrint(listener.SslConfiguration)) + var modeToBool bool = false + if mode == util.MutualTlsAuthenticationVerify { + modeToBool = true + } + if *listener.SslConfiguration.VerifyPeerCertificate != modeToBool || *listener.SslConfiguration.VerifyDepth != deepth { + klog.Infof(" mtls port config %d needs update mode: %s", *listener.Name, mode) + needsUpdate = true + + if mode == util.MutualTlsAuthenticationVerify { + listener.SslConfiguration.VerifyPeerCertificate = common.Bool(true) + listener.SslConfiguration.VerifyDepth = &deepth + caBundleId, _ := CreateOrGetCaBundleForBackendSet(namespace, artifact, c.defaultCompartmentId, c.client) + if err != nil { + klog.Infof(" syncListener CreateOrGetCaBundleForBackendSet port : %s ", port) + return err + } + if caBundleId != nil { + caBundleIds := []string{*caBundleId} + listener.SslConfiguration.TrustedCertificateAuthorityIds = caBundleIds + } + + } else { + listener.SslConfiguration.VerifyPeerCertificate = common.Bool(false) + listener.SslConfiguration.VerifyDepth = common.Int(1) + listener.SslConfiguration.TrustedCertificateAuthorityIds = nil + + } + } + + klog.Infof(" syncListener after : %s ", util.PrettyPrint(listener.SslConfiguration)) + } protocol := stateStore.GetListenerProtocol(int32(*listener.Port)) diff --git a/pkg/controllers/ingress/util.go b/pkg/controllers/ingress/util.go index def83675..7fc00be3 100644 --- a/pkg/controllers/ingress/util.go +++ b/pkg/controllers/ingress/util.go @@ -184,6 +184,8 @@ func CreateCaBundle(certificateName string, compartmentId string, certificatesCl CreateCaBundleDetails: caBundleDetails, OpcRetryToken: &certificateName, } + klog.Infof(" createCaBundleRequest ********** certificateName %s createCaBundleRequest : %s ", certificateName, util.PrettyPrint(createCaBundleRequest)) + createCaBundle, err := certificatesClient.CreateCaBundle(context.TODO(), createCaBundleRequest) if err != nil { return nil, err diff --git a/pkg/loadbalancer/loadbalancer.go b/pkg/loadbalancer/loadbalancer.go index e3190af5..8088627d 100644 --- a/pkg/loadbalancer/loadbalancer.go +++ b/pkg/loadbalancer/loadbalancer.go @@ -584,6 +584,10 @@ func (lbc *LoadBalancerClient) UpdateListener(ctx context.Context, lbId *string, if sslConfigurationDetails == nil && l.SslConfiguration != nil { sslConfigurationDetails = &loadbalancer.SslConfigurationDetails{ CertificateIds: l.SslConfiguration.CertificateIds, + //add for mtls verify + VerifyDepth: l.SslConfiguration.VerifyDepth, + VerifyPeerCertificate: l.SslConfiguration.VerifyPeerCertificate, + TrustedCertificateAuthorityIds: l.SslConfiguration.TrustedCertificateAuthorityIds, } } diff --git a/pkg/state/ingressstate.go b/pkg/state/ingressstate.go index c66d129a..1b4f3617 100644 --- a/pkg/state/ingressstate.go +++ b/pkg/state/ingressstate.go @@ -112,6 +112,7 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { allBackendSets := sets.NewString(DefaultIngressName) allListeners := sets.NewInt32() + // add mtls verify configmap mutualTlsPortConfigMap := make(map[int32]MutualTlsPortConfig) bsHealthCheckerMap[DefaultIngressName] = util.GetDefaultHeathChecker() @@ -121,18 +122,20 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { hostSecretMap := make(map[string]string) desiredPorts := sets.NewInt32() - // mtlsVerifyPorts := sets.NewInt32() - //stroe mutual tls port - mutualTlsPortConfigMap := make(map[int32]MutualTlsPortConfig) - // add need mutual tls verifycaiton ports here - mtlsPortsAnnotation := util.GetMutualTlsVerifyAnnotation(ing) - //parse mtls port to json - validateMtlsPortAnnotationJson, err := ParseMutualTlsAnnotationJSON(mtlsPortsAnnotation) + validateMtlsPortAnnotationJson, err := ParseMutualTlsAnnotationJSON(ing) + klog.Infof("Ingress name: %s, validateMtlsPortAnnotationJson %s", util.PrettyPrint(ing.Name), util.PrettyPrint(validateMtlsPortAnnotationJson)) + if err != nil { - klog.Infof("Error ******* parsing validateMtlsPortAnnotationJson JSON:", err) + klog.Infof("Error parsing validateMtlsPortAnnotationJson JSON:", err) return nil } + for _, configPort := range validateMtlsPortAnnotationJson { + // add new mtls port to map + mutualTlsPortConfigMap[configPort.Port] = configPort + } + klog.Infof(" mutualTlsPortConfigMap %s, mutualTlsPortConfigMap %s", util.PrettyPrint(ing.Name), util.PrettyPrint(mutualTlsPortConfigMap)) + // we always expect the default_ingress backendset desiredBackendSets := sets.NewString(DefaultIngressName) @@ -154,14 +157,6 @@ func (s *StateStore) BuildState(ingressClass *networkingv1.IngressClass) error { desiredPorts.Insert(servicePort) allListeners.Insert(servicePort) - //add mtls ports to state map - for _, config := range validateMtlsPortAnnotationJson { - if config.Port == servicePort { - mutualTlsPortConfigMap[servicePort] = config - break - } - } - bsName := util.GenerateBackendSetName(ing.Namespace, serviceName, servicePort) desiredBackendSets.Insert(bsName) allBackendSets.Insert(bsName) @@ -376,7 +371,16 @@ func validatePortInUse(listenerTLSConfig TlsConfig, secretName string, certifica return nil } -func ParseMutualTlsAnnotationJSON(s string) ([]MutualTlsPortConfig, error) { +func ParseMutualTlsAnnotationJSON(ing *networkingv1.Ingress) ([]MutualTlsPortConfig, error) { + + mtlsPortsAnnotation := util.GetMutualTlsVerifyAnnotation(ing) + klog.Infof("Ingress name ParseMutualTlsAnnotationJSON %s, mtlsPortsAnnotation %s", util.PrettyPrint(ing.Name), util.PrettyPrint(mtlsPortsAnnotation)) + + s := mtlsPortsAnnotation + //check if s is empty + if s == "" { + return []MutualTlsPortConfig{}, nil + } // Check if the string conforms to JSON format if !isValidJSON(s) { return nil, fmt.Errorf("Original string does not conform to JSON format") diff --git a/pkg/state/ingressstate_test.go b/pkg/state/ingressstate_test.go index 52d326c6..17c41e96 100644 --- a/pkg/state/ingressstate_test.go +++ b/pkg/state/ingressstate_test.go @@ -36,6 +36,7 @@ const ( TestIngressStateWithPortNameFilePath = "test-ingress-state_withportname.yaml" TestIngressStateWithNamedClassesFilePath = "test-ingress-state_withnamedclasses.yaml" TestSslTerminationAtLb = "test-ssl-termination-lb.yaml" + TestMtlsAuthVerfify = "validate-mutual-tls-authentication.yaml" ) func setUp(ctx context.Context, ingressClassList *networkingv1.IngressClassList, ingressList *networkingv1.IngressList, testService *v1.ServiceList) (networkinglisters.IngressClassLister, networkinglisters.IngressLister, corelisters.ServiceLister) { @@ -449,3 +450,25 @@ func TestSslTerminationAtLB(t *testing.T) { Expect(lstTlsConfig.Artifact).Should(Equal(certificateId)) Expect(lstTlsConfig.Type).Should(Equal(ArtifactTypeCertificate)) } + +func TestMtlsAuthVerifyPortConfig(t *testing.T) { + RegisterTestingT(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ingressClassList := testutil.GetIngressClassList() + + ingressList := testutil.ReadResourceAsIngressList(TestMtlsAuthVerfify) + + testService := testutil.GetServiceListResource("default", "mtls-auth-verify-annotation", 943) + ingressClassLister, ingressLister, serviceLister := setUp(ctx, ingressClassList, ingressList, testService) + + stateStore := NewStateStore(ingressClassLister, ingressLister, serviceLister, nil) + err := stateStore.BuildState(&ingressClassList.Items[0]) + Expect(err).NotTo(HaveOccurred()) + + mode, deepth := stateStore.GetMutualTlsPortConfigForListener(943) + + Expect(mode).Should(Equal(util.MutualTlsAuthenticationVerify)) + Expect(deepth).Should(Equal(1)) + +} diff --git a/pkg/state/validate-mutual-tls-authentication.yaml b/pkg/state/validate-mutual-tls-authentication.yaml index 8fa89e0d..16d18396 100644 --- a/pkg/state/validate-mutual-tls-authentication.yaml +++ b/pkg/state/validate-mutual-tls-authentication.yaml @@ -10,7 +10,7 @@ metadata: name: ingress-mutual-authentication-annotation-one namespace: default annotations: - oci-native-ingress.oraclecloud.com/mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 443, "mode": "verify","depth":1 }]' + oci-native-ingress.oraclecloud.com/mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 943, "mode": "verify","depth":1 }]' spec: rules: - http: @@ -19,6 +19,6 @@ spec: path: "/HCPath" backend: service: - name: test-health-checker-annotation + name: mtls-auth-verify-annotation port: - number: 800 + number: 443 diff --git a/pkg/util/util.go b/pkg/util/util.go index 22a44659..bd7651d4 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -91,6 +91,8 @@ const ( CertificateCacheMaxAgeInMinutes = 10 LBCacheMaxAgeInMinutes = 1 WAFCacheMaxAgeInMinutes = 5 + + MutualTlsAuthenticationVerify = "verify" ) var ErrIngressClassNotReady = errors.New("ingress class not ready") From d3e222e7b3f45b80c5e9e1bd59f022a16430ef4b Mon Sep 17 00:00:00 2001 From: george Date: Wed, 24 Apr 2024 13:31:12 +0800 Subject: [PATCH 4/6] reversion Makefile to original version Signed-off-by: george --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 7d00f8f7..fcad8fdd 100644 --- a/Makefile +++ b/Makefile @@ -62,8 +62,6 @@ build: ./main.go CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GO111MODULE=on go build -mod vendor -a -o dist/onic ./main.go image: - - docker build --platform linux/amd64 --build-arg goos=$(GOOS) --build-arg goarch=$(GOARCH) -t ${IMAGE_PATH} -f Dockerfile . docker build --build-arg goos=$(GOOS) --build-arg goarch=$(GOARCH) -t ${IMAGE_PATH} -f Dockerfile . push: From 1a2d3d04f66fb63d0077624977e8a91bfe79b66f Mon Sep 17 00:00:00 2001 From: george Date: Tue, 14 May 2024 15:37:31 +0800 Subject: [PATCH 5/6] Add trustCACert part in oci-native-ingress.oraclecloud.com/mutual-tls-authentication Signed-off-by: george --- pkg/controllers/ingress/ingress.go | 46 ++++++++++++------- pkg/state/ingressstate.go | 13 +++--- pkg/state/ingressstate_test.go | 2 +- .../validate-mutual-tls-authentication.yaml | 2 +- 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/pkg/controllers/ingress/ingress.go b/pkg/controllers/ingress/ingress.go index 20e78c7c..e7d971c4 100644 --- a/pkg/controllers/ingress/ingress.go +++ b/pkg/controllers/ingress/ingress.go @@ -375,7 +375,7 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass * // listenerSslConfig.VerifyPeerCertificate - mode, deepth := stateStore.GetMutualTlsPortConfigForListener(port) + mode, deepth, trustcacert := stateStore.GetMutualTlsPortConfigForListener(port) mtlsPorts := stateStore.IngressGroupState.MtlsPorts klog.Infof(" GetMutualTlsPortConfigForListener ********** mtlsPorts : %s ", util.PrettyPrint(mtlsPorts)) @@ -384,16 +384,23 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass * // listenerSslConfig.VerifyDepth listenerSslConfig.VerifyPeerCertificate = common.Bool(true) listenerSslConfig.VerifyDepth = &deepth - caBundleId, _ := CreateOrGetCaBundleForBackendSet(ingress.Namespace, artifact, c.defaultCompartmentId, c.client) - if err != nil { - klog.Infof(" CreateOrGetCaBundleForBackendSet ********** CreateOrGetCaBundleForBackendSet port : %s ", port) - return err - } - if caBundleId != nil { - caBundleIds := []string{*caBundleId} + + // check wethear the trustcacert is valid ca bundle + if isTrustAuthorityCaBundle(trustcacert) { + caBundleIds := []string{trustcacert} listenerSslConfig.TrustedCertificateAuthorityIds = caBundleIds } + // caBundleId, _ := CreateOrGetCaBundleForBackendSet(ingress.Namespace, artifact, c.defaultCompartmentId, c.client) + // if err != nil { + // klog.Infof(" CreateOrGetCaBundleForBackendSet ********** CreateOrGetCaBundleForBackendSet port : %s ", port) + // return err + // } + // if caBundleId != nil { + // caBundleIds := []string{*caBundleId} + // listenerSslConfig.TrustedCertificateAuthorityIds = caBundleIds + // } + } klog.Infof(" GetMutualTlsPortConfigForListener after : %s ", util.PrettyPrint(listenerSslConfig)) @@ -522,7 +529,7 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string, } var port = int32(*listener.Port) - mode, deepth := stateStore.GetMutualTlsPortConfigForListener(port) + mode, deepth, trustcacert := stateStore.GetMutualTlsPortConfigForListener(port) mtlsPorts := stateStore.IngressGroupState.MtlsPorts klog.Infof(" syncListenerr mtlsPorts : %s ", util.PrettyPrint(mtlsPorts)) @@ -538,16 +545,23 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string, if mode == util.MutualTlsAuthenticationVerify { listener.SslConfiguration.VerifyPeerCertificate = common.Bool(true) listener.SslConfiguration.VerifyDepth = &deepth - caBundleId, _ := CreateOrGetCaBundleForBackendSet(namespace, artifact, c.defaultCompartmentId, c.client) - if err != nil { - klog.Infof(" syncListener CreateOrGetCaBundleForBackendSet port : %s ", port) - return err - } - if caBundleId != nil { - caBundleIds := []string{*caBundleId} + + // check wethear the trustcacert is valid ca bundle + if isTrustAuthorityCaBundle(trustcacert) { + caBundleIds := []string{trustcacert} listener.SslConfiguration.TrustedCertificateAuthorityIds = caBundleIds } + // caBundleId, _ := CreateOrGetCaBundleForBackendSet(namespace, artifact, c.defaultCompartmentId, c.client) + // if err != nil { + // klog.Infof(" syncListener CreateOrGetCaBundleForBackendSet port : %s ", port) + // return err + // } + // if caBundleId != nil { + // caBundleIds := []string{*caBundleId} + // listener.SslConfiguration.TrustedCertificateAuthorityIds = caBundleIds + // } + } else { listener.SslConfiguration.VerifyPeerCertificate = common.Bool(false) listener.SslConfiguration.VerifyDepth = common.Int(1) diff --git a/pkg/state/ingressstate.go b/pkg/state/ingressstate.go index 1b4f3617..7bbcc151 100644 --- a/pkg/state/ingressstate.go +++ b/pkg/state/ingressstate.go @@ -42,9 +42,10 @@ type TlsConfig struct { Type string } type MutualTlsPortConfig struct { - Port int32 `json:"port"` - Mode string `json:"mode"` - Depth int `json:"depth,omitempty"` + Port int32 `json:"port"` + Mode string `json:"mode"` + Depth int `json:"depth,omitempty"` + TrustCACert string `json:"trustcacert"` } type StateStore struct { IngressClassLister networkinglisters.IngressClassLister @@ -335,12 +336,12 @@ func (s *StateStore) GetTLSConfigForListener(port int32) (string, string) { // return MutualTlsPortConfig{} // } -func (s *StateStore) GetMutualTlsPortConfigForListener(port int32) (string, int) { +func (s *StateStore) GetMutualTlsPortConfigForListener(port int32) (string, int, string) { portMtlsConfig, ok := s.IngressGroupState.MtlsPorts[port] if ok { - return portMtlsConfig.Mode, portMtlsConfig.Depth + return portMtlsConfig.Mode, portMtlsConfig.Depth, portMtlsConfig.TrustCACert } - return "", 0 + return "", 0, "" } func (s *StateStore) GetTLSConfigForBackendSet(bsName string) (string, string) { diff --git a/pkg/state/ingressstate_test.go b/pkg/state/ingressstate_test.go index 17c41e96..660b0519 100644 --- a/pkg/state/ingressstate_test.go +++ b/pkg/state/ingressstate_test.go @@ -466,7 +466,7 @@ func TestMtlsAuthVerifyPortConfig(t *testing.T) { err := stateStore.BuildState(&ingressClassList.Items[0]) Expect(err).NotTo(HaveOccurred()) - mode, deepth := stateStore.GetMutualTlsPortConfigForListener(943) + mode, deepth, _ := stateStore.GetMutualTlsPortConfigForListener(943) Expect(mode).Should(Equal(util.MutualTlsAuthenticationVerify)) Expect(deepth).Should(Equal(1)) diff --git a/pkg/state/validate-mutual-tls-authentication.yaml b/pkg/state/validate-mutual-tls-authentication.yaml index 16d18396..da62eb55 100644 --- a/pkg/state/validate-mutual-tls-authentication.yaml +++ b/pkg/state/validate-mutual-tls-authentication.yaml @@ -10,7 +10,7 @@ metadata: name: ingress-mutual-authentication-annotation-one namespace: default annotations: - oci-native-ingress.oraclecloud.com/mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 943, "mode": "verify","depth":1 }]' + oci-native-ingress.oraclecloud.com/mutual-tls-authentication: '[{"port": 80, "mode": "passthrough"}, {"port": 943, "mode": "verify","depth":1 ,"trustCACert" : "ocid1.cabundle.oc1.phx.amaaaaaacuco5yqaiwnnqo54ffsumwoxjxtgwtvyyau3dv7gyeisykfavzta" }]' spec: rules: - http: From 799aece8c5230ed2ef4c00b9fa148b3df735431a Mon Sep 17 00:00:00 2001 From: george Date: Tue, 4 Jun 2024 16:02:57 +0800 Subject: [PATCH 6/6] add verify trustcacert is a ca bundle Signed-off-by: george --- pkg/controllers/ingress/ingress.go | 44 +++++++++++------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/pkg/controllers/ingress/ingress.go b/pkg/controllers/ingress/ingress.go index e7d971c4..34ac1830 100644 --- a/pkg/controllers/ingress/ingress.go +++ b/pkg/controllers/ingress/ingress.go @@ -386,20 +386,18 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass * listenerSslConfig.VerifyDepth = &deepth // check wethear the trustcacert is valid ca bundle - if isTrustAuthorityCaBundle(trustcacert) { + if mode == util.MutualTlsAuthenticationVerify && isTrustAuthorityCaBundle(trustcacert) { caBundleIds := []string{trustcacert} listenerSslConfig.TrustedCertificateAuthorityIds = caBundleIds - } + } else { + if mode == util.MutualTlsAuthenticationVerify { + klog.Error("verify trustcacert error, not a valid CA bundle ID: %s", util.PrettyPrint(trustcacert)) + } + listenerSslConfig.VerifyPeerCertificate = common.Bool(false) + listenerSslConfig.VerifyDepth = common.Int(1) + listenerSslConfig.TrustedCertificateAuthorityIds = nil - // caBundleId, _ := CreateOrGetCaBundleForBackendSet(ingress.Namespace, artifact, c.defaultCompartmentId, c.client) - // if err != nil { - // klog.Infof(" CreateOrGetCaBundleForBackendSet ********** CreateOrGetCaBundleForBackendSet port : %s ", port) - // return err - // } - // if caBundleId != nil { - // caBundleIds := []string{*caBundleId} - // listenerSslConfig.TrustedCertificateAuthorityIds = caBundleIds - // } + } } klog.Infof(" GetMutualTlsPortConfigForListener after : %s ", util.PrettyPrint(listenerSslConfig)) @@ -542,27 +540,17 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string, klog.Infof(" mtls port config %d needs update mode: %s", *listener.Name, mode) needsUpdate = true - if mode == util.MutualTlsAuthenticationVerify { + // check wethear the trustcacert is valid ca bundle + if mode == util.MutualTlsAuthenticationVerify && isTrustAuthorityCaBundle(trustcacert) { listener.SslConfiguration.VerifyPeerCertificate = common.Bool(true) listener.SslConfiguration.VerifyDepth = &deepth - - // check wethear the trustcacert is valid ca bundle - if isTrustAuthorityCaBundle(trustcacert) { - caBundleIds := []string{trustcacert} - listener.SslConfiguration.TrustedCertificateAuthorityIds = caBundleIds - } - - // caBundleId, _ := CreateOrGetCaBundleForBackendSet(namespace, artifact, c.defaultCompartmentId, c.client) - // if err != nil { - // klog.Infof(" syncListener CreateOrGetCaBundleForBackendSet port : %s ", port) - // return err - // } - // if caBundleId != nil { - // caBundleIds := []string{*caBundleId} - // listener.SslConfiguration.TrustedCertificateAuthorityIds = caBundleIds - // } + caBundleIds := []string{trustcacert} + listener.SslConfiguration.TrustedCertificateAuthorityIds = caBundleIds } else { + if mode == util.MutualTlsAuthenticationVerify { + klog.Error("verify trustcacert error, not a valid CA bundle ID: %s", util.PrettyPrint(trustcacert)) + } listener.SslConfiguration.VerifyPeerCertificate = common.Bool(false) listener.SslConfiguration.VerifyDepth = common.Int(1) listener.SslConfiguration.TrustedCertificateAuthorityIds = nil