Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions api/v1alpha1/backendtrafficpolicy_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,24 @@ type BackendTelemetry struct {
//
// +optional
Tracing *Tracing `json:"tracing,omitempty"`
// Metrics defines metrics configuration for the backend or Route.
//
// +optional
Metrics *BackendMetrics `json:"metrics,omitempty"`
}

type BackendMetrics struct {
// RouteStatName defines the value of the Route stat_prefix, determining how the route stats are named.
// For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#config-route-v3-route
// The supported operators for this pattern are:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

routes can be attached to multiple gateways/parentRefs which will need to be represented here in case the user doesnt want to aggregate them into 1, and prefers per parent per route metrics
https://gateway-api.sigs.k8s.io/reference/spec/#httproutespec

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As far as I understand your point, I thought this function was supposed to set the stats for the route in the gateways.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/envoyproxy/gateway/blob/main/internal/gatewayapi/testdata/httproute-with-multiple-gateways-from-different-ns.out.yaml should help understand how 1 route can attach to multiple gateways
and this introduces two uses cases for metrics granularity that is mentioned in #6310 (comment)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to check and validate this somehow?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/envoyproxy/gateway/blob/main/internal/gatewayapi/testdata/httproute-with-multiple-gateways-from-different-ns.out.yaml should help understand how 1 route can attach to multiple gateways and this introduces two uses cases for metrics granularity that is mentioned in #6310 (comment)

It seems I understand what this is about. In the function https://github.com/envoyproxy/gateway/blob/26f9b20a7d5f3060f841787e677c0ed127adc759/internal/gatewayapi/backendtrafficpolicy.go#L664, there is processing for the Gateway, but here we have the IR, and we need to somehow map it to a resource to get the template parameters. We could do this via "resources", but that would be expensive — we'd have to go through all resources for every route, which is O(n²). Maybe we should reconsider the metadata field for all routes?)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you should have access to it indirectly via the IR listener name

func irListenerName(listener *ListenerContext) string {

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thinking out loud, it may help to start off with an API within EnvoyProxy which is easier to build out its not tied to policy translation

// %ROUTE_NAME%: name of Gateway API xRoute resource
// %ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource
// %ROUTE_KIND%: kind of Gateway API xRoute resource
// Example: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME% => httproute/my-ns/my-route
// Disabled by default.
//
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
//
// Disabled by default.
//

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apply

// +optional
RouteStatName *string `json:"routeStatName,omitempty"`
}

// ProtocolUpgradeConfig specifies the configuration for protocol upgrades.
Expand Down
45 changes: 36 additions & 9 deletions api/v1alpha1/validation/envoyproxy_validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (
egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
)

var statNameRegex = regexp.MustCompile("%[^%]*%")

// ValidateEnvoyProxy validates the provided EnvoyProxy.
func ValidateEnvoyProxy(proxy *egv1a1.EnvoyProxy) error {
var errs []error
Expand Down Expand Up @@ -213,8 +215,8 @@ func validateProxyTelemetry(spec *egv1a1.EnvoyProxySpec) []error {
}

if spec.Telemetry.Metrics.ClusterStatName != nil {
if clusterStatErrs := validateClusterStatName(*spec.Telemetry.Metrics.ClusterStatName); clusterStatErrs != nil {
errs = append(errs, clusterStatErrs...)
if clusterStatErr := ValidateClusterStatName(*spec.Telemetry.Metrics.ClusterStatName); clusterStatErr != nil {
errs = append(errs, clusterStatErr)
}
}
}
Expand Down Expand Up @@ -294,7 +296,22 @@ func validateFilterOrder(filterOrder []egv1a1.FilterPosition) error {
return nil
}

func validateClusterStatName(clusterStatName string) []error {
func ValidateRouteStatName(routeStatName string) error {
supportedOperators := map[string]bool{
egv1a1.StatFormatterRouteName: true,
egv1a1.StatFormatterRouteNamespace: true,
egv1a1.StatFormatterRouteKind: true,
egv1a1.StatFormatterRouteRuleName: true,
}

if err := validateStatName(routeStatName, supportedOperators); err != nil {
return fmt.Errorf("unable to configure Route Stat Name: %w", err)
}

return nil
}

func ValidateClusterStatName(clusterStatName string) error {
supportedOperators := map[string]bool{
egv1a1.StatFormatterRouteName: true,
egv1a1.StatFormatterRouteNamespace: true,
Expand All @@ -304,15 +321,25 @@ func validateClusterStatName(clusterStatName string) []error {
egv1a1.StatFormatterBackendRefs: true,
}

var errs []error
re := regexp.MustCompile("%[^%]*%")
matches := re.FindAllString(clusterStatName, -1)
if err := validateStatName(clusterStatName, supportedOperators); err != nil {
return fmt.Errorf("unable to configure Cluster Stat Name: %w", err)
}

return nil
}

func validateStatName(statName string, supportedOperators map[string]bool) error {
var unsupportedOperators []string
matches := statNameRegex.FindAllString(statName, -1)
for _, operator := range matches {
if _, ok := supportedOperators[operator]; !ok {
err := fmt.Errorf("unable to configure Cluster Stat Name with unsupported operator: %s", operator)
errs = append(errs, err)
unsupportedOperators = append(unsupportedOperators, operator)
}
}

return errs
if len(unsupportedOperators) > 0 {
return fmt.Errorf("unsupported operators: %v", unsupportedOperators)
}

return nil
}
25 changes: 25 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -2265,6 +2265,22 @@ spec:
Telemetry configures the telemetry settings for the policy target (Gateway or xRoute).
This will override the telemetry settings in the EnvoyProxy resource.
properties:
metrics:
description: Metrics defines metrics configuration for the backend
or Route.
properties:
routeStatName:
description: |-
RouteStatName defines the value of the Route stat_prefix, determining how the route stats are named.
For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#config-route-v3-route
The supported operators for this pattern are:
%ROUTE_NAME%: name of Gateway API xRoute resource
%ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource
%ROUTE_KIND%: kind of Gateway API xRoute resource
Example: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME% => httproute/my-ns/my-route
Disabled by default.
type: string
type: object
tracing:
description: Tracing configures the tracing settings for the backend
or HTTPRoute.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2264,6 +2264,22 @@ spec:
Telemetry configures the telemetry settings for the policy target (Gateway or xRoute).
This will override the telemetry settings in the EnvoyProxy resource.
properties:
metrics:
description: Metrics defines metrics configuration for the backend
or Route.
properties:
routeStatName:
description: |-
RouteStatName defines the value of the Route stat_prefix, determining how the route stats are named.
For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#config-route-v3-route
The supported operators for this pattern are:
%ROUTE_NAME%: name of Gateway API xRoute resource
%ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource
%ROUTE_KIND%: kind of Gateway API xRoute resource
Example: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME% => httproute/my-ns/my-route
Disabled by default.
type: string
type: object
tracing:
description: Tracing configures the tracing settings for the backend
or HTTPRoute.
Expand Down
50 changes: 50 additions & 0 deletions internal/gatewayapi/backendtrafficpolicy.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"

egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
egv1a1validation "github.com/envoyproxy/gateway/api/v1alpha1/validation"
"github.com/envoyproxy/gateway/internal/gatewayapi/resource"
"github.com/envoyproxy/gateway/internal/gatewayapi/status"
"github.com/envoyproxy/gateway/internal/ir"
Expand Down Expand Up @@ -705,6 +706,11 @@ func (t *Translator) applyTrafficFeatureToRoute(route RouteContext,
x *ir.Xds,
policyTargetListener *gwapiv1.SectionName,
) {
routeStatName := ""
if tf.Telemetry != nil && tf.Telemetry.Metrics != nil {
routeStatName = ptr.Deref(tf.Telemetry.Metrics.RouteStatName, "")
}

prefix := irRoutePrefix(route)
for _, tcp := range x.TCP {
// if listenerName is not nil, only apply to the specific listener
Expand All @@ -728,6 +734,7 @@ func (t *Translator) applyTrafficFeatureToRoute(route RouteContext,
setIfNil(&r.Timeout, tf.Timeout)
setIfNil(&r.BackendConnection, tf.BackendConnection)
setIfNil(&r.DNS, tf.DNS)
setIfNil(&r.StatName, buildRouteStatName(routeStatName, r.Metadata))
}
}
}
Expand Down Expand Up @@ -773,6 +780,7 @@ func (t *Translator) applyTrafficFeatureToRoute(route RouteContext,
continue
}

r.StatName = buildRouteStatName(routeStatName, r.Metadata)
if errs != nil {
// Return a 500 direct response
r.DirectResponse = &ir.CustomResponse{
Expand Down Expand Up @@ -889,6 +897,11 @@ func (t *Translator) buildTrafficFeatures(policy *egv1a1.BackendTrafficPolicy, r
errs = errors.Join(errs, err)
}

if err = validateTelemetry(policy.Spec.Telemetry); err != nil {
err = perr.WithMessage(err, "Telemetry")
errs = errors.Join(errs, err)
}

cp = buildCompression(policy.Spec.Compression, policy.Spec.Compressor)
httpUpgrade = buildHTTPProtocolUpgradeConfig(policy.Spec.HTTPUpgrade)

Expand Down Expand Up @@ -925,6 +938,11 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(
return errs
}

routeStatName := ""
if tf.Telemetry != nil && tf.Telemetry.Metrics != nil {
routeStatName = ptr.Deref(tf.Telemetry.Metrics.RouteStatName, "")
}

// Apply IR to all the routes within the specific Gateway
// If the feature is already set, then skip it, since it must have
// set by a policy attaching to the route
Expand Down Expand Up @@ -955,6 +973,7 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(
setIfNil(&r.TCPKeepalive, tf.TCPKeepalive)
setIfNil(&r.Timeout, tf.Timeout)
setIfNil(&r.DNS, tf.DNS)
setIfNil(&r.StatName, buildRouteStatName(routeStatName, r.Metadata))
}
}

Expand Down Expand Up @@ -1002,6 +1021,7 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(
continue
}

setIfNil(&r.StatName, buildRouteStatName(routeStatName, r.Metadata))
if errs != nil {
// Return a 500 direct response
r.DirectResponse = &ir.CustomResponse{
Expand Down Expand Up @@ -1613,3 +1633,33 @@ func buildHTTPProtocolUpgradeConfig(cfgs []*egv1a1.ProtocolUpgradeConfig) []ir.H

return result
}

func validateTelemetry(telemetry *egv1a1.BackendTelemetry) error {
if telemetry == nil {
return nil
}

if telemetry.Metrics != nil && ptr.Deref(telemetry.Metrics.RouteStatName, "") != "" {
return egv1a1validation.ValidateRouteStatName(*telemetry.Metrics.RouteStatName)
}

return nil
}

func buildRouteStatName(routeStatName string, metadata *ir.ResourceMetadata) *string {
if routeStatName == "" || metadata == nil {
return nil
}

statName := strings.ReplaceAll(routeStatName, egv1a1.StatFormatterRouteName, metadata.Name)
statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteNamespace, metadata.Namespace)
statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteKind, metadata.Kind)

if metadata.SectionName == "" {
statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteRuleName, "-")
} else {
statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteRuleName, metadata.SectionName)
}

return &statName
}
3 changes: 2 additions & 1 deletion internal/gatewayapi/route.go
Original file line number Diff line number Diff line change
Expand Up @@ -1256,6 +1256,7 @@ func (t *Translator) processTLSRouteParentRefs(tlsRoute *TLSRouteContext, resour
Settings: destSettings,
Metadata: buildResourceMetadata(tlsRoute, nil),
},
Metadata: buildResourceMetadata(tlsRoute, nil),
}
irListener.Routes = append(irListener.Routes, irRoute)

Expand Down Expand Up @@ -1554,9 +1555,9 @@ func (t *Translator) processTCPRouteParentRefs(tcpRoute *TCPRouteContext, resour
Destination: &ir.RouteDestination{
Name: destName,
Settings: destSettings,
// tcpRoute Must have a single rule, so can use index 0.
Metadata: buildResourceMetadata(tcpRoute, tcpRoute.Spec.Rules[0].Name),
},
Metadata: buildResourceMetadata(tcpRoute, nil),
}

if irListener.TLS != nil {
Expand Down
Loading