Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ A drop-in OpenTelemetry agent for Go applications that minimizes code changes wh
- [Kafka Support](#-kafka-support) - Producers • Consumers
- [HTTP Client](#-http-client-support)
- [Metrics Support](#-metrics-support) - Automatic • Custom • Runtime
- [Route Exclusion](#-route-exclusion) - Skip tracing for health checks & internal paths
- [Configuration](#️-configuration)
- [Requirements & Compatibility](#-requirements--compatibility)
- [Testing](#-testing)
Expand All @@ -26,6 +27,7 @@ A drop-in OpenTelemetry agent for Go applications that minimizes code changes wh
- 🎯 **Auto-instrumentation** - HTTP, gRPC, SQL, Redis, Kafka automatically traced with proper span nesting
- 📊 **Automatic metrics** - Runtime (memory, GC, goroutines), HTTP, gRPC, database, Kafka, Redis metrics out-of-the-box
- 📈 **Custom metrics** - Simple helpers for counters, histograms, gauges for business metrics
- 🚫 **Route exclusion** - Automatically skips health checks and infrastructure endpoints from tracing
- ⚙️ **Environment-based config** - Uses standard OpenTelemetry environment variables (no hardcoded config)
- 🔍 **Complete observability** - Full distributed tracing + metrics across all layers (HTTP → gRPC → DB → External APIs)

Expand Down Expand Up @@ -618,6 +620,45 @@ func processOrder(ctx context.Context, value float64) {
}
```

## 🚫 Route Exclusion

The agent automatically skips tracing for common health check and infrastructure endpoints, reducing noise in your traces. This works across all supported frameworks (net/http, Gin, Chi, Echo, Gorilla Mux, gRPC-Gateway).

### Default Excluded Routes

Out of the box, the following paths are excluded from tracing:

- **Exact paths**: `/health`, `/healthz`, `/metrics`, `/ready`, `/live`, `/ping`
- **Glob patterns**: `/*/health`, `/*/healthz`, `/*/metrics`, `/*/ready`, `/*/live`, `/*/ping`

### Configuration

Three environment variables control route exclusion:

| Variable | Default | Description |
|----------|---------|-------------|
| `LAST9_EXCLUDED_PATHS` | `/health,/healthz,/metrics,/ready,/live,/ping` | Exact path matches |
| `LAST9_EXCLUDED_PATH_PREFIXES` | *(none)* | Prefix matches (e.g., `/internal/`) |
| `LAST9_EXCLUDED_PATH_PATTERNS` | `/*/health,/*/healthz,/*/metrics,/*/ready,/*/live,/*/ping` | Glob patterns using `path.Match` semantics |

### Examples

```bash
# Add a prefix exclusion for internal debug endpoints
export LAST9_EXCLUDED_PATH_PREFIXES="/internal/,/debug/"

# Custom exact paths to exclude
export LAST9_EXCLUDED_PATHS="/health,/healthz,/status,/version"

# Disable all default exclusions (trace everything)
export LAST9_EXCLUDED_PATHS=""
export LAST9_EXCLUDED_PATH_PATTERNS=""
```

### How It Works

Route matching is evaluated in order: exact match (O(1) map lookup) → prefix match → glob pattern match. The first match wins. Setting an environment variable to an empty string (`""`) disables its defaults, allowing you to opt out of default exclusions.

## ⚙️ Configuration

The agent reads configuration from environment variables following OpenTelemetry standards:
Expand All @@ -630,6 +671,9 @@ The agent reads configuration from environment variables following OpenTelemetry
| `OTEL_SERVICE_VERSION` | No | - | Service version (e.g., git commit SHA) |
| `OTEL_RESOURCE_ATTRIBUTES` | No | - | Additional attributes (key=value pairs) |
| `OTEL_TRACES_SAMPLER` | No | `always_on` | Sampling strategy |
| `LAST9_EXCLUDED_PATHS` | No | `/health,/healthz,...` | Exact paths to exclude from tracing |
| `LAST9_EXCLUDED_PATH_PREFIXES` | No | - | Path prefixes to exclude from tracing |
| `LAST9_EXCLUDED_PATH_PATTERNS` | No | `/*/health,/*/healthz,...` | Glob patterns to exclude from tracing |

### Resource Attributes

Expand Down
15 changes: 15 additions & 0 deletions agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"time"

"github.com/last9/go-agent/config"
"github.com/last9/go-agent/internal/routematcher"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
Expand All @@ -41,6 +42,7 @@ var (
// Agent represents the Last9 telemetry agent
type Agent struct {
config *config.Config
routeMatcher *routematcher.RouteMatcher
tracerProvider *sdktrace.TracerProvider
meterProvider *metric.MeterProvider
shutdown func(context.Context) error
Expand Down Expand Up @@ -115,8 +117,12 @@ func Start() error {
log.Printf("[Last9 Agent] Warning: Failed to start runtime metrics: %v", runtimeErr)
}

// Build route matcher for path exclusion
rm := routematcher.New(cfg.ExcludedPaths, cfg.ExcludedPathPrefixes, cfg.ExcludedPathPatterns)

globalAgent = &Agent{
config: cfg,
routeMatcher: rm,
tracerProvider: tp,
meterProvider: mp,
shutdown: func(ctx context.Context) error {
Expand Down Expand Up @@ -175,6 +181,15 @@ func GetConfig() *config.Config {
return globalAgent.config
}

// GetRouteMatcher returns the route matcher for path exclusion (or nil if not initialized).
// Nil is safe to use — RouteMatcher.ShouldExclude on nil returns false (no exclusion).
func GetRouteMatcher() *routematcher.RouteMatcher {
if globalAgent == nil {
return nil
}
return globalAgent.routeMatcher
}

// createResource creates an OpenTelemetry resource with service information
func createResource(cfg *config.Config) (*resource.Resource, error) {
// Build base attributes
Expand Down
46 changes: 46 additions & 0 deletions agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,52 @@ func TestCreateSampler(t *testing.T) {
}
}

func TestGetRouteMatcher(t *testing.T) {
defer Reset()

// Before Start, returns nil
rm := GetRouteMatcher()
if rm != nil {
t.Error("GetRouteMatcher() should return nil before Start()")
}
// Nil receiver is safe — should not exclude anything
if rm.ShouldExclude("/health") {
t.Error("nil RouteMatcher should not exclude /health")
}

os.Setenv("OTEL_SERVICE_NAME", "test-service")
defer os.Unsetenv("OTEL_SERVICE_NAME")
// Use defaults (health endpoints excluded)
os.Unsetenv("LAST9_EXCLUDED_PATHS")
os.Unsetenv("LAST9_EXCLUDED_PATH_PREFIXES")
os.Unsetenv("LAST9_EXCLUDED_PATH_PATTERNS")

if err := Start(); err != nil {
t.Fatalf("Start() failed: %v", err)
}

rm = GetRouteMatcher()
if rm == nil {
t.Fatal("GetRouteMatcher() should return non-nil after Start()")
}
if rm.IsEmpty() {
t.Error("RouteMatcher should not be empty with default exclusion rules")
}

// Default exact excludes /health
if !rm.ShouldExclude("/health") {
t.Error("default RouteMatcher should exclude /health")
}
// Default pattern excludes /v1/health
if !rm.ShouldExclude("/v1/health") {
t.Error("default RouteMatcher should exclude /v1/health via glob pattern")
}
// Should not exclude normal paths
if rm.ShouldExclude("/api/users") {
t.Error("default RouteMatcher should not exclude /api/users")
}
}

func TestParseSamplerRatio(t *testing.T) {
tests := []struct {
input string
Expand Down
51 changes: 51 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,21 @@ type Config struct {

// ResourceAttributes contains additional resource attributes
ResourceAttributes []attribute.KeyValue

// ExcludedPaths is a list of exact URL paths to exclude from tracing (from LAST9_EXCLUDED_PATHS).
// Default: /health,/healthz,/metrics,/ready,/live,/ping
// Set LAST9_EXCLUDED_PATHS="" to disable defaults.
ExcludedPaths []string

// ExcludedPathPrefixes is a list of URL path prefixes to exclude from tracing (from LAST9_EXCLUDED_PATH_PREFIXES).
// Default: none
ExcludedPathPrefixes []string

// ExcludedPathPatterns is a list of glob patterns to exclude from tracing (from LAST9_EXCLUDED_PATH_PATTERNS).
// Uses path.Match semantics (not filepath.Match).
// Default: /*/health,/*/healthz,/*/metrics,/*/ready,/*/live,/*/ping
// Set LAST9_EXCLUDED_PATH_PATTERNS="" to disable defaults.
ExcludedPathPatterns []string
}

// Load reads configuration from environment variables.
Expand All @@ -53,6 +68,20 @@ func Load() *Config {
cfg.ServiceVersion,
)

// Parse route exclusion configuration
cfg.ExcludedPaths = parseCommaSeparatedWithDefault(
"LAST9_EXCLUDED_PATHS",
"/health,/healthz,/metrics,/ready,/live,/ping",
)
cfg.ExcludedPathPrefixes = parseCommaSeparatedWithDefault(
"LAST9_EXCLUDED_PATH_PREFIXES",
"",
)
cfg.ExcludedPathPatterns = parseCommaSeparatedWithDefault(
"LAST9_EXCLUDED_PATH_PATTERNS",
"/*/health,/*/healthz,/*/metrics,/*/ready,/*/live,/*/ping",
)

// Validate configuration
if cfg.Endpoint == "" {
log.Println("[Last9 Agent] Warning: OTEL_EXPORTER_OTLP_ENDPOINT not set - telemetry will not be exported")
Expand Down Expand Up @@ -116,6 +145,28 @@ func parseResourceAttributes(attrsStr string, serviceVersion string) ([]attribut
return attrs, environment, serviceVersion
}

// parseCommaSeparatedWithDefault reads an env var and splits it by comma.
// If the env var is not set, defaultVal is used. If the env var is explicitly
// set to "", an empty slice is returned (opt-out of defaults).
func parseCommaSeparatedWithDefault(envKey, defaultVal string) []string {
raw, ok := os.LookupEnv(envKey)
if !ok {
raw = defaultVal
}
if raw == "" {
return nil
}

parts := strings.Split(raw, ",")
result := make([]string, 0, len(parts))
for _, p := range parts {
if v := strings.TrimSpace(p); v != "" {
result = append(result, v)
}
}
return result
}

// getEnvOrDefault returns the environment variable value or a default
func getEnvOrDefault(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
Expand Down
138 changes: 138 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
package config

import (
"os"
"reflect"
"testing"
)

func TestParseCommaSeparatedWithDefault(t *testing.T) {
tests := []struct {
name string
envKey string
envVal *string // nil = not set
defaultVal string
want []string
}{
{
name: "env not set uses default",
envKey: "TEST_PARSE_CSV_1",
envVal: nil,
defaultVal: "/health,/metrics",
want: []string{"/health", "/metrics"},
},
{
name: "env set to empty opts out",
envKey: "TEST_PARSE_CSV_2",
envVal: strPtr(""),
defaultVal: "/health,/metrics",
want: nil,
},
{
name: "env set to custom value",
envKey: "TEST_PARSE_CSV_3",
envVal: strPtr("/custom,/paths"),
defaultVal: "/health,/metrics",
want: []string{"/custom", "/paths"},
},
{
name: "whitespace trimmed",
envKey: "TEST_PARSE_CSV_4",
envVal: strPtr(" /health , /metrics "),
defaultVal: "",
want: []string{"/health", "/metrics"},
},
{
name: "empty default with env not set",
envKey: "TEST_PARSE_CSV_5",
envVal: nil,
defaultVal: "",
want: nil,
},
{
name: "trailing comma ignored",
envKey: "TEST_PARSE_CSV_6",
envVal: strPtr("/health,,/metrics,"),
defaultVal: "",
want: []string{"/health", "/metrics"},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
os.Unsetenv(tt.envKey)
if tt.envVal != nil {
os.Setenv(tt.envKey, *tt.envVal)
defer os.Unsetenv(tt.envKey)
}

got := parseCommaSeparatedWithDefault(tt.envKey, tt.defaultVal)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseCommaSeparatedWithDefault(%q) = %v, want %v", tt.envKey, got, tt.want)
}
})
}
}

func TestLoadExcludedPathsDefaults(t *testing.T) {
// Ensure env vars are not set so defaults apply
os.Unsetenv("LAST9_EXCLUDED_PATHS")
os.Unsetenv("LAST9_EXCLUDED_PATH_PREFIXES")
os.Unsetenv("LAST9_EXCLUDED_PATH_PATTERNS")

cfg := Load()

expectedPaths := []string{"/health", "/healthz", "/metrics", "/ready", "/live", "/ping"}
if !reflect.DeepEqual(cfg.ExcludedPaths, expectedPaths) {
t.Errorf("ExcludedPaths = %v, want %v", cfg.ExcludedPaths, expectedPaths)
}

if cfg.ExcludedPathPrefixes != nil {
t.Errorf("ExcludedPathPrefixes = %v, want nil", cfg.ExcludedPathPrefixes)
}

expectedPatterns := []string{"/*/health", "/*/healthz", "/*/metrics", "/*/ready", "/*/live", "/*/ping"}
if !reflect.DeepEqual(cfg.ExcludedPathPatterns, expectedPatterns) {
t.Errorf("ExcludedPathPatterns = %v, want %v", cfg.ExcludedPathPatterns, expectedPatterns)
}
}

func TestLoadExcludedPathsOptOut(t *testing.T) {
// Set env vars to empty to opt out
os.Setenv("LAST9_EXCLUDED_PATHS", "")
os.Setenv("LAST9_EXCLUDED_PATH_PATTERNS", "")
defer os.Unsetenv("LAST9_EXCLUDED_PATHS")
defer os.Unsetenv("LAST9_EXCLUDED_PATH_PATTERNS")

cfg := Load()

if cfg.ExcludedPaths != nil {
t.Errorf("ExcludedPaths = %v, want nil (opted out)", cfg.ExcludedPaths)
}
if cfg.ExcludedPathPatterns != nil {
t.Errorf("ExcludedPathPatterns = %v, want nil (opted out)", cfg.ExcludedPathPatterns)
}
}

func TestLoadExcludedPathsCustom(t *testing.T) {
os.Setenv("LAST9_EXCLUDED_PATHS", "/custom-health")
os.Setenv("LAST9_EXCLUDED_PATH_PREFIXES", "/internal/")
os.Setenv("LAST9_EXCLUDED_PATH_PATTERNS", "/v*/status")
defer os.Unsetenv("LAST9_EXCLUDED_PATHS")
defer os.Unsetenv("LAST9_EXCLUDED_PATH_PREFIXES")
defer os.Unsetenv("LAST9_EXCLUDED_PATH_PATTERNS")

cfg := Load()

if !reflect.DeepEqual(cfg.ExcludedPaths, []string{"/custom-health"}) {
t.Errorf("ExcludedPaths = %v, want [/custom-health]", cfg.ExcludedPaths)
}
if !reflect.DeepEqual(cfg.ExcludedPathPrefixes, []string{"/internal/"}) {
t.Errorf("ExcludedPathPrefixes = %v, want [/internal/]", cfg.ExcludedPathPrefixes)
}
if !reflect.DeepEqual(cfg.ExcludedPathPatterns, []string{"/v*/status"}) {
t.Errorf("ExcludedPathPatterns = %v, want [/v*/status]", cfg.ExcludedPathPatterns)
}
}

func strPtr(s string) *string { return &s }
Loading
Loading