Multiple Cluster Monitoring Setup Issue

[bbenson29]

Goal - To be able to monitor nodes, pods, metric etc running in multiple regions from a central cluster (Observer)
What is happening - At the moment I can only monitor the observer cluster.
What I expect to happen - Thanos query (Observer) queries other clusters in different networks and providers. Cluster can monitor

• Version => Helm chart v8.0.3
• Thanos running on 1.21.5-gke.1802
• Thanos components deployed on Observer cluster
  • compactor
  • query
  • query-frontend
  • storegateway
• ExternalDNS
• Ingress (Ingress Nginx)

Observer cluster helm values files

thanos-values.yaml

        compactor:
          enabled: true
        
        storegateway:
          enabled: true
        
        query:
          ingress:
            enabled: true
            ingressClassName: nginx
            hostname: tq.icezone.domain.example
            tls:
              - secretName: thanos-query-general-tls
                hosts:
                  - tq.icezone.domain.example
            annotations:
              cert-manager.io/cluster-issuer: stage-acme-issuer
              nginx.ingress.kubernetes.io/auth-url: "https://auth.icezone.domain.example/oauth2/auth"
              nginx.ingress.kubernetes.io/auth-signin: "https://auth.icezone.domain.example/oauth2/start?rd=$scheme://$best_http_host$request_uri"
          # adding envoy container
          sidecars:
            - name: envoy-sidecar
              image: 'envoyproxy/envoy:v1.16.0'
              args:
                - '-c'
                - /config/envoy.yaml
                - '-l'
                - debug
              ports:
                - name: envoy
                  containerPort: 10000
                  protocol: TCP
                - name: myts-workload
                  containerPort: 10001
                  protocol: TCP
              resources: {}
              volumeMounts:
                - name: config
                  mountPath: /config
                  mountPropagation: None
                - name: certs
                  mountPath: /certs
                  mountPropagation: None
              terminationMessagePath: /dev/termination-log
              terminationMessagePolicy: File
              imagePullPolicy: IfNotPresent
          # Volume for envoy
          extraVolumes:
            - name: config
              configMap:
                name: envoy-conf
                defaultMode: 420
                optional: false
            - name: certs
              secret:
                secretName: thanos-global-test-envoy-certs
                defaultMode: 420
                optional: false
          # Added section for envoy ports configuration on service
          service:
          
            extraPorts:
            - name: envoy
              protocol: TCP
              port: 10000
              targetPort: 10000
            - name: myts-workload
              protocol: TCP
              port: 10001
              targetPort: 10001
          
          
        
        
          dnsDiscovery:
            sidecarsService: "monitor-stack-kube-prometh-thanos-discovery"
            sidecarsNamespace: "monitoring"
          stores:
            - dnssrv+_envoy._tcp.thanos-query.monitoring.svc.cluster.local
            - dnssrv+_myts-workload._tcp.thanos-query.monitoring.svc.cluster.local
            - small-sidecar.google-test.domain.example:443
          grpcTLS:
            client:
              secure: true
              existingSecret:
                name: thanos-global-test-envoy-certs
                keyMapping:
                  ca-cert: ca.crt
                  tls-cert: tls.crt
                  tls-key: tls.key
        
        existingObjstoreSecret: thanos-objstore-config

prometheus-values.yamls

     defaultRules:
       create: true
       rules:
         kubeScheduler: false
         etcd: false
     
     
     alertmanager:
       ingress:
         enabled: true
         ingressClassName: nginx
         hosts:
           - am.icezone.domain.example
         tls:
           - secretName: alertmanager-general-tls
             hosts:
               - am.icezone.domain.example
         annotations:
           cert-manager.io/cluster-issuer: stage-acme-issuer
           nginx.ingress.kubernetes.io/auth-url: "https://auth.icezone.domain.example/oauth2/auth"
           nginx.ingress.kubernetes.io/auth-signin: "https://auth.icezone.domain.example/oauth2/start?rd=$scheme://$best_http_host$request_uri"
     
       alertmanagerSpec:
         image:
           tag: v0.22.2
         storage:
           volumeClaimTemplate:
             spec:
               storageClassName: monitor-storageclass
               accessModes: ["ReadWriteOnce"]
               resources:
                 requests:
                   storage: 10Gi
     
     ## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml
     ##
     grafana:
       sidecar:
         dashboards:
         #  multicluster: true
           enabled: true
         datasources:
           enabled: true
       dashboardProviders:
         dashboardproviders.yaml:
           apiVersion: 1
           providers:
           - name: 'cost'
             orgId: 1
             folder: 'Costs Overview'
             type: file
             disableDeletion: false
             editable: true
             options:
               path: /var/lib/grafana/dashboards/costsOverview
           - name: 'domainapps'
             orgId: 1
             folder: 'domainapps'
             type: file
             disableDeletion: true
             editable: false
             options:
               path: /var/lib/grafana/dashboards/domainapps
       dashboards:
         domainapps:
           blackbox:
           # https://grafana.com/grafana/dashboards/7587
             gnetId: 7587
             revision: 3
             datasource: thanos
           ingress:
           # https://grafana.com/grafana/dashboards/9614
             gnetId: 9614
             revision: 1
             datasource: thanos
       adminPassword: ""
       persistence:
         type: pvc
         enabled: true
         storageClassName: monitor-storageclass
         accessModes:
           - ReadWriteOnce
         size: 10Gi
         finalizers:
           - kubernetes.io/pvc-protection
       additionalDataSources:
       - name: thanos
         type: prometheus
         url: http://thanos-query:9090
         isDefault: false
     
     #  ingress:
       ingress:
         enabled: true
         ingressClassName: nginx
         annotations:
           cert-manager.io/cluster-issuer: "stage-acme-issuer"
           acme.cert-manager.io/http01-ingress-class: "nginx"
         hosts:
         #  - gf.icezone.domain.example # grafana host name
           - grafana.mon.domain.app
         path: /
         tls:
           - secretName: grafana-general-tls
             hosts:
             #  - gf.icezone.domain.example
               - grafana.mon.domain.app
       grafana.ini:
         server:
           root_url: https://grafana.mon.domain.app
         auth.azuread:
           
       extraSecretMounts:
         - name: auth-generic-oauth-mount
           secretName: auth-generic-oauth
           defaultMode: 0440
           mountPath: /etc/secrets/auth_generic_oauth
           readOnly: true
     
     # beause we use GKE we don't have access to this
     kubeControllerManager:
       enabled: false
     kubeProxy:
       enabled: false
     kubeEtcd:
       enabled: false
     kubeScheduler:
       enabled: false
     coreDns:
       enabled: false
     
     prometheusOperator:
       image:
         tag: v0.52.0
     
       thanosImage:
         repository: quay.io/thanos/thanos
         tag: v0.17.2
     
     prometheus:
       ingress:
         enabled: true
         ingressClassName: nginx
         hosts:
           - prom.icezone.domain.example
         tls:
           - secretName: prometheus-general-tls
             hosts:
               - prom.icezone.domain.example
         annotations:
           cert-manager.io/cluster-issuer: stage-acme-issuer
           nginx.ingress.kubernetes.io/auth-url: "https://auth.icezone.domain.example/oauth2/auth"
           nginx.ingress.kubernetes.io/auth-signin: "https://auth.icezone.domain.example/oauth2/start?rd=$scheme://$best_http_host$request_uri"
     
       thanosService:
         enabled: true
     
       prometheusSpec:
         externalLabels:
           cluster: prometheus-cluster
           owner: domain
         image:
           tag: v2.28.1
     
         retention: 10d
         storageSpec:
           volumeClaimTemplate:
             spec:
               storageClassName: monitor-storageclass
               accessModes: ["ReadWriteOnce"]
               resources:
                 requests:
                   storage: 50Gi
     
         thanos:  # Enabling Sidecar for thanos
           image: quay.io/thanos/thanos:v0.23.1
           objectStorageConfig:
             key: objstore.yml
             name: thanos-objstore-config
     
         # without these, creating a Probe wasn't working
         serviceMonitorSelectorNilUsesHelmValues: false
         podMonitorSelectorNilUsesHelmValues: false
         probeSelectorNilUsesHelmValues: false
         # this is so labels aren't needed for creating Prometheus Rules
         ruleSelectorNilUsesHelmValues: false

envoy.yaml

apiVersion: v1
kind: ConfigMap
metadata:
    name: envoy-conf
    namespace: monitoring
data:
    envoy.yaml: |
    admin:
        access_log_path: /tmp/admin_access.log
        address:
        socket_address: { address: 0.0.0.0, port_value: 9901 }

    static_resources:
        listeners:
        - name: small-sidecar
        address:
            socket_address:
            address: 0.0.0.0
            port_value: 10000
        filter_chains:
        - filters:
            - name: envoy.http_connection_manager
            typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                codec_type: AUTO
                access_log:
                - name: envoy.access_loggers.file
                typed_config:
                    "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
                    path: /dev/stdout
                    log_format:
                    text_format: |
                        [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%"
                        %RESPONSE_CODE% %RESPONSE_FLAGS% %RESPONSE_CODE_DETAILS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION%
                        %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%"
                        "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" "%UPSTREAM_TRANSPORT_FAILURE_REASON%"\n                  
                - name: envoy.access_loggers.file
                typed_config:
                    "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
                    path: /dev/stdout
                stat_prefix: ingress_http
                route_config:
                name: local_route
                virtual_hosts:
                - name: local_service
                    domains: ["*"]
                    routes:
                    - match:
                        prefix: "/"
                        grpc: {}
                    route:
                        cluster: small-sidecar
                        host_rewrite_literal: small-sidecar.google-test.domain.example
                http_filters:
                - name: envoy.filters.http.router
        clusters:
        - name: small-sidecar
        connect_timeout: 30s
        type: logical_dns
        http2_protocol_options: {}
        dns_lookup_family: V4_ONLY
        lb_policy: ROUND_ROBIN
        load_assignment:
            cluster_name: small-sidecar
            endpoints:
            - lb_endpoints:
                - endpoint:
                    address:
                    socket_address:
                        address: small-sidecar.google-test.domain.example
                        port_value: 443
        transport_socket:
            name: envoy.transport_sockets.tls
            typed_config:
            "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
            common_tls_context:
                tls_certificates:
                - certificate_chain:
                    filename: /certs/tls.crt
                    private_key:
                    filename: /certs/tls.key
                validation_context:
                trusted_ca:
                    filename: /certs/ca.crt
                alpn_protocols:
                - h2
                - http/1.1
            sni: small-sidecar.google-test.domain.example

kubectl logs -n monitoring thanos-query-84db586988-7k9fx -f --tail 120 query

    level=warn ts=2022-01-17T14:13:20.117590423Z caller=endpointset.go:525 component=endpointset msg="update of node failed" err="getting metadata: fallback fetching info from 100.65.140.66:10001: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 100.65.140.66:10001: connect: connection refused\"" address=100.65.140.66:10001
    level=warn ts=2022-01-17T14:13:20.131960988Z caller=endpointset.go:525 component=endpointset msg="update of node failed" err="getting metadata: fallback fetching info from 100.65.140.66:10000: rpc error: code = Unavailable desc = Bad Gateway: HTTP status code 502; transport: received the unexpected content-type \"text/html\"" address=100.65.140.66:10000
    level=warn ts=2022-01-17T14:13:20.14445307Z caller=endpointset.go:525 component=endpointset msg="update of node failed" err="getting metadata: fallback fetching info from small-sidecar.google-test.domain.example:443: rpc error: code = Unavailable desc = connection closed" address=small-sidecar.google-test.domain.example:443

kubectl logs -n monitoring thanos-query-84db586988-7k9fx -f --tail envoy-sidecar

    [2022-01-17 14:16:00.103][8][debug][main] [source/server/server.cc:190] flushing stats
    [2022-01-17 14:16:00.117][16][debug][conn_handler] [source/server/connection_handler_impl.cc:459] [C46] new connection
    [2022-01-17 14:16:00.117][16][debug][http2] [source/common/http/http2/codec_impl_legacy.cc:1140] [C46] updating connection-level initial window size to 268435456
    [2022-01-17 14:16:00.118][16][debug][http] [source/common/http/conn_manager_impl.cc:225] [C46] new stream
    [2022-01-17 14:16:00.118][16][debug][http] [source/common/http/conn_manager_impl.cc:837] [C46][S6858705958373302663] request headers complete (end_stream=false):
    ':method', 'POST'
    ':scheme', 'http'
    ':path', '/thanos.Store/Info'
    ':authority', '100.65.140.66:10000'
    'content-type', 'application/grpc'
    'user-agent', 'grpc-go/1.29.1'
    'te', 'trailers'
    'grpc-timeout', '4998293u'

    [2022-01-17 14:16:00.118][16][debug][router] [source/common/router/router.cc:429] [C46][S6858705958373302663] cluster 'small-sidecar' match for URL '/thanos.Store/Info'
    [2022-01-17 14:16:00.118][16][debug][router] [source/common/router/router.cc:586] [C46][S6858705958373302663] router decoding headers:
    ':method', 'POST'
    ':scheme', 'https'
    ':path', '/thanos.Store/Info'
    ':authority', 'small-sidecar.google-test.domain.example'
    'content-type', 'application/grpc'
    'user-agent', 'grpc-go/1.29.1'
    'te', 'trailers'
    'grpc-timeout', '4998293u'
    'x-forwarded-proto', 'http'
    'x-request-id', '9430eda1-c7a0-426f-a7a9-9e93c722fcf9'
    'x-envoy-expected-rq-timeout-ms', '15000'

    [2022-01-17 14:16:00.118][16][debug][pool] [source/common/conn_pool/conn_pool_base.cc:174] [C1] using existing connection
    [2022-01-17 14:16:00.118][16][debug][pool] [source/common/conn_pool/conn_pool_base.cc:126] [C1] creating stream
    [2022-01-17 14:16:00.118][16][debug][router] [source/common/router/upstream_request.cc:357] [C46][S6858705958373302663] pool ready
    [2022-01-17 14:16:00.118][16][debug][http] [source/common/http/filter_manager.cc:721] [C46][S6858705958373302663] request end stream
    [2022-01-17 14:16:00.120][16][debug][router] [source/common/router/router.cc:1178] [C46][S6858705958373302663] upstream headers complete: end_stream=false
    [2022-01-17 14:16:00.121][16][debug][http] [source/common/http/conn_manager_impl.cc:1435] [C46][S6858705958373302663] encoding headers via codec (end_stream=false):
    ':status', '502'
    'date', 'Mon, 17 Jan 2022 14:16:00 GMT'
    'content-type', 'text/html'
    'content-length', '150'
    'strict-transport-security', 'max-age=15724800; includeSubDomains'
    'x-envoy-upstream-service-time', '2'
    'server', 'envoy'

    [2022-01-17 14:16:00.121][16][debug][client] [source/common/http/codec_client.cc:109] [C1] response complete
    [2022-01-17 14:16:00.121][16][debug][pool] [source/common/conn_pool/conn_pool_base.cc:151] [C1] destroying stream: 0 remaining
    [2022-01-17 14:16:00.121][16][debug][http2] [source/common/http/http2/codec_impl_legacy.cc:933] [C46] stream closed: 0
    [2022-01-17 14:16:00.121][16][debug][http2] [source/common/http/http2/codec_impl.cc:964] [C1] stream closed: 0
    [2022-01-17 14:16:00.121][16][debug][connection] [source/common/network/connection_impl.cc:579] [C46] remote close
    [2022-01-17 14:16:00.121][16][debug][connection] [source/common/network/connection_impl.cc:202] [C46] closing socket: 0
    [2022-01-17 14:16:00.121][16][debug][conn_handler] [source/server/connection_handler_impl.cc:152] [C46] adding to cleanup list
    [2022-01-17T14:15:55.119Z] "POST /thanos.Store/Info HTTP/2"
    502 - via_upstream 5 150 2
    2 "-" "grpc-go/1.29.1"
    "6badba22-3447-4d89-85d6-96721a5abfd0" "small-sidecar.google-test.domain.example" "34.X.X.X:443" "-"\n                  
    [2022-01-17T14:15:55.119Z] "POST /thanos.Store/Info HTTP/2" 502 - 5 150 2 2 "-" "grpc-go/1.29.1" "6badba22-3447-4d89-85d6-96721a5abfd0" "small-sidecar.google-test.domain.example" "34.X.X.X:443"
    [2022-01-17T14:16:00.118Z] "POST /thanos.Store/Info HTTP/2"
    502 - via_upstream 5 150 2
    2 "-" "grpc-go/1.29.1"
    "9430eda1-c7a0-426f-a7a9-9e93c722fcf9" "small-sidecar.google-test.domain.example" "34.X.X.X:443" "-"\n                  
    [2022-01-17T14:16:00.118Z] "POST /thanos.Store/Info HTTP/2" 502 - 5 150 2 2 "-" "grpc-go/1.29.1" "9430eda1-c7a0-426f-a7a9-9e93c722fcf9" "small-sidecar.google-test.domain.example" "34.X.X.X:443"
    [2022-01-17 14:16:00.642][8][debug][upstream] [source/common/upstream/logical_dns_cluster.cc:100] starting async DNS resolution for small-sidecar.google-test.domain.example
    [2022-01-17 14:16:00.647][8][debug][upstream] [source/common/upstream/logical_dns_cluster.cc:108] async DNS resolution complete for small-sidecar.google-test.domain.example
    [2022-01-17 14:16:00.647][8][debug][upstream] [source/common/upstream/logical_dns_cluster.cc:153] DNS refresh rate reset for small-sidecar.google-test.domain.example, refresh rate 5000 ms

Client Cluster

• Network - Different Network (no peering)
• Cluster - 1.21.5-gke.1802
• Stack - prometheus-community/kube-prometheus-stack version 20.0.1
• Proxy - Envoyproxy

client-cluster-prometheus.yaml

     commonLabels: 
       cluster: test-small-cluster
     alertmanager:
       enabled: false
     grafana:
       enabled: false
     prometheus:  
       thanosIngress:
         enabled: true 
         ingressClassName: nginx
         annotations: 
           #nginx.ingress.kubernetes.io/ssl-redirect: "true"
           nginx.ingress.kubernetes.io/force-ssl-redirect: 'true'
           nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
           nginx.ingress.kubernetes.io/auth-tls-secret: "monitoring/thanos-ca-secret"
           #nginx.ingress.kubernetes.io/ssl-passthrough: "true"
           #nginx.ingress.kubernetes.io/auth-tls-verify-client: "on"
           nginx.ingress.kubernetes.io/protocol: h2c
           nginx.ingress.kubernetes.io/proxy-read-timeout: '160'
         labels: {}
         servicePort: 10901
         hosts:
         -  small-sidecar.google-test.domain.example
         paths: 
         - /
         pathType: ImplementationSpecific
         tls: 
         - hosts:
           - small-sidecar.google-test.domain.example
           secretName: thanos-ingress-tls
       
       thanosService:
         enabled: true
         annotations: {}
         labels: {}
         portName: grpc
         port: 10901
         targetPort: "grpc"
         clusterIP: "None"
     
         ## Service type
         ##
         type: ClusterIP
     
         ## Port to expose on each node
         ##
         nodePort: 30901
     
         # Service for external access to sidecar
         # Enabling this creates a service to expose thanos-sidecar outside the cluster.
       thanosServiceExternal:
         enabled: false
         annotations:
           #external-dns.alpha.kubernetes.io/hostname: thanos-small-1.google-test.domain.example
           # kubernetes.io/ingress.class: "nginx"
           # cert-manager.io/cluster-issuer: "acme-clusterissuer-stage"
           # acme.cert-manager.io/http01-ingress-class: "nginx"
         labels: {}
         portName: grpc
         port: 10901
         targetPort: "grpc"
     
         ## Service type
         ##
         type: LoadBalancer
       thanosService:
         enabled: true
       prometheusSpec:
         image:
           tag: v2.28.1
           retention: 10d
           
         externalLabels: 
           cluster: small-cluster
           platfrom: gke
           owner: ICE
           team: ICE
           region: london
         
         serviceMonitorNamespaceSelector: 
           matchExpressions:
           - key: "non-existent-label"
             operator: "DoesNotExist"
         thanos:
           tag: v0.21.1
           objectStorageConfig:
             key: objstore.yml
             name: thanos-objstore-config
           grpcServerTlsConfig:
             caFile: "/etc/tls/grpc/ca.crt"
             certFile: "/etc/tls/grpc/tls.crt"
             keyFile: "/etc/tls/grpc/tls.key"
         volumes:
         - name: thanos-sidecar-tls
           secret:
             secretName: thanos-ca-secret
         containers:
         - name: thanos-sidecar
           volumeMounts:
           - mountPath: /etc/tls/grpc
             name: thanos-sidecar-tls
         
         storageSpec: 
           volumeClaimTemplate:
             spec:
               storageClassName: monitor-test-sc
               accessModes: ["ReadWriteOnce"]
               resources:
                 requests:
                   storage: 50Gi

[bbenson29]

Would like some help on this. tnx