ArgoCD 可觀測性實戰 - Metrics¶

k3d cluster create --api-port 6443 \
--port 8080:80@loadbalancer --port 8443:443@loadbalancer

kubectl cluster-info

Kubernetes control plane is running at https://0.0.0.0:6443
CoreDNS is running at https://0.0.0.0:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
Metrics-server is running at https://0.0.0.0:6443/api/v1/namespaces/kube-system/services/https:metrics-server:/proxy

$ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
$ helm repo update 

grafana:
  ingress:
    ## If true, Grafana Ingress will be created
    enabled: true
    hosts: 
      - grafana.example.it
  # change timezone setting base on browser
  defaultDashboardsTimezone: browser
  grafana.ini:
    users:
      viewers_can_edit: true
    auth:
      disable_login_form: false
      disable_signout_menu: false
    auth.anonymous:
      enabled: true
      org_role: Viewer
  sidecar:
    datasources:
      logLevel: "DEBUG"
      enabled: true
      searchNamespace: "ALL"
    dashboards:
      logLevel: "DEBUG"
      # enable the cluster wide search for dashbaords and adds/updates/deletes them in grafana
      enabled: true
      searchNamespace: "ALL"
      label: grafana_dashboard
      labelValue: "1"

prometheus:
  ingress:
    ## If true, Grafana Ingress will be created
    enabled: true
    hosts: 
      - prometheus.example.it
  prometheusSpec:
    # enable the cluster wide search for ServiceMonitor CRD
    serviceMonitorSelectorNilUsesHelmValues: false
    # enable the cluster wide search for PodMonitor CRD
    podMonitorSelectorNilUsesHelmValues: false
    # enable the cluster wide search for PrometheusRule CRD
    ruleSelectorNilUsesHelmValues: false
    probeSelectorNilUsesHelmValues: false

alertmanager:
  ingress:
    ## If true, Grafana Ingress will be created
    enabled: true
    hosts: 
      - alertmanager.example.it

helm upgrade --install --wait --create-namespace --namespace monitoring  \
kube-stack-prometheus prometheus-community/kube-prometheus-stack \
--values kube-stack-prometheus-values.yaml

$ kubectl get ing -n monitoring

NAME                                      CLASS    HOSTS                     ADDRESS      PORTS   AGE
kube-stack-prometheus-kube-prometheus     <none>   prometheus.example.it     172.22.0.2   80      119s
kube-stack-prometheus-kube-alertmanager   <none>   alertmanager.example.it   172.22.0.2   80      119s
kube-stack-prometheus-grafana             <none>   grafana.example.it        172.22.0.2   80      119s

# 1.選擇使用本機的 127.0.0.1 作為南北向的進入點
#   這樣的Request會通過外部的LoadBalancer (port:8080)來存取
127.0.0.1   prometheus.example.it
127.0.0.1   alertmanager.example.it
127.0.0.1   grafana.example.it

## 或是

# 2.選擇 docker network 的虛擬網絡的 IP 作為南北向的進入點
#   要注意配置在K8S集群裡的LoadBalancer所取得的外部 IP
172.22.0.2   prometheus.example.it
172.22.0.2   alertmanager.example.it
172.22.0.2   grafana.example.it

kubectl port-forward --namespace monitoring svc/kube-stack-prometheus-kube-prometheus 9090:9090 --address="0.0.0.0"

kubectl port-forward svc/kube-stack-prometheus-kube-alertmanager 9093:9093 --address="0.0.0.0"

kubectl port-forward --namespace monitoring svc/kube-stack-prometheus-grafana 3000:80 --address="0.0.0.0"

helm repo add argo https://argoproj.github.io/argo-helm

helm repo update

## Argo CD server
server:
  extraArgs:
  # -- Run server without TLS
  - --insecure
  ingress:
    # -- Enable an ingress resource for the Argo CD server
    enabled: true
    # -- List of ingress hosts
    ## Argo Ingress.
    ## Hostnames must be provided if Ingress is enabled.
    ## Secrets must be manually created in the namespace
    hosts: 
      - argocd.example.it
  metrics:
    # -- Deploy metrics service
    enabled: true
    service:
      # -- Metrics service port
      servicePort: 8083
      # -- Metrics service port name
      portName: http-metrics
    serviceMonitor:
      # -- Enable a prometheus ServiceMonitor
      enabled: true

## Repo Server
repoServer:
  ## Repo server metrics service configuration
  metrics:
    # -- Deploy metrics service
    enabled: true
    service:
      # -- Metrics service port
      servicePort: 8084
      # -- Metrics service port name
      portName: http-metrics
    serviceMonitor:
      # -- Enable a prometheus ServiceMonitor
      enabled: true

## Application controller
controller:
  ## Application controller metrics configuration
  metrics:
    # -- Deploy metrics service
    enabled: true
    applicationLabels:
      # -- Enables additional labels in argocd_app_labels metric
      enabled: true
    service:
      # -- Metrics service port
      servicePort: 8082
      # -- Metrics service port name
      portName: http-metrics
    serviceMonitor:
      # -- Enable a prometheus ServiceMonitor
      enabled: true
    rules:
      # -- Deploy a PrometheusRule for the application controller
      enabled: true
      # -- PrometheusRule.Spec for the application controller
      spec:
       - alert: ArgoAppMissing
         expr: |
           absent(argocd_app_info) == 1
         for: 15m
         labels:
           severity: critical
         annotations:
           summary: "[Argo CD] No reported applications"
           description: >
             Argo CD has not reported any applications data for the past 15 minutes which
             means that it must be down or not functioning properly.  This needs to be
             resolved for this cloud to continue to maintain state.
       - alert: ArgoAppNotSynced
         expr: |
           argocd_app_info{sync_status!="Synced"} == 1
         for: 12h
         labels:
           severity: warning
         annotations:
           summary: "[{{`{{$labels.name}}`}}] Application not synchronized"
           description: >
             The application [{{`{{$labels.name}}`}} has not been synchronized for over
             12 hours which means that the state of this cloud has drifted away from the
             state inside Git.
      selector:
        prometheus: kube-prometheus
      namespace: monitoring

## ApplicationSet controller
applicationSet:
  ## Metrics service configuration
  metrics:
    # -- Deploy metrics service
    enabled: true
    service:
      # -- Metrics service port
      servicePort: 8085
      # -- Metrics service port name
      portName: http-metrics
    serviceMonitor:
      # -- Enable a prometheus ServiceMonitor
      enabled: true

helm upgrade --install --create-namespace --namespace argocd  \
     argocd argo/argo-cd \
     --values argocd-values.yaml

$ kubectl get ing -n argocd

NAME            CLASS    HOSTS               ADDRESS      PORTS   AGE
argocd-server   <none>   argocd.example.it   172.22.0.2   80      19s

# 1.選擇使用本機的 127.0.0.1 作為南北向的進入點
#   這樣的Request會通過外部的LoadBalancer (port:8080)來存取
127.0.0.1   argocd.example.it

## 或是

# 2.選擇 docker network 的虛擬網絡的 IP 作為南北向的進入點
#   要注意配置在K8S集群裡的LoadBalancer所取得的外部 IP
172.22.0.2   argocd.example.it

kubectl port-forward service/argocd-server -n argocd 8080:80

kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d

4geyMPlYJQvp-hj6

$ kubectl get svc -n argocd

NAME                                       TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)             AGE
argocd-redis                               ClusterIP   10.43.173.60    <none>        6379/TCP            138m
argocd-applicationset-controller           ClusterIP   10.43.110.159   <none>        7000/TCP            138m
argocd-repo-server                         ClusterIP   10.43.140.104   <none>        8081/TCP            138m
argocd-server                              ClusterIP   10.43.209.235   <none>        80/TCP,443/TCP      138m
argocd-dex-server                          ClusterIP   10.43.219.251   <none>        5556/TCP,5557/TCP   138m
argocd-server-metrics                      ClusterIP   10.43.80.20     <none>        8083/TCP            46m
argocd-application-controller-metrics      ClusterIP   10.43.215.81    <none>        8082/TCP            22m
argocd-applicationset-controller-metrics   ClusterIP   10.43.11.97     <none>        8085/TCP            22m
argocd-repo-server-metrics                 ClusterIP   10.43.144.153   <none>        8084/TCP            22m

$ kubectl get servicemonitor -n argocd

NAME                               AGE
argocd-server                      49m
argocd-application-controller      24m
argocd-applicationset-controller   24m
argocd-repo-server                 24m

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: argocd-application-controller
  namespace: monitoring
spec:
  groups:
  - name: argocd
    partial_response_strategy: ""
    rules:
    - alert: ArgoAppMissing
      annotations:
        description: |
          Argo CD has not reported any applications data for the past 15 minutes which means that it must be down or not functioning properly.  This needs to be resolved for this cloud to continue to maintain state.
        summary: '[Argo CD] No reported applications'
      expr: |
        absent(argocd_app_info) == 1
      for: 15m
      labels:
        severity: critical
    - alert: ArgoAppNotSynced
      annotations:
        description: |
          The application [{{`{{$labels.name}}`}} has not been synchronized for over 12 hours which means that the state of this cloud has drifted away from the state inside Git.
        summary: '[{{`{{$labels.name}}`}}] Application not synchronized'
      expr: |
        argocd_app_info{sync_status!="Synced"} == 1
      for: 12h
      labels:
        severity: warning

wget https://raw.githubusercontent.com/argoproj/argo-cd/master/examples/dashboard.json

kubectl create configmap cm-argocd-dashboard \
  --from-file=dashboard.json \
  --dry-run=client -o yaml > cm-argocd-dashboard.yaml

apiVersion: v1
data:
  dashboard.json: |-
    ...
    ...
kind: ConfigMap
metadata:
  name: cm-argocd-dashboard.yaml
  labels:
    grafana_dashboard: "1"

kubectl apply -f cm-argocd-dashboard.yaml

argocd_app_reconcile_count