Merge pull request #361 from kartverket/monitoring

Monitoring

README.md

@@ -180,8 +180,12 @@ spec:
     activeDeadlineSeconds: 10
     backoffLimit: 10
     suspend: false
-    ttlSecondsAfterFinished: 10
+    ttlSecondsAfterFinished: 
     
+  prometheus:
+    path: /metrics
+    port: 8080  
+  
   container:
     # Pod
     image: ""

api/v1alpha1/skipjob_types.go

@@ -63,6 +63,10 @@ type SKIPJobSpec struct {
 	//
 	// +kubebuilder:validation:Required
 	Container ContainerSettings `json:"container"`
+
+	// Prometheus settings for pod running in job. Fields are identical to Application and if set,
+	// a monitorngs object is created.
+	Prometheus *PrometheusConfig `json:"prometheus,omitempty"`
 }
 
 // +kubebuilder:object:generate=true

config/crd/skiperator.kartverket.no_skipjobs.yaml

@@ -674,6 +674,25 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              prometheus:
+                description: Prometheus settings for pod running in job. Fields are
+                  identical to Application and if set, a monitorngs object is created.
+                properties:
+                  path:
+                    default: /metrics
+                    description: The HTTP path where Prometheus compatible metrics
+                      exists
+                    type: string
+                  port:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: The port number or name where metrics are exposed
+                      (at the Pod level).
+                    x-kubernetes-int-or-string: true
+                required:
+                - port
+                type: object
             required:
             - container
             type: object

config/rbac/role.yaml

@@ -121,6 +121,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - podmonitors
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - monitoring.coreos.com
   resources:

controllers/application/controller.go

@@ -45,6 +45,7 @@ import (
 // +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get
 // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=podmonitors,verbs=get;list;watch;create;update;patch;delete
 
 type ApplicationReconciler struct {
 	util.ReconcilerBase

controllers/skipjob/controller.go

@@ -107,6 +107,7 @@ func (r *SKIPJobReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		r.reconcileEgressServiceEntry,
 		r.reconcileConfigMap,
 		r.reconcileJob,
+		r.reconcilePodMonitor,
 	}
 
 	for _, fn := range controllerDuties {

controllers/skipjob/job.go

@@ -43,6 +43,17 @@ func (r *SKIPJobReconciler) reconcileJob(ctx context.Context, skipJob *skiperato
 	if err != nil {
 		return reconcile.Result{}, err
 	}
+	// By specifying port and path annotations, Istio will scrape metrics from the application
+	// and merge it together with its own metrics.
+	//
+	// See
+	//  - https://superorbital.io/blog/istio-metrics-merging/
+	//  - https://androidexample365.com/an-example-of-how-istio-metrics-merging-works/
+	istioEnabled := r.IsIstioEnabledForNamespace(ctx, skipJob.Namespace)
+	if istioEnabled && skipJob.Spec.Prometheus != nil {
+		skipJob.Annotations["prometheus.io/port"] = skipJob.Spec.Prometheus.Port.StrVal
+		skipJob.Annotations["prometheus.io/path"] = skipJob.Spec.Prometheus.Path
+	}
 
 	if skipJob.Spec.Cron != nil {
 		err = r.GetClient().Get(ctx, types.NamespacedName{

controllers/skipjob/pod_monitor.go

@@ -0,0 +1,66 @@
+package skipjobcontroller
+
+import (
+	"context"
+	skiperatorv1alpha1 "github.com/kartverket/skiperator/api/v1alpha1"
+	"github.com/kartverket/skiperator/pkg/util"
+	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+func (r *SKIPJobReconciler) reconcilePodMonitor(ctx context.Context, skipJob *skiperatorv1alpha1.SKIPJob) (reconcile.Result, error) {
+	podMonitor := pov1.PodMonitor{ObjectMeta: metav1.ObjectMeta{
+		Name:      skipJob.Name + "-monitor",
+		Namespace: skipJob.Namespace,
+		Labels:    map[string]string{"instance": "primary"},
+	}}
+
+	shouldReconcile, err := r.ShouldReconcile(ctx, &podMonitor)
+	if err != nil || !shouldReconcile {
+		return reconcile.Result{}, err
+	}
+
+	if skipJob.Spec.Prometheus == nil {
+		err := client.IgnoreNotFound(r.GetClient().Delete(ctx, &podMonitor))
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+		return reconcile.Result{}, nil
+	}
+
+	_, err = ctrlutil.CreateOrPatch(ctx, r.GetClient(), &podMonitor, func() error {
+		err := ctrlutil.SetControllerReference(skipJob, &podMonitor, r.GetScheme())
+		if err != nil {
+			return err
+		}
+		podMonitor.Spec = pov1.PodMonitorSpec{
+			Selector: metav1.LabelSelector{
+				MatchLabels: util.GetPodAppSelector(skipJob.Name),
+			},
+			NamespaceSelector: pov1.NamespaceSelector{
+				MatchNames: []string{skipJob.Namespace},
+			},
+			PodMetricsEndpoints: r.determineEndpoint(ctx, skipJob),
+		}
+		return nil
+	})
+	return reconcile.Result{}, err
+}
+
+func (r *SKIPJobReconciler) determineEndpoint(ctx context.Context, application *skiperatorv1alpha1.SKIPJob) []pov1.PodMetricsEndpoint {
+	ep := pov1.PodMetricsEndpoint{
+		Path: util.IstioMetricsPath, TargetPort: &util.IstioMetricsPortName,
+	}
+	if r.IsIstioEnabledForNamespace(ctx, application.Namespace) {
+		return []pov1.PodMetricsEndpoint{ep}
+	}
+	return []pov1.PodMetricsEndpoint{
+		{
+			Path:       application.Spec.Prometheus.Path,
+			TargetPort: &application.Spec.Prometheus.Port,
+		},
+	}
+}

pkg/util/reconciler.go

@@ -120,9 +120,9 @@ func (r *ReconcilerBase) IsIstioEnabledForNamespace(ctx context.Context, namespa
 		return false
 	}
 
-	_, exists := namespace.Labels[IstioRevisionLabel]
+	v, exists := namespace.Labels[IstioRevisionLabel]
 
-	return exists
+	return exists && len(v) > 0
 }
 
 func hasIgnoreLabel(obj client.Object) bool {

samples/skipjob-metrics.yaml

@@ -0,0 +1,12 @@
+apiVersion: skiperator.kartverket.no/v1alpha1
+kind: SKIPJob
+metadata:
+  namespace: sample
+  name: sample-job-metrics
+spec:
+  container:
+    image: perl:5.34.0
+    command: ["perl",  "-Mbignum=bpi", "-wle", "print bpi(2000)"]
+  prometheus:
+    port: 8080
+    path: "/metrics"

tests/application/service-monitor/05-remove-istio.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test
+  labels:
+    istio.io/rev: ""

tests/skipjob/podmonitor/00-assert.yaml

@@ -0,0 +1,60 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: podmonitor-skipjob
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: podmonitor
+  labels:
+    app: podmonitor-skipjob
+    skiperator.kartverket.no/skipjob: "true"
+    skiperator.kartverket.no/skipjobName: podmonitor
+spec:
+  suspend: false
+  backoffLimit: 6
+  completionMode: NonIndexed
+  parallelism: 1
+  template:
+    metadata:
+      labels:
+        job-name: podmonitor
+    spec:
+      containers:
+        - name: podmonitor-skipjob
+          image: "perl:5.34.0"
+          imagePullPolicy: Always
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsGroup: 150
+            runAsUser: 150
+      imagePullSecrets:
+        - name: github-auth
+      priorityClassName: skip-medium
+      restartPolicy: Never
+      securityContext:
+        fsGroup: 150
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccountName: podmonitor-skipjob
+
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    instance: primary
+  name: podmonitor-monitor
+spec:
+  namespaceSelector:
+    matchNames:
+      - test
+  podMetricsEndpoints:
+    - targetPort: 8080
+      path: "/metrics"
+  selector:
+    matchLabels:
+      app: podmonitor

tests/skipjob/podmonitor/00-job.yaml

@@ -0,0 +1,15 @@
+apiVersion: skiperator.kartverket.no/v1alpha1
+kind: SKIPJob
+metadata:
+  name: podmonitor
+spec:
+  container:
+    image: "perl:5.34.0"
+    command:
+      - "perl"
+      - "-Mbignum=bpi"
+      - "-wle"
+      - "print bpi(2000)"
+  prometheus:
+    path: /metrics
+    port: 8080

tests/skipjob/podmonitor/01-delete-job.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: skiperator.kartverket.no/v1alpha1
+    kind: SKIPJob
+    name: podmonitor

tests/skipjob/podmonitor/02-assert.yaml

@@ -0,0 +1,60 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: podmonitor-skipjob
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: podmonitor
+  labels:
+    app: podmonitor-skipjob
+    skiperator.kartverket.no/skipjob: "true"
+    skiperator.kartverket.no/skipjobName: podmonitor
+spec:
+  suspend: false
+  backoffLimit: 6
+  completionMode: NonIndexed
+  parallelism: 1
+  template:
+    metadata:
+      labels:
+        job-name: podmonitor
+    spec:
+      containers:
+        - name: podmonitor-skipjob
+          image: "perl:5.34.0"
+          imagePullPolicy: Always
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsGroup: 150
+            runAsUser: 150
+      imagePullSecrets:
+        - name: github-auth
+      priorityClassName: skip-medium
+      restartPolicy: Never
+      securityContext:
+        fsGroup: 150
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccountName: podmonitor-skipjob
+
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  labels:
+    instance: primary
+  name: podmonitor-monitor
+spec:
+  namespaceSelector:
+    matchNames:
+      - test
+  podMetricsEndpoints:
+    - targetPort: istio-metrics
+      path: "/stats/prometheus"
+  selector:
+    matchLabels:
+      app: podmonitor

tests/skipjob/podmonitor/02-job-istio.yaml

@@ -0,0 +1,22 @@
+apiVersion: skiperator.kartverket.no/v1alpha1
+kind: SKIPJob
+metadata:
+  name: podmonitor
+spec:
+  container:
+    image: "perl:5.34.0"
+    command:
+      - "perl"
+      - "-Mbignum=bpi"
+      - "-wle"
+      - "print bpi(2000)"
+  prometheus:
+    path: /metrics
+    port: 8080
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test
+  labels:
+    istio.io/rev: "revision-1"

tests/skipjob/podmonitor/03-remove-istio.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test
+  labels:
+    istio.io/rev: ""

tests/skipjob/podmonitor/04-delete-job.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: skiperator.kartverket.no/v1alpha1
+    kind: SKIPJob
+    name: podmonitor