Devops Observability With Golang

---

✅ What We'll Build:

A simple observability tool in Go with:

1. Metrics: Prometheus metrics (latency, request count)

2. Logging: Structured JSON logging

3. Tracing: Distributed tracing with OpenTelemetry + Jaeger

---

📦 Dependencies:

We'll use:

go get github.com/prometheus/client_golang/prometheus
go get github.com/prometheus/client_golang/prometheus/promhttp
go get go.opentelemetry.io/otel
go get go.opentelemetry.io/otel/exporters/jaeger
go get go.opentelemetry.io/otel/sdk/trace
go get github.com/sirupsen/logrus

---

🧩 Code: main.go

package main

import (
    "context"
    "fmt"
    "log"
    "math/rand"
    "net/http"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "github.com/sirupsen/logrus"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/jaeger"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    "go.opentelemetry.io/otel/trace"
)

var (
    requestCount = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total number of requests",
        },
        []string{"path"},
    )

    requestDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "http_request_duration_seconds",
            Help:    "Duration of HTTP requests",
            Buckets: prometheus.DefBuckets,
        },
        []string{"path"},
    )
)

var tracer trace.Tracer

func initMetrics() {
    prometheus.MustRegister(requestCount)
    prometheus.MustRegister(requestDuration)
}

func initLogger() {
    logrus.SetFormatter(&logrus.JSONFormatter{})
    logrus.SetLevel(logrus.InfoLevel)
}

func initTracer() {
    exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint("http://localhost:14268/api/traces")))
    if err != nil {
        log.Fatalf("failed to create Jaeger exporter: %v", err)
    }

    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exp),
    )
    otel.SetTracerProvider(tp)
    tracer = tp.Tracer("observability-tool")
}

func helloHandler(w http.ResponseWriter, r *http.Request) {
    start := time.Now()
    ctx, span := tracer.Start(r.Context(), "helloHandler")
    defer span.End()

    path := r.URL.Path
    requestCount.WithLabelValues(path).Inc()

    // Simulate random processing delay
    delay := time.Duration(rand.Intn(500)) * time.Millisecond
    time.Sleep(delay)

    duration := time.Since(start).Seconds()
    requestDuration.WithLabelValues(path).Observe(duration)

    logrus.WithFields(logrus.Fields{
        "path":     path,
        "duration": duration,
        "method":   r.Method,
    }).Info("Handled request")

    fmt.Fprintf(w, "Hello, World!")
}

func main() {
    initLogger()
    initMetrics()
    initTracer()

    http.Handle("/metrics", promhttp.Handler())
    http.HandleFunc("/", helloHandler)

    logrus.Info("Starting server at :8080")
    log.Fatal(http.ListenAndServe(":8080", nil))
}

---

📈 How to Use:

1. Run Jaeger: Start the Jaeger service to collect and visualize trace data.

docker run -d --name jaeger \
  -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
  -p 16686:16686 -p 14268:14268 -p 9411:9411 \
  jaegertracing/all-in-one:1.43

2. Run Go App:

   Execute the Go application to start the server.

 run main.go

3. Visit Endpoints:

• http://localhost:8080 → Hello endpoint

• http://localhost:8080/metrics → Prometheus metrics

• http://localhost:16686 → Jaeger UI for traces

---

🛠️ Next Steps:

• Add context propagation to downstream services.

• Integrate with Grafana and Prometheus.

• Use OpenTelemetry SDKs for advanced metrics and spans.

---

✅ Final Structure

observability-tool/
├── go.mod
├── go.sum
├── Dockerfile
├── main.go
├── handlers/
│   └── hello.go
│   └── health.go
├── metrics/
│   └── prometheus.go
├── tracing/
│   └── jaeger.go
├── logger/
│   └── logrus.go

---

1️⃣ Modularization

---

metrics/prometheus.go

package metrics

import (
    "github.com/prometheus/client_golang/prometheus"
)

var (
    RequestCount = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total HTTP requests",
        },
        []string{"path"},
    )

    RequestDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "http_request_duration_seconds",
            Help:    "Request duration",
            Buckets: prometheus.DefBuckets,
        },
        []string{"path"},
    )
)

func Init() {
    prometheus.MustRegister(RequestCount)
    prometheus.MustRegister(RequestDuration)
}

---

logger/logrus.go

package logger

import "github.com/sirupsen/logrus"

func Init() {
    logrus.SetFormatter(&logrus.JSONFormatter{})
    logrus.SetLevel(logrus.InfoLevel)
}

---

tracing/jaeger.go

package tracing

import (
    "log"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/jaeger"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
)

func Init(serviceName string) {
    exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint("http://jaeger:14268/api/traces")))
    if err != nil {
        log.Fatalf("failed to create Jaeger exporter: %v", err)
    }

    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exp),
    )
    otel.SetTracerProvider(tp)
}

---

handlers/hello.go

package handlers

import (
    "fmt"
    "math/rand"
    "net/http"
    "time"

    "github.com/sirupsen/logrus"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/trace"

    "observability-tool/metrics"
)

var tracer trace.Tracer = otel.Tracer("observability-tool")

func HelloHandler(w http.ResponseWriter, r *http.Request) {
    start := time.Now()
    ctx, span := tracer.Start(r.Context(), "HelloHandler")
    defer span.End()

    path := r.URL.Path
    metrics.RequestCount.WithLabelValues(path).Inc()

    delay := time.Duration(rand.Intn(500)) * time.Millisecond
    time.Sleep(delay)

    duration := time.Since(start).Seconds()
    metrics.RequestDuration.WithLabelValues(path).Observe(duration)

    logrus.WithFields(logrus.Fields{
        "path":     path,
        "duration": duration,
        "method":   r.Method,
    }).Info("Handled request")

    fmt.Fprintf(w, "Hello, World!")
}

---

handlers/health.go

package handlers

import (
    "fmt"
    "net/http"
)

func HealthHandler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprint(w, "OK")
}

---

main.go

package main

import (
    "net/http"

    "github.com/prometheus/client_golang/prometheus/promhttp"
    "github.com/sirupsen/logrus"

    "observability-tool/logger"
    "observability-tool/metrics"
    "observability-tool/tracing"
    "observability-tool/handlers"
)

func main() {
    logger.Init()
    metrics.Init()
    tracing.Init("observability-tool")

    http.Handle("/metrics", promhttp.Handler())
    http.HandleFunc("/", handlers.HelloHandler)
    http.HandleFunc("/health", handlers.HealthHandler)

    logrus.Info("Server starting on :8080")
    http.ListenAndServe(":8080", nil)
}

---

2️⃣ Dockerization

---

Dockerfile

FROM golang:1.21-alpine

WORKDIR /app

COPY go.mod go.sum ./
RUN go mod download

COPY . .

RUN go build -o main .

EXPOSE 8080

CMD ["./main"]

---

docker-compose.yml

version: '3'

services:
  app:
    build: .
    ports:
      - "8080:8080"
    depends_on:
      - jaeger

  jaeger:
    image: jaegertracing/all-in-one:1.43
    ports:
      - "16686:16686" # UI
      - "14268:14268" # Collector

---

3️⃣ Run App

docker-compose up --build

Visit:

• http://localhost:8080 → Hello endpoint

• http://localhost:8080/health → Health check

• http://localhost:8080/metrics → Prometheus metrics

• http://localhost:16686 → Jaeger tracing UI

---

---

📈 Metrics Sample

Visit http://localhost:8080/metrics

http_requests_total{path="/"} 42
http_request_duration_seconds_bucket{path="/",le="0.1"} 25
...

---

📍 Jaeger Tracing UI

Visit http://localhost:16686

You’ll see all spans for / and /health requests including duration and trace structure.

---

🛠️ Development

# Format and tidy
go fmt ./...
go mod tidy

---

✅ Future Enhancements

• Add pprof support for profiling

• Add support for log exporters (Loki)

• Middleware for trace ID injection in logs

• CI/CD pipeline and GitHub Actions

---

## 🧰 `Makefile` (Optional)

```makefile
.PHONY: build run test fmt lint docker-up docker-down

build:
    go build -o bin/observability-tool main.go

run:
    go run main.go

fmt:
    go fmt ./...

lint:
    golangci-lint run

test:
    go test ./...

docker-up:
    docker-compose up --build

docker-down:
    docker-compose down

---

Integrating Trivy for vulnerability scanning:

We'll add:

1. trivy scanning for:

   • The Docker image

   • Go modules (go.mod / go.sum)

2. A Makefile target: make scan

3. GitHub Actions CI integration

---

---

🛠️ 2. Update Makefile

scan:
    docker run --rm -v $(PWD):/app aquasec/trivy \
        fs --exit-code 1 --severity HIGH,CRITICAL /app

📌 This scans your entire project directory (source code + dependencies).

---

🧪 Optional: Scan the Docker image

Add this to your Makefile:

scan-image:
    docker build -t observability-tool:latest .
    docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
        -v $(PWD)/trivy-cache:/root/.cache/ \
        aquasec/trivy image observability-tool:latest

---

🔁 3.Update GitHub Actions Workflow

name: Security Scan

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  trivy-scan:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Trivy
        uses: aquasecurity/trivy-action@master
        with:
          scan-type: fs
          scan-ref: .
          severity: HIGH,CRITICAL

---

🔐 What Trivy Scans

• Vulnerabilities in Go dependencies (go.mod, go.sum)

• OS packages (via Alpine base image)

• Secrets or keys in source code

• Misconfigurations (with config scan enabled)

---

✅ Results

• make scan – local developer scan

• make scan-image – Docker image scan

• GitHub PRs automatically fail if HIGH/CRITICAL vulnerabilities are found

---

📌 Final Tips

• Run make scan in pre-commit hooks or CI.

• Use --ignore-unfixed to skip issues with no patches.

• Create a .trivyignore to suppress false positives.