OpenTelemetryをGCPで活用する方法｜Cloud Trace・Monitoring連携と実装手順

extensions:
  googleclientauth:

exporters:
  otlphttp:
    encoding: proto
    endpoint: https://telemetry.googleapis.com
    auth:
      authenticator: googleclientauth

service:
  extensions: [googleclientauth]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch, resourcedetection]
      exporters: [otlphttp]
    metrics:
      receivers: [otlp]
      processors: [batch, resourcedetection]
      exporters: [otlphttp]

# service.yaml
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
  name: my-app
  annotations:
    run.googleapis.com/launch-stage: BETA
spec:
  template:
    metadata:
      annotations:
        run.googleapis.com/container-dependencies: '{"my-app":["otel-collector"]}'
    spec:
      containers:
        - image: my-app-image:latest
          name: my-app
          ports:
            - containerPort: 8080
          env:
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: "http://localhost:4318"
        - image: us-docker.pkg.dev/cloud-ops-agents-artifacts/google-cloud-opentelemetry-collector/otelcol-google:0.143.0
          name: otel-collector
          startupProbe:
            httpGet:
              path: /
              port: 13133

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: otel-collector
  template:
    metadata:
      labels:
        app: otel-collector
    spec:
      serviceAccountName: otel-collector-sa
      containers:
        - name: otel-collector
          image: otel/opentelemetry-collector-contrib:latest
          ports:
            - containerPort: 4317  # gRPC
            - containerPort: 4318  # HTTP
          volumeMounts:
            - name: config
              mountPath: /etc/otelcol-contrib
      volumes:
        - name: config
          configMap:
            name: otel-collector-config

# Kubernetes ServiceAccountとGCP IAMの紐付け
gcloud iam service-accounts add-iam-policy-binding \
  otel-sa@PROJECT_ID.iam.gserviceaccount.com \
  --role roles/iam.workloadIdentityUser \
  --member "serviceAccount:PROJECT_ID.svc.id.goog[monitoring/otel-collector-sa]"

// tracing.js - モジュールロード時に初期化
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { Resource } = require('@opentelemetry/resources');
const { ATTR_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');

const sdk = new NodeSDK({
  resource: new Resource({
    [ATTR_SERVICE_NAME]: process.env.K_SERVICE || 'my-function',
  }),
  traceExporter: new OTLPTraceExporter({
    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces',
  }),
  instrumentations: [getNodeAutoInstrumentations()],
});

sdk.start();

// index.js
require('./tracing');
const functions = require('@google-cloud/functions-framework');

functions.http('myFunction', (req, res) => {
  // 自動計装により HTTP リクエストのスパンが生成される
  res.send('Hello from instrumented function!');
});

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
  prometheus:
    config:
      scrape_configs:
        - job_name: 'app-metrics'
          scrape_interval: 30s
          static_configs:
            - targets: ['localhost:9090']

processors:
  batch:
    send_batch_size: 200
    timeout: 5s
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128
  resourcedetection:
    detectors: [env, gcp]
    timeout: 5s
    override: false
  resource:
    attributes:
      - key: service.namespace
        value: "production"
        action: upsert

exporters:
  otlphttp:
    encoding: proto
    endpoint: https://telemetry.googleapis.com
    auth:
      authenticator: googleclientauth

extensions:
  googleclientauth:
  health_check:
    endpoint: 0.0.0.0:13133

service:
  extensions: [googleclientauth, health_check]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, resourcedetection, resource]
      exporters: [otlphttp]
    metrics:
      receivers: [otlp, prometheus]
      processors: [memory_limiter, batch, resourcedetection, resource]
      exporters: [otlphttp]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch, resourcedetection, resource]
      exporters: [otlphttp]

processors:
  # 確率的サンプリング: 10%のトレースのみ保持
  probabilistic_sampler:
    sampling_percentage: 10

  # Tail-based サンプリング: エラーや高レイテンシのトレースを優先保持
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: errors-policy
        type: status_code
        status_code: {status_codes: [ERROR]}
      - name: latency-policy
        type: latency
        latency: {threshold_ms: 1000}
      - name: probabilistic-policy
        type: probabilistic
        probabilistic: {sampling_percentage: 5}

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor
from flask import Flask

# リソース定義
resource = Resource.create({
    "service.name": "order-service",
    "service.namespace": "production",
    "deployment.environment": "prod",
})

# TracerProvider初期化
provider = TracerProvider(resource=resource)
processor = BatchSpanProcessor(
    OTLPSpanExporter()  # OTEL_EXPORTER_OTLP_ENDPOINT 環境変数を参照
)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

# Flask自動計装
app = Flask(__name__)
FlaskInstrumentor().instrument_app(app)
RequestsInstrumentor().instrument()

tracer = trace.get_tracer(__name__)

@app.route("/orders/<order_id>")
def get_order(order_id):
    with tracer.start_as_current_span("fetch-order-detail") as span:
        span.set_attribute("order.id", order_id)
        # DB呼び出しなどのビジネスロジック
        return {"order_id": order_id, "status": "confirmed"}

package main

import (
	"context"
	"log"
	"net/http"

	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

func initTracer() func() {
	ctx := context.Background()
	exporter, err := otlptracehttp.New(ctx)
	if err != nil {
		log.Fatalf("failed to create exporter: %v", err)
	}

	res, _ := resource.Merge(
		resource.Default(),
		resource.NewWithAttributes(
			semconv.SchemaURL,
			semconv.ServiceName("payment-service"),
			semconv.DeploymentEnvironmentName("production"),
		),
	)

	tp := sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(exporter),
		sdktrace.WithResource(res),
	)
	otel.SetTracerProvider(tp)

	return func() { tp.Shutdown(ctx) }
}

func main() {
	shutdown := initTracer()
	defer shutdown()

	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		tracer := otel.Tracer("payment-service")
		_, span := tracer.Start(r.Context(), "process-payment")
		defer span.End()

		w.Write([]byte("payment processed"))
	})

	wrappedHandler := otelhttp.NewHandler(handler, "server")
	log.Fatal(http.ListenAndServe(":8080", wrappedHandler))
}

// tracing.js
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-http');
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { Resource } = require('@opentelemetry/resources');
const { ATTR_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');

const sdk = new NodeSDK({
  resource: new Resource({
    [ATTR_SERVICE_NAME]: process.env.K_SERVICE || 'user-service',
  }),
  traceExporter: new OTLPTraceExporter(),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter(),
    exportIntervalMillis: 30000,
  }),
  instrumentations: [getNodeAutoInstrumentations({
    '@opentelemetry/instrumentation-fs': { enabled: false },
  })],
});

sdk.start();
process.on('SIGTERM', () => sdk.shutdown());

// app.js
require('./tracing');
const express = require('express');
const { trace } = require('@opentelemetry/api');

const app = express();
const tracer = trace.getTracer('user-service');

app.get('/users/:id', async (req, res) => {
  const span = tracer.startSpan('fetch-user-profile');
  span.setAttribute('user.id', req.params.id);
  try {
    // ビジネスロジック
    res.json({ id: req.params.id, name: 'example' });
  } finally {
    span.end();
  }
});

app.listen(8080);

gcloud services list --enabled --filter="name:cloudtrace.googleapis.com OR name:monitoring.googleapis.com"

gcloud projects get-iam-policy PROJECT_ID \
  --flatten="bindings[].members" \
  --filter="bindings.members:serviceAccount:SA_NAME@PROJECT_ID.iam.gserviceaccount.com"

service:
  telemetry:
    logs:
      level: debug

# NG: BatchSpanProcessor（第1世代では送信前に終了する可能性）
from opentelemetry.sdk.trace.export import BatchSpanProcessor

# OK: SimpleSpanProcessor（同期送信で確実に送信）
from opentelemetry.sdk.trace.export import SimpleSpanProcessor

receivers:
  prometheus:
    config:
      scrape_configs:
        - job_name: 'my-app'
          scrape_interval: 15s
          kubernetes_sd_configs:
            - role: pod
          relabel_configs:
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
              action: keep
              regex: true
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
              action: replace
              target_label: __address__
              regex: (.+)
              replacement: $${1}:$$2