OpenTelemetryとDatadogを統合するための実践ガイド【構成パターンと設定手順】

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: "0.0.0.0:4317"
      http:
        endpoint: "0.0.0.0:4318"

processors:
  batch:
    send_batch_size: 100
    send_batch_max_size: 1000
    timeout: 10s
  resourcedetection:
    detectors: [env, system, docker]
    timeout: 5s

exporters:
  datadog:
    api:
      site: "datadoghq.com"   # USサイトの場合
      key: "${env:DD_API_KEY}"
    traces:
      span_name_as_resource_name: true
    metrics:
      resource_attributes_as_tags: true

connectors:
  datadog/connector:

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [resourcedetection, batch]
      exporters: [datadog/connector, datadog]
    metrics:
      receivers: [otlp, datadog/connector]
      processors: [resourcedetection, batch]
      exporters: [datadog]
    logs:
      receivers: [otlp]
      processors: [resourcedetection, batch]
      exporters: [datadog]

# builder-config.yaml
dist:
  name: otelcol-datadog
  output_path: ./build

exporters:
  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter v0.115.0

connectors:
  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector v0.115.0

receivers:
  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.115.0

processors:
  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.115.0

ocb --config builder-config.yaml

# /etc/datadog-agent/datadog.yaml
otlp_config:
  receiver:
    protocols:
      grpc:
        endpoint: "0.0.0.0:4317"
      http:
        endpoint: "0.0.0.0:4318"

# ログの OTLP 受信を有効化（デフォルトは無効）
otlp_config:
  logs:
    enabled: true

# ログ収集自体を有効化（前提条件）
logs_enabled: true

environment:
  - DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT=0.0.0.0:4317
  - DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT=0.0.0.0:4318
  - DD_OTLP_CONFIG_LOGS_ENABLED=true
  - DD_LOGS_ENABLED=true
  - DD_API_KEY=${DD_API_KEY}

package main

import (
    "context"
    "log"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
)

func initTracer() (*sdktrace.TracerProvider, error) {
    ctx := context.Background()

    // OTel Collector または Datadog Agent の OTLP エンドポイントへ送信
    exporter, err := otlptracegrpc.New(ctx,
        otlptracegrpc.WithEndpoint("localhost:4317"),
        otlptracegrpc.WithInsecure(),
    )
    if err != nil {
        return nil, err
    }

    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceNameKey.String("my-go-service"),
            semconv.DeploymentEnvironmentKey.String("production"),
            semconv.ServiceVersionKey.String("1.0.0"),
        ),
    )
    if err != nil {
        return nil, err
    }

    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter),
        sdktrace.WithResource(res),
    )
    otel.SetTracerProvider(tp)
    return tp, nil
}

func main() {
    tp, err := initTracer()
    if err != nil {
        log.Fatal(err)
    }
    defer tp.Shutdown(context.Background())

    tracer := otel.Tracer("my-go-service")
    ctx, span := tracer.Start(context.Background(), "handleRequest")
    defer span.End()

    // ビジネスロジック
    processOrder(ctx)
}

func processOrder(ctx context.Context) {
    tracer := otel.Tracer("my-go-service")
    _, span := tracer.Start(ctx, "processOrder")
    defer span.End()
    // ...
}

package main

import (
    "context"

    "go.opentelemetry.io/otel"
    ddotel "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentelemetry"
    ddtracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
)

func main() {
    // Datadog TracerProvider を OTel API に登録
    tp := ddotel.NewTracerProvider(
        ddtracer.WithService("my-go-service"),
        ddtracer.WithEnv("production"),
    )
    defer tp.Shutdown()
    otel.SetTracerProvider(tp)

    // 以降は標準の OTel API でスパン生成
    tracer := otel.Tracer("my-go-service")
    ctx, span := tracer.Start(context.Background(), "handleRequest")
    defer span.End()

    processOrder(ctx)
}

# OTel Java Agent をダウンロード
curl -L -o opentelemetry-javaagent.jar \
  https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar

# アプリケーション起動時に Agent をアタッチ
java -javaagent:opentelemetry-javaagent.jar \
  -Dotel.service.name=my-java-service \
  -Dotel.exporter.otlp.endpoint=http://localhost:4317 \
  -Dotel.exporter.otlp.protocol=grpc \
  -Dotel.resource.attributes=deployment.environment=production \
  -jar my-application.jar

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource, SERVICE_NAME

resource = Resource.create({
    SERVICE_NAME: "my-python-service",
    "deployment.environment": "production",
})

provider = TracerProvider(resource=resource)
otlp_exporter = OTLPSpanExporter(
    endpoint="localhost:4317",
    insecure=True,
)
provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
trace.set_tracer_provider(provider)

tracer = trace.get_tracer("my-python-service")

with tracer.start_as_current_span("handle_request") as span:
    span.set_attribute("http.method", "GET")
    span.set_attribute("http.url", "/api/orders")
    # ビジネスロジック

traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
             |   |                                 |                |
             ver  trace-id (128bit)                span-id (64bit)  flags

# datadog.yaml
apm_config:
  # W3C Trace Context を優先的に使用
  propagation_style:
    - tracecontext
    - datadog

import (
    "go.opentelemetry.io/otel/trace"
    "go.uber.org/zap"
    "fmt"
    "strconv"
)

func logWithTrace(ctx context.Context, logger *zap.Logger, msg string) {
    spanCtx := trace.SpanContextFromContext(ctx)
    if spanCtx.IsValid() {
        // Datadog の 128bit Trace ID → 下位 64bit を 10進数に変換
        traceID := spanCtx.TraceID()
        lower64 := traceID[8:]  // 下位8バイト
        ddTraceID := fmt.Sprintf("%d",
            binary.BigEndian.Uint64(lower64))
        ddSpanID := fmt.Sprintf("%d",
            binary.BigEndian.Uint64(spanCtx.SpanID()))

        logger.Info(msg,
            zap.String("dd.trace_id", ddTraceID),
            zap.String("dd.span_id", ddSpanID),
            zap.String("dd.service", "my-go-service"),
            zap.String("dd.env", "production"),
        )
    }
}

// Go OTel SDK でのサンプリング率設定
tp := sdktrace.NewTracerProvider(
    sdktrace.WithSampler(
        sdktrace.ParentBased(
            sdktrace.TraceIDRatioBased(0.1), // 10% サンプリング
        ),
    ),
    sdktrace.WithBatcher(exporter),
)

# OTel Collector の tail_sampling processor 設定
processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 50000
    policies:
      # エラーを含むトレースは全て保持
      - name: errors-policy
        type: status_code
        status_code:
          status_codes:
            - ERROR
      # レイテンシが 1 秒以上のトレースを保持
      - name: latency-policy
        type: latency
        latency:
          threshold_ms: 1000
      # それ以外は 10% をサンプリング
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

# Helm values (datadog/datadog チャート)
datadog:
  apiKey: <DATADOG_API_KEY>
  otlp:
    receiver:
      protocols:
        grpc:
          enabled: true
          endpoint: "0.0.0.0:4317"
        http:
          enabled: true
          endpoint: "0.0.0.0:4318"
  logs:
    enabled: true
    containerCollectAll: true
  apm:
    portEnabled: true

agents:
  containers:
    agent:
      env:
        - name: DD_OTLP_CONFIG_LOGS_ENABLED
          value: "true"

# アプリケーション Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-service
spec:
  template:
    spec:
      containers:
        - name: app
          image: my-service:latest
          env:
            # DaemonSet Agent の IP を自動取得
            - name: DD_AGENT_HOST
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: "http://$(DD_AGENT_HOST):4317"
            - name: OTEL_SERVICE_NAME
              value: "my-service"
            - name: OTEL_RESOURCE_ATTRIBUTES
              value: "deployment.environment=production,service.version=1.0.0"

# Helm values (datadog/datadog チャート)
datadog:
  apiKey: <DATADOG_API_KEY>
  otelCollector:
    enabled: true
    # 既存の OTel Collector 設定をそのまま指定
    config: |
      receivers:
        otlp:
          protocols:
            grpc:
              endpoint: "0.0.0.0:4317"
      processors:
        infraattributes:
          cardinality: 2
        batch:
          timeout: 10s
      exporters:
        datadog:
          api:
            key: ${env:DD_API_KEY}
      service:
        pipelines:
          traces:
            receivers: [otlp]
            processors: [infraattributes, batch]
            exporters: [datadog]
          metrics:
            receivers: [otlp]
            processors: [infraattributes, batch]
            exporters: [datadog]

# 移行期間中の Datadog Agent 設定
# 既存の dd-trace と OTel OTLP の両方を受信
apm_config:
  enabled: true
otlp_config:
  receiver:
    protocols:
      grpc:
        endpoint: "0.0.0.0:4317"

# 高カーディナリティ属性を除外する filter processor の例
processors:
  filter/drop-debug:
    metrics:
      exclude:
        match_type: regexp
        metric_names:
          - ".*debug.*"
  transform/remove-high-cardinality:
    metric_statements:
      - context: datapoint
        statements:
          - delete_key(attributes, "request_id")
          - delete_key(attributes, "user_id")