forked from mirror/redis
chore: improve OpenTelemetry example
This commit is contained in:
Vladimir Mihailenco
parent
f6940c80ca
commit
9070a20f41
|
|
@ -1,40 +1,47 @@
|
|||
# Example for go-redis OpenTelemetry instrumentation
|
||||
|
||||
This example demonstrates how to monitor Redis using OpenTelemetry and
|
||||
[Uptrace](https://github.com/uptrace/uptrace). It requires Docker to start Redis Server and Uptrace.
|
||||
|
||||
See
|
||||
[Monitoring Go Redis Performance and Errors](https://redis.uptrace.dev/guide/go-redis-monitoring.html)
|
||||
for details.
|
||||
|
||||
This example requires Redis Server on port `:6379`. You can start Redis Server using Docker:
|
||||
**Step 1**. Download the example using Git:
|
||||
|
||||
```shell
|
||||
git clone https://github.com/go-redis/redis.git
|
||||
cd example/otel
|
||||
```
|
||||
|
||||
**Step 2**. Start the services using Docker:
|
||||
|
||||
```shell
|
||||
docker-compose pull
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
You can run this example with different OpenTelemetry exporters by providing environment variables.
|
||||
|
||||
**Stdout** exporter (default):
|
||||
**Step 3**. Make sure Uptrace is running:
|
||||
|
||||
```shell
|
||||
go run .
|
||||
docker-compose logs uptrace
|
||||
```
|
||||
|
||||
[Uptrace](https://github.com/uptrace/uptrace) exporter:
|
||||
**Step 4**. Run the Redis client example:
|
||||
|
||||
```shell
|
||||
UPTRACE_DSN="https://<token>@uptrace.dev/<project_id>" go run .
|
||||
UPTRACE_DSN=http://project2_secret_token@localhost:14317/2 go run client.go
|
||||
```
|
||||
|
||||
**Jaeger** exporter:
|
||||
**Step 5**. Follow the link from the CLI to view the trace:
|
||||
|
||||
```shell
|
||||
OTEL_EXPORTER_JAEGER_ENDPOINT=http://localhost:14268/api/traces go run .
|
||||
UPTRACE_DSN=http://project2_secret_token@localhost:14317/2 go run client.go
|
||||
trace: http://localhost:14318/traces/ee029d8782242c8ed38b16d961093b35
|
||||
```
|
||||
|
||||
To instrument Redis Cluster client, see
|
||||
[go-redis-cluster](https://github.com/uptrace/opentelemetry-go-extra/tree/main/example/go-redis-cluster)
|
||||
example.
|
||||
|
||||
## Links
|
||||
|
||||
- [Uptrace open-source APM](https://uptrace.dev/get/open-source-apm.html)
|
||||
- [OpenTelemetry Go instrumentations](https://uptrace.dev/opentelemetry/instrumentations/?lang=go)
|
||||
- [OpenTelemetry Go Tracing API](https://uptrace.dev/opentelemetry/go-tracing.html)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/uptrace/opentelemetry-go-extra/otelplay"
|
||||
"github.com/uptrace/uptrace-go/uptrace"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
|
|
@ -13,13 +15,19 @@ import (
|
|||
"github.com/go-redis/redis/v9"
|
||||
)
|
||||
|
||||
var tracer = otel.Tracer("redisexample")
|
||||
var tracer = otel.Tracer("github.com/go-redis/redis/example/otel")
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
shutdown := otelplay.ConfigureOpentelemetry(ctx)
|
||||
defer shutdown()
|
||||
uptrace.ConfigureOpentelemetry(
|
||||
// copy your project DSN here or use UPTRACE_DSN env var
|
||||
// uptrace.WithDSN("https://AQDan_E_EPe3QAF9fMP0PiVr5UWOu4q5@uptrace.dev/1"),
|
||||
|
||||
uptrace.WithServiceName("myservice"),
|
||||
uptrace.WithServiceVersion("v1.0.0"),
|
||||
)
|
||||
defer uptrace.Shutdown(ctx)
|
||||
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: ":6379",
|
||||
|
|
@ -27,16 +35,26 @@ func main() {
|
|||
if err := redisotel.InstrumentTracing(rdb); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctx, span := tracer.Start(ctx, "handleRequest")
|
||||
defer span.End()
|
||||
|
||||
if err := handleRequest(ctx, rdb); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
if err := redisotel.InstrumentMetrics(rdb); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
otelplay.PrintTraceID(ctx)
|
||||
for i := 0; i < 1e6; i++ {
|
||||
ctx, rootSpan := tracer.Start(ctx, "handleRequest")
|
||||
|
||||
if err := handleRequest(ctx, rdb); err != nil {
|
||||
rootSpan.RecordError(err)
|
||||
rootSpan.SetStatus(codes.Error, err.Error())
|
||||
}
|
||||
|
||||
rootSpan.End()
|
||||
|
||||
if i == 0 {
|
||||
fmt.Printf("view trace: %s\n", uptrace.TraceURL(rootSpan))
|
||||
}
|
||||
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func handleRequest(ctx context.Context, rdb *redis.Client) error {
|
||||
|
|
@ -1,9 +1,65 @@
|
|||
version: '3'
|
||||
|
||||
services:
|
||||
clickhouse:
|
||||
image: clickhouse/clickhouse-server:22.7
|
||||
restart: on-failure
|
||||
environment:
|
||||
CLICKHOUSE_DB: uptrace
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'localhost:8123/ping']
|
||||
interval: 1s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
volumes:
|
||||
- ch_data:/var/lib/clickhouse
|
||||
ports:
|
||||
- '8123:8123'
|
||||
- '9000:9000'
|
||||
|
||||
uptrace:
|
||||
image: 'uptrace/uptrace:latest'
|
||||
#image: 'uptrace/uptrace-dev:latest'
|
||||
volumes:
|
||||
- uptrace_data:/var/lib/uptrace
|
||||
- ./uptrace.yml:/etc/uptrace/uptrace.yml
|
||||
#environment:
|
||||
# - DEBUG=2
|
||||
ports:
|
||||
- '14317:14317'
|
||||
- '14318:14318'
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.58.0
|
||||
restart: on-failure
|
||||
user: '0:0' # required for logs
|
||||
volumes:
|
||||
- ./otel-collector.yaml:/etc/otelcol-contrib/config.yaml
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
- /var/log:/var/log:ro
|
||||
ports:
|
||||
- '4317:4317'
|
||||
- '4318:4318'
|
||||
|
||||
vector:
|
||||
image: timberio/vector:0.24.X-alpine
|
||||
volumes:
|
||||
- ./vector.toml:/etc/vector/vector.toml:ro
|
||||
|
||||
redis-server:
|
||||
image: redis
|
||||
ports:
|
||||
- '6379:6379'
|
||||
redis-cli:
|
||||
image: redis
|
||||
|
||||
volumes:
|
||||
uptrace_data:
|
||||
driver: local
|
||||
ch_data:
|
||||
driver: local
|
||||
alertmanager_data:
|
||||
driver: local
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
extensions:
|
||||
health_check:
|
||||
pprof:
|
||||
endpoint: 0.0.0.0:1777
|
||||
zpages:
|
||||
endpoint: 0.0.0.0:55679
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
http:
|
||||
hostmetrics:
|
||||
collection_interval: 10s
|
||||
scrapers:
|
||||
cpu:
|
||||
disk:
|
||||
load:
|
||||
filesystem:
|
||||
memory:
|
||||
network:
|
||||
paging:
|
||||
redis:
|
||||
endpoint: 'redis-server:6379'
|
||||
collection_interval: 10s
|
||||
jaeger:
|
||||
protocols:
|
||||
grpc:
|
||||
|
||||
processors:
|
||||
resourcedetection:
|
||||
detectors: ['system']
|
||||
batch:
|
||||
send_batch_size: 10000
|
||||
timeout: 10s
|
||||
|
||||
exporters:
|
||||
logging:
|
||||
logLevel: debug
|
||||
otlp:
|
||||
endpoint: uptrace:14317
|
||||
tls:
|
||||
insecure: true
|
||||
headers: { 'uptrace-dsn': 'http://project2_secret_token@localhost:14317/2' }
|
||||
|
||||
service:
|
||||
# telemetry:
|
||||
# logs:
|
||||
# level: DEBUG
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp, jaeger]
|
||||
processors: [batch]
|
||||
exporters: [otlp, logging]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [otlp]
|
||||
metrics/hostmetrics:
|
||||
receivers: [hostmetrics, redis]
|
||||
processors: [batch, resourcedetection]
|
||||
exporters: [otlp]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [otlp]
|
||||
|
||||
extensions: [health_check, pprof, zpages]
|
||||
|
|
@ -0,0 +1,247 @@
|
|||
##
|
||||
## Uptrace configuration file.
|
||||
## See https://uptrace.dev/get/config.html for details.
|
||||
##
|
||||
## You can use environment variables anywhere in this file, for example:
|
||||
##
|
||||
## foo: $FOO
|
||||
## bar: ${BAR}
|
||||
## baz: ${BAZ:default}
|
||||
##
|
||||
## To escape `$`, use `$$`, for example:
|
||||
##
|
||||
## foo: $$FOO_BAR
|
||||
##
|
||||
|
||||
##
|
||||
## A list of pre-configured projects. Each project is fully isolated.
|
||||
##
|
||||
projects:
|
||||
# Conventionally, the first project is used to monitor Uptrace itself.
|
||||
- id: 1
|
||||
name: Uptrace
|
||||
# Token grants write access to the project. Keep a secret.
|
||||
token: project1_secret_token
|
||||
pinned_attrs:
|
||||
- service.name
|
||||
- host.name
|
||||
- deployment.environment
|
||||
|
||||
# Other projects can be used to monitor your applications.
|
||||
# To monitor micro-services or multiple related services, use a single project.
|
||||
- id: 2
|
||||
name: My project
|
||||
token: project2_secret_token
|
||||
pinned_attrs:
|
||||
- service.name
|
||||
- host.name
|
||||
- deployment.environment
|
||||
|
||||
##
|
||||
## To require authentication, uncomment the following section.
|
||||
##
|
||||
auth:
|
||||
# users:
|
||||
# - username: uptrace
|
||||
# password: uptrace
|
||||
# - username: admin
|
||||
# password: admin
|
||||
|
||||
# # Cloudflare user provider: uses Cloudflare Zero Trust Access (Identity)
|
||||
# # See https://developers.cloudflare.com/cloudflare-one/identity/ for more info.
|
||||
# cloudflare:
|
||||
# # The base URL of the Cloudflare Zero Trust team.
|
||||
# - team_url: https://myteam.cloudflareaccess.com
|
||||
# # The Application Audience (AUD) Tag for this application.
|
||||
# # You can retrieve this from the Cloudflare Zero Trust 'Access' Dashboard.
|
||||
# audience: bea6df23b944e4a0cd178609ba1bb64dc98dfe1f66ae7b918e563f6cf28b37e0
|
||||
|
||||
# # OpenID Connect (Single Sign-On)
|
||||
# oidc:
|
||||
# # The ID is used in API endpoints, for example, in redirect URL
|
||||
# # `http://<uptrace-host>/api/v1/sso/<oidc-id>/callback`.
|
||||
# - id: keycloak
|
||||
# # Display name for the button in the login form.
|
||||
# # Default to 'OpenID Connect'
|
||||
# display_name: Keycloak
|
||||
# # The base URL for the OIDC provider.
|
||||
# issuer_url: http://localhost:8080/realms/uptrace
|
||||
# # The OAuth 2.0 Client ID
|
||||
# client_id: uptrace
|
||||
# # The OAuth 2.0 Client Secret
|
||||
# client_secret: ogbhd8Q0X0e5AZFGSG3m9oirPvnetqkA
|
||||
# # Additional OAuth 2.0 scopes to request from the OIDC provider.
|
||||
# # Defaults to 'profile'. 'openid' is requested by default and need not be specified.
|
||||
# scopes:
|
||||
# - profile
|
||||
# # The OIDC UserInfo claim to use as the user's username.
|
||||
# # Defaults to 'preferred_username'.
|
||||
# claim: preferred_username
|
||||
|
||||
##
|
||||
## ClickHouse database credentials.
|
||||
##
|
||||
ch:
|
||||
# Connection string for ClickHouse database. For example:
|
||||
# clickhouse://<user>:<password>@<host>:<port>/<database>?sslmode=disable
|
||||
#
|
||||
# See https://clickhouse.uptrace.dev/guide/golang-clickhouse.html#options
|
||||
dsn: 'clickhouse://default:@clickhouse:9000/uptrace?sslmode=disable'
|
||||
|
||||
##
|
||||
## Alerting rules for monitoring metrics.
|
||||
##
|
||||
## See https://uptrace.dev/get/alerting.html for details.
|
||||
##
|
||||
alerting:
|
||||
rules:
|
||||
- name: Network errors
|
||||
metrics:
|
||||
- system.network.errors as $net_errors
|
||||
query:
|
||||
- $net_errors > 0 group by host.name
|
||||
# for the last 5 minutes
|
||||
for: 5m
|
||||
# in the project id=1
|
||||
projects: [1]
|
||||
|
||||
- name: Filesystem usage >= 90%
|
||||
metrics:
|
||||
- system.filesystem.usage as $fs_usage
|
||||
query:
|
||||
- group by host.name
|
||||
- group by device
|
||||
- where device !~ "loop"
|
||||
- $fs_usage{state="used"} / $fs_usage >= 0.9
|
||||
for: 5m
|
||||
projects: [1]
|
||||
|
||||
- name: Uptrace is dropping spans
|
||||
metrics:
|
||||
- uptrace.projects.spans as $spans
|
||||
query:
|
||||
- $spans{type=dropped} > 0
|
||||
for: 1m
|
||||
projects: [1]
|
||||
|
||||
- name: Always firing (for fun and testing)
|
||||
metrics:
|
||||
- process.runtime.go.goroutines as $goroutines
|
||||
query:
|
||||
- $goroutines >= 0 group by host.name
|
||||
for: 1m
|
||||
projects: [1]
|
||||
|
||||
# Create alerts from error logs and span events.
|
||||
create_alerts_from_spans:
|
||||
enabled: true
|
||||
labels:
|
||||
alert_kind: error
|
||||
|
||||
##
|
||||
## AlertManager client configuration.
|
||||
## See https://uptrace.dev/get/alerting.html for details.
|
||||
##
|
||||
## Note that this is NOT an AlertManager config and you need to configure AlertManager separately.
|
||||
## See https://prometheus.io/docs/alerting/latest/configuration/ for details.
|
||||
##
|
||||
alertmanager_client:
|
||||
# AlertManager API endpoints that Uptrace uses to manage alerts.
|
||||
# urls:
|
||||
# - 'http://alertmanager:9093/api/v2/alerts'
|
||||
|
||||
##
|
||||
## Various options to tweak ClickHouse schema.
|
||||
## For changes to take effect, you need reset the ClickHouse database with `ch reset`.
|
||||
##
|
||||
ch_schema:
|
||||
# Compression codec, for example, LZ4, ZSTD(3), or Default.
|
||||
compression: ZSTD(3)
|
||||
|
||||
# Whether to use ReplicatedMergeTree instead of MergeTree.
|
||||
replicated: false
|
||||
|
||||
# Cluster name for Distributed tables and ON CLUSTER clause.
|
||||
#cluster: uptrace1
|
||||
|
||||
spans:
|
||||
storage_policy: 'default'
|
||||
# Delete spans data after 30 days.
|
||||
ttl_delete: 30 DAY
|
||||
|
||||
metrics:
|
||||
storage_policy: 'default'
|
||||
# Delete metrics data after 90 days.
|
||||
ttl_delete: 90 DAY
|
||||
|
||||
##
|
||||
## Addresses on which Uptrace receives gRPC and HTTP requests.
|
||||
##
|
||||
listen:
|
||||
# OTLP/gRPC API.
|
||||
grpc:
|
||||
addr: ':14317'
|
||||
# tls:
|
||||
# cert_file: config/tls/uptrace.crt
|
||||
# key_file: config/tls/uptrace.key
|
||||
|
||||
# OTLP/HTTP API and Uptrace API with UI.
|
||||
http:
|
||||
addr: ':14318'
|
||||
# tls:
|
||||
# cert_file: config/tls/uptrace.crt
|
||||
# key_file: config/tls/uptrace.key
|
||||
|
||||
##
|
||||
## Various options for Uptrace UI.
|
||||
##
|
||||
site:
|
||||
# Overrides public URL for Vue-powered UI in case you put Uptrace behind a proxy.
|
||||
#addr: 'https://uptrace.mydomain.com'
|
||||
|
||||
##
|
||||
## Spans processing options.
|
||||
##
|
||||
spans:
|
||||
# The size of the Go chan used to buffer incoming spans.
|
||||
# If the buffer is full, Uptrace starts to drop spans.
|
||||
#buffer_size: 100000
|
||||
|
||||
# The number of spans to insert in a single query.
|
||||
#batch_size: 10000
|
||||
|
||||
##
|
||||
## Metrics processing options.
|
||||
##
|
||||
metrics:
|
||||
# List of attributes to drop for being noisy.
|
||||
drop_attrs:
|
||||
- telemetry.sdk.language
|
||||
- telemetry.sdk.name
|
||||
- telemetry.sdk.version
|
||||
|
||||
# The size of the Go chan used to buffer incoming measures.
|
||||
# If the buffer is full, Uptrace starts to drop measures.
|
||||
#buffer_size: 100000
|
||||
|
||||
# The number of measures to insert in a single query.
|
||||
#batch_size: 10000
|
||||
|
||||
##
|
||||
## SQLite/PostgreSQL db that is used to store metadata such us metric names, dashboards, alerts,
|
||||
## and so on.
|
||||
##
|
||||
db:
|
||||
# Either sqlite or postgres.
|
||||
driver: sqlite
|
||||
# Database connection string.
|
||||
#
|
||||
# Uptrace automatically creates SQLite database file in the current working directory.
|
||||
# Make sure the directory is writable by Uptrace process.
|
||||
dsn: 'file:uptrace.sqlite3?_pragma=foreign_keys(1)&_pragma=busy_timeout(1000)'
|
||||
|
||||
# Secret key that is used to sign JWT tokens etc.
|
||||
secret_key: 102c1a557c314fc28198acd017960843
|
||||
|
||||
# Enable to log HTTP requests and database queries.
|
||||
debug: false
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
[sources.syslog_logs]
|
||||
type = "demo_logs"
|
||||
format = "syslog"
|
||||
interval = 0.1
|
||||
|
||||
[sources.apache_common_logs]
|
||||
type = "demo_logs"
|
||||
format = "apache_common"
|
||||
interval = 0.1
|
||||
|
||||
[sources.apache_error_logs]
|
||||
type = "demo_logs"
|
||||
format = "apache_error"
|
||||
interval = 0.1
|
||||
|
||||
[sources.json_logs]
|
||||
type = "demo_logs"
|
||||
format = "json"
|
||||
interval = 0.1
|
||||
|
||||
# Parse Syslog logs
|
||||
# See the Vector Remap Language reference for more info: https://vrl.dev
|
||||
[transforms.parse_logs]
|
||||
type = "remap"
|
||||
inputs = ["syslog_logs"]
|
||||
source = '''
|
||||
. = parse_syslog!(string!(.message))
|
||||
'''
|
||||
|
||||
# Export data to Uptrace.
|
||||
[sinks.uptrace]
|
||||
type = "http"
|
||||
inputs = ["parse_logs", "apache_common_logs", "apache_error_logs", "json_logs"]
|
||||
encoding.codec = "json"
|
||||
framing.method = "newline_delimited"
|
||||
compression = "gzip"
|
||||
uri = "http://uptrace:14318/api/v1/vector/logs"
|
||||
#uri = "https://api.uptrace.dev/api/v1/vector/logs"
|
||||
headers.uptrace-dsn = "http://project2_secret_token@localhost:14317/2"
|
||||
Loading…
Reference in New Issue