Merge pull request #41 from coroot/otel_logs_service_names

configuring OpenTelemetry service names for k8s container logs

common/otel.go

@@ -0,0 +1,24 @@
+package common
+
+import (
+	"regexp"
+	"strings"
+)
+
+var (
+	deploymentPodRegex  = regexp.MustCompile(`(/k8s/[a-z0-9-]+/[a-z0-9-]+)-[0-9a-f]{1,10}-[bcdfghjklmnpqrstvwxz2456789]{5}/.+`)
+	daemonsetPodRegex   = regexp.MustCompile(`(/k8s/[a-z0-9-]+/[a-z0-9-]+)-[bcdfghjklmnpqrstvwxz2456789]{5}/.+`)
+	statefulsetPodRegex = regexp.MustCompile(`(/k8s/[a-z0-9-]+/[a-z0-9-]+)-\d+/.+`)
+)
+
+func ContainerIdToOtelServiceName(containerId string) string {
+	if !strings.HasPrefix(containerId, "/k8s/") {
+		return containerId
+	}
+	for _, r := range []*regexp.Regexp{deploymentPodRegex, daemonsetPodRegex, statefulsetPodRegex} {
+		if g := r.FindStringSubmatch(containerId); len(g) == 2 {
+			return g[1]
+		}
+	}
+	return containerId
+}

common/otel_test.go

@@ -0,0 +1,37 @@
+package common
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestContainerIdToServiceName(t *testing.T) {
+	f := ContainerIdToOtelServiceName
+	assert.Equal(t,
+		f("/k8s/otel-demo/otel-demo-frauddetectionservice-64cd4f9686-mvtnb/frauddetectionservice"),
+		"/k8s/otel-demo/otel-demo-frauddetectionservice")
+
+	assert.Equal(t,
+		f("/k8s/coroot/coroot-node-agent-np9pk/node-agent"),
+		"/k8s/coroot/coroot-node-agent")
+
+	assert.Equal(t,
+		f("/k8s/coroot/pyroscope-df884bb79-hhxtv/pyroscope"),
+		"/k8s/coroot/pyroscope")
+
+	assert.Equal(t,
+		f("/k8s/default/cassandra-main-12/cassandra"),
+		"/k8s/default/cassandra-main")
+
+	assert.Equal(t,
+		f("/k8s/default/hello-28283967-khz2f/xz"),
+		"/k8s/default/hello")
+
+	assert.Equal(t,
+		f("/system.slice/k3s.service"),
+		"/system.slice/k3s.service")
+
+	assert.Equal(t,
+		f("/docker/container_name"),
+		"/docker/container_name")
+}

go.mod

@@ -131,8 +131,8 @@ require (
 	golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
 	google.golang.org/grpc v1.58.2 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
@@ -140,4 +140,7 @@ require (
 	k8s.io/client-go v0.26.4 // indirect
 )
 
-replace github.com/optiopay/kafka => github.com/cilium/kafka v0.0.0-20180809090225-01ce283b732b
+replace (
+	github.com/agoda-com/opentelemetry-logs-go => github.com/coroot/opentelemetry-logs-go v0.0.0-20231011124644-be9c59fab4e6
+	github.com/optiopay/kafka => github.com/cilium/kafka v0.0.0-20180809090225-01ce283b732b
+)

go.sum

@@ -52,8 +52,6 @@ github.com/Microsoft/hcsshim v0.8.25/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/agoda-com/opentelemetry-logs-go v0.3.0 h1:d2lMVUfCDeLzVgTxMeSU8IWaMXjwD4sVKigEZBGwcsw=
-github.com/agoda-com/opentelemetry-logs-go v0.3.0/go.mod h1:k3QR1O5AOl+dFC7pkrK9wWmoD72jjDONPFHi9dAgLJc=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -126,10 +124,10 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/coroot/logparser v1.0.8 h1:Z/Lhg0cVbsHh6t34VcSEFdZR7zWmssKHfHRA+XlWiFY=
-github.com/coroot/logparser v1.0.8/go.mod h1:GHsVO1xE8pR5mmu9Eiop9IXHwN/QzNRx1s0fuzVxq7I=
 github.com/coroot/logparser v1.0.9 h1:EQRFzTtVMnqpYPJx6dRUFN5Ax2qrHF/ATJJyspSZgBo=
 github.com/coroot/logparser v1.0.9/go.mod h1:GHsVO1xE8pR5mmu9Eiop9IXHwN/QzNRx1s0fuzVxq7I=
+github.com/coroot/opentelemetry-logs-go v0.0.0-20231011124644-be9c59fab4e6 h1:Txhbk1d5QxzmECGvQSs+Z4nQlvA/dY4V5ELwUuegDIQ=
+github.com/coroot/opentelemetry-logs-go v0.0.0-20231011124644-be9c59fab4e6/go.mod h1:CeDuVaK9yCWN+8UjOW8AciYJE0rl7K/mw4ejBntGYkc=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -995,11 +993,11 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g=
-google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw=
-google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 h1:o4LtQxebKIJ4vkzyhtD2rfUNZ20Zf0ik5YVP5E7G7VE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
+google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA=
+google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44=
+google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

logs/otel.go

@@ -7,6 +7,7 @@ import (
 	"github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs/otlplogshttp"
 	otelLogs "github.com/agoda-com/opentelemetry-logs-go/logs"
 	sdk "github.com/agoda-com/opentelemetry-logs-go/sdk/logs"
+	"github.com/coroot/coroot-node-agent/common"
 	"github.com/coroot/logparser"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/sdk/resource"
@@ -27,6 +28,7 @@ func Init(machineId, hostname, version string) {
 	klog.Infoln("OpenTelemetry logs collector endpoint:", endpoint)
 
 	exporter, _ := otlplogs.NewExporter(context.Background(), otlplogs.WithClient(otlplogshttp.NewClient()))
+
 	loggerProvider := sdk.NewLoggerProvider(
 		sdk.WithBatcher(exporter),
 		sdk.WithResource(
@@ -61,14 +63,18 @@ func OtelLogEmitter(containerId string) logparser.OnMsgCallbackF {
 		case logparser.LevelDebug:
 			severityNumber = otelLogs.DEBUG
 		}
+
 		otelLogger.Emit(
 			otelLogs.NewLogRecord(otelLogs.LogRecordConfig{
 				ObservedTimestamp: ts,
 				SeverityText:      &severityText,
 				SeverityNumber:    &severityNumber,
 				Body:              &msg,
-				Attributes: &[]attribute.KeyValue{
+				Resource: resource.NewSchemaless(
+					semconv.ServiceName(common.ContainerIdToOtelServiceName(containerId)),
 					semconv.ContainerID(containerId),
+				),
+				Attributes: &[]attribute.KeyValue{
 					attribute.Key("pattern.hash").String(patternHash),
 				},
 			}),