package tracing import ( "context" "fmt" "sort" "time" "strconv" "github.com/coroot/coroot-node-agent/ebpftracer" "github.com/coroot/coroot-node-agent/ebpftracer/l7" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" semconv "go.opentelemetry.io/otel/semconv/v1.18.0" "go.opentelemetry.io/otel/trace" "inet.af/netaddr" ) /** * Trace */ func (t *Trace) setContext(ctx context.Context) { t.lock.Lock() defer t.lock.Unlock() t.ctx = ctx } func (t *Trace) setSpan(span trace.Span) { t.lock.Lock() defer t.lock.Unlock() t.span = span } type TimeMap struct { Time uint64 Type int Map *ebpftracer.StackFunEvent } func (t *Trace) buildFun() { mapSlice := []TimeMap{} for i, v := range t.stack { timeStartMap := TimeMap{} if v.StackEvent.Location == 0 { timeStartMap = TimeMap{ Time: v.StackEvent.TimeNsStart, Type: 0, Map: &t.stack[i], } } else { timeStartMap = TimeMap{ Time: v.StackEvent.TimeNsEnd, Type: 1, Map: &t.stack[i], } } mapSlice = append(mapSlice, timeStartMap) } sort.Slice(mapSlice, func(i, j int) bool { return mapSlice[i].Time < mapSlice[j].Time }) funStack := []TimeMap{} currentfunNum := 1 // for k, v := range mapSlice { // fmt.Println("---SliceSliceindex", k, "value", v.Time, v.Type, v.Map.Uprobe.Funcname, v.Map.StackEvent.Nid) // } mapSliceLen := len(mapSlice) for k, v := range mapSlice { // fmt.Println("SliceSliceindex", k, "value", v.Time, v.Type, v.Map.Uprobe.Funcname, v.Map.StackEvent.Nid) if v.Type == 0 { // 函数入口 funStack = append(funStack, v) } else if v.Type == 1 { // 函数出口 len := len(funStack) if len < 1 { fmt.Printf("buildFun ErrorError return before enter: %v\n", v) continue } currnt := funStack[len-1] if currnt.Map.StackEvent.Location != 0 { fmt.Printf("currnt StackEvent ErrorError is not enter: %v\n", v) continue } if k < mapSliceLen-1 && len >= 2 { nextfun := mapSlice[k+1] preCurrnt := funStack[len-2] // // 处理 .netcore 多次 returun // 下一个事件是 return 并且函数名跟当前事件是一样的,且上一个函数不是当前函数 if nextfun.Map.StackEvent.Location == 1 && nextfun.Map.Uprobe.Funcname == currnt.Map.Uprobe.Funcname && preCurrnt.Map.Uprobe.Funcname != currnt.Map.Uprobe.Funcname { currentfunNum++ continue } } funStack = funStack[:len-1] duration := v.Map.StackEvent.TimeNsEnd - currnt.Map.StackEvent.TimeNsStart t.FuncTraceQuery(currnt.Map.Uprobe.Funcname, time.Duration(duration), currnt.Map.StackEvent.TimeNsStart, v.Map.StackEvent.TimeNsEnd, currentfunNum) currentfunNum = 1 } } } func (t *Trace) TraceStart(method, path string, status l7.Status, duration time.Duration) { if t == nil || method == "" { return } t.createParentSpan("APPLICATION", duration, status >= 400, semconv.HTTPURL(fmt.Sprintf("http://%s%s", t.destination.String(), path)), semconv.HTTPMethod(method), //semconv.HTTPStatusCode(int(status)), attribute.String("http.uri", path), ) } func (t *Trace) TraceEnd(r *l7.RequestData) { if t == nil { return } t.span.SetAttributes( semconv.HTTPStatusCode(int(r.Status)), attribute.String("server.trace_id_from", r.ParentSpanContext.TraceIdFrom), ) CalledId, err := strconv.ParseInt(r.ParentSpanContext.CalledId, 10, 64) if err == nil && CalledId != 0 { t.span.SetAttributes(attribute.Int64("server.called_id", CalledId)) } InstanceIdFrom, err := strconv.ParseInt(r.ParentSpanContext.InstanceIdFrom, 10, 64) if err == nil && InstanceIdFrom != 0 { t.span.SetAttributes(attribute.Int64("server.instance_id_from", InstanceIdFrom)) } AppIdFrom, err := strconv.ParseInt(r.ParentSpanContext.AppIdFrom, 10, 64) if err == nil && AppIdFrom != 0 { t.span.SetAttributes(attribute.Int64("server.app_id_from", AppIdFrom)) } if r.ParentSpanContext.SpanIdFrom != "0000000000000000" { t.span.SetAttributes(attribute.String("server.span_id_from", r.ParentSpanContext.SpanIdFrom)) } // for _, v := range t.stack { // fmt.Printf("TraceEndTraceEndTraceEnd%s\n", v) // } t.buildFun() t.span.End(trace.WithTimestamp(time.Now())) } func (t *Trace) createParentSpan(name string, duration time.Duration, error bool, attrs ...attribute.KeyValue) { end := time.Now() start := end.Add(-duration) ctx, span := tracer(t.containerId).Start(context.Background(), name, trace.WithTimestamp(start), trace.WithSpanKind(trace.SpanKindClient)) span.SetAttributes(attrs...) span.SetAttributes(t.commonAttrs...) if error { span.SetStatus(codes.Error, "") } t.setContext(ctx) t.setSpan(span) } func (t *Trace) createTraceSpan(name string, duration time.Duration, error bool, attrs ...attribute.KeyValue) { end := time.Now() start := end.Add(-duration) //fmt.Println("createTraceSpan:", t.ctx) _, span := tracer(t.containerId).Start(t.ctx, name, trace.WithTimestamp(start), trace.WithSpanKind(trace.SpanKindClient)) span.SetAttributes(t.commonAttrs...) span.SetAttributes(attrs...) if error { span.SetStatus(codes.Error, "") } span.End(trace.WithTimestamp(end)) } func (t *Trace) MysqlTraceQuery(query string, error bool, duration time.Duration, destination netaddr.IPPort) { if t == nil || query == "" { return } t.createTraceSpan(l7.ProtocolMysql.String(), duration, error, semconv.DBSystemMySQL, semconv.DBStatement(query), semconv.NetPeerName(destination.IP().String()), semconv.NetPeerPort(int(destination.Port())), ) } func (t *Trace) RedisTraceQuery(cmd, args string, error bool, duration time.Duration) { if t == nil || cmd == "" { return } statement := cmd if args != "" { statement += " " + args } t.createTraceSpan(l7.ProtocolRedis.String(), duration, error, semconv.DBSystemRedis, semconv.DBOperation(cmd), semconv.DBStatement(statement), ) } func (t *Trace) HttpTraceRequest(method, path, ip string, port uint16, r *l7.RequestData) { if t == nil || method == "" { return } assumedAppID, err := strconv.ParseInt(r.AssumedAppId, 10, 64) if err != nil { assumedAppID = 0 } status := r.Status duration := r.Duration t.createTraceSpan(l7.ProtocolHTTP.String(), duration, status >= 400, semconv.HTTPURL(fmt.Sprintf("http://%s%s", t.destination.String(), path)), semconv.HTTPMethod(method), semconv.HTTPStatusCode(int(status)), attribute.String("http.uri", path), attribute.String("http.ip", ip), attribute.Int64("http.assumed_app_id", assumedAppID), attribute.String("http.span_id", r.SpanId), attribute.Int("http.port", int(port)), ) } func (t *Trace) FuncTraceQuery(funcname string, duration time.Duration, start uint64, end uint64, num int) { if t == nil || funcname == "" { return } t.createTraceSpanNoTime(funcname, duration, false, start, end, attribute.Int("num", num)) } func (t *Trace) createTraceSpanNoTime(name string, duration time.Duration, error bool, start uint64, end uint64, attrs ...attribute.KeyValue) { // end := time.Now() // start := end.Add(-duration) startTime := time.Unix(0, int64(start)) endTime := time.Unix(0, int64(end)) //fmt.Println("createTraceSpan:", t.ctx) _, span := tracer(t.containerId).Start(t.ctx, name, trace.WithTimestamp(startTime), trace.WithSpanKind(trace.SpanKindClient)) span.SetAttributes(t.commonAttrs...) span.SetAttributes(attrs...) if error { span.SetStatus(codes.Error, "") } span.End(trace.WithTimestamp(endTime)) }