Merge branch 'dev-trace-header-server-perf' into 'dev'

Dev trace header server perf

See merge request TSB/euspace!3

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "files.associations": {
+        "*.toml": "toml",
+        "numbers": "c"
+    }
+}

Makefile2

@@ -14,16 +14,16 @@ endif
 debug:
 	exit
 build:
-	CGO_ENABLED=1 go build -buildvcs=false -o euspace
+	CGO_ENABLED=1 go build -gcflags="all=-N -l" -buildvcs=false -o euspace
 c:
 	#docker exec -it 9d928d96d4d0 sh -c 'cd /opt/github/euspace/ebpftracer && sh build.sh${PARAMS}'
-	docker exec -it 62d0676aa0b7 sh -c 'cd /opt/github/euspace/ebpftracer && sh build_linux.sh${PARAMS}'
+	docker exec -it 3c1a4817a7ed sh -c 'cd /opt/github/euspace/ebpftracer && sh build_linux.sh ${PARAMS}'
 c-build: c
 
 go-build:
 	#ssh [email protected] 'export https_proxy=http://10.0.22.50:4780 && source ~/.g/env && cd /opt/github/euspace && make -f Makefile2 build'
-	docker exec -it 62d0676aa0b7 bash -c 'cd /opt/github/euspace && source ~/.g/env && make -f Makefile2 build'
-go: go-build run
+	docker exec -it 3c1a4817a7ed bash -c 'cd /opt/github/euspace && source ~/.g/env && make -f Makefile2 build'
+go: go-build
 
 run:
 	ssh [email protected] 'cd /opt/github/euspace && TRACES_ENDPOINT=http://10.0.12.192:18080/docp/api/v2/data/receive ${FILTER} ./euspace --listen="0.0.0.0:8123"'

build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+rm ./euspace
+make -f Makefile2 all debug=1 pid=1121
+pid=`ps aux | grep ebpfdemo81 | grep -v grep | awk '{print $2}'`
+echo $pid
+TRACES_ENDPOINT=http://10.2.31.73:8099/docp/api/v2/data/receive SEND=1 FILTER_PID=$pid WHITE_LIST=".*HandleFunc|.*main.*|.*serverHandler.*|.*ServeHTTP.*" ./euspace  --listen="0.0.0.0:8123"

cbuild.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+rm ./euspace
+make -f Makefile2 c debug=1 pid=1121
+# pid=`ps aux | grep ebpfdemo81 | grep -v grep | awk '{print $2}'`
+# echo $pid
+# TRACES_ENDPOINT=http://10.2.29.1:8099/docp/api/v2/data/receive FILTER_PID=$pid ./euspace --listen="0.0.0.0:8123"

cmd

@@ -0,0 +1,2 @@
+make -f Makefile2 all debug=1 pid=1121
+TRACES_ENDPOINT=http://10.2.29.1:8099/docp/api/v2/data/receive FILTER_PID=493419 ./euspace --listen="0.0.0.0:8123"

containers/container.go

@@ -146,11 +146,8 @@ type Container struct {
 
 	done chan struct{}
 
-	traceMap      map[uint64]*tracing.Trace
-	instanceID    utils.ID
-	goEventStack  map[uint64]uint64
-	goEvents      map[uint64][]ebpftracer.StackFunEvent
-	goEventStacks map[uint64]map[uint64][]ebpftracer.StackFunEvent
+	traceMap   map[uint64]*tracing.Trace
+	instanceID utils.ID
 }
 
 func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, hostConntrack *Conntrack, pid uint32) (*Container, error) {
@@ -186,10 +183,8 @@ func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, host
 
 		hostConntrack: hostConntrack,
 
-		done:         make(chan struct{}),
-		traceMap:     make(map[uint64]*tracing.Trace),
-		goEventStack: map[uint64]uint64{},
-		goEvents:     map[uint64][]ebpftracer.StackFunEvent{},
+		done:     make(chan struct{}),
+		traceMap: make(map[uint64]*tracing.Trace),
 	}
 
 	for _, n := range md.networks {

containers/container_apm.go

@@ -1,12 +1,13 @@
 package containers
 
 import (
+	"bufio"
 	"debug/elf"
 	"fmt"
-	"math/rand"
+	"os"
 	"sort"
 	"strconv"
-	"time"
+	"strings"
 
 	"github.com/coroot/coroot-node-agent/ebpftracer"
 	"github.com/coroot/coroot-node-agent/ebpftracer/l7"
@@ -231,116 +232,31 @@ func (c *Container) buildInstanceID() {
 }
 
 func (c *Container) StackProcess(event ebpftracer.StackEvent, tracer *ebpftracer.Tracer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	// get the associated uprobe
-	switch event.Location {
-	case 0: // ret
-		uprobe, err := c.GetUprobe(event, tracer)
-		if err != nil {
-			//fmt.Println("GetUprobeGetUprobe errer: %v", err)
-			// log.Errorf("failed to get uprobe for event %+v: %+v", event, err)
-			return
-		}
-
-		if event.TraceId <= 0 {
-			//fmt.Println("StackProcess TraceId id 0")
-			// log.Errorf("failed to get uprobe for event %+v: %+v", event, err)
-			return
-		}
-
-		//fmt.Println("StackProcess 函数入口开始处理 fun:", event.TraceId, uprobe.Funcname, event.TimeNsEnd-event.TimeNsStart)
-		apmTrace, ok := c.getTrace(event.TraceId)
-		if ok {
-			//fmt.Println("append FuncTraceQuery fun:", event.TraceId, uprobe.Funcname, event.Pid)
-			duration := event.TimeNsEnd - event.TimeNsStart
-			apmTrace.FuncTraceQuery(uprobe.Funcname, time.Duration(duration), int(event.Level), int(event.Fpid), int(event.Nid))
-		}
-	case 2: // coroutine
-		//fmt.Println("StackProcess 协程入口开始处理 fun:", event.TraceId, event.Goid, event.TimeNsEnd-event.TimeNsStart)
-		apmTrace, ok := c.getTrace(event.TraceId)
-		if ok {
-			//fmt.Println("append FuncTraceQuery fun:", event.TraceId, "coroutine"+strconv.FormatUint(event.Goid, 10), event.Pid, event.Fpid)
-			duration := event.TimeNsEnd - event.TimeNsStart
-			apmTrace.FuncTraceQuery("coroutine"+strconv.FormatUint(event.Goid, 10), time.Duration(duration), int(event.Level), int(event.Fpid), int(event.Nid))
-		}
-	}
-}
-
-func (c *Container) StackProcessBak(event ebpftracer.StackEvent, tracer *ebpftracer.Tracer) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	// get the associated uprobe
 	uprobe, err := c.GetUprobe(event, tracer)
 	if err != nil {
-		//fmt.Println("GetUprobeGetUprobe errer: %v", err)
+		fmt.Println("GetUprobeGetUprobe errer: %v", err)
 		// log.Errorf("failed to get uprobe for event %+v: %+v", event, err)
 		return
 	}
 
 	if event.TraceId <= 0 {
-		//fmt.Println("StackProcess TraceId id 0")
+		fmt.Println("StackProcess TraceId id 0")
 		// log.Errorf("failed to get uprobe for event %+v: %+v", event, err)
 		return
 	}
 
-	length := len(c.goEventStacks[event.TraceId])
-	if length <= 0 {
-		c.goEventStacks = map[uint64]map[uint64][]ebpftracer.StackFunEvent{}
-		c.goEventStacks[event.TraceId] = map[uint64][]ebpftracer.StackFunEvent{}
-		c.goEventStacks[event.TraceId][event.Goid] = []ebpftracer.StackFunEvent{}
-	}
+	// fmt.Printf("StackProcess 函数入口开始处理 fun:TraceId:%lld, Funcname:%s, time: %lld\n", event.TraceId, uprobe.Funcname, event.TimeNsEnd-event.TimeNsStart)
+	stackFun := ebpftracer.StackFunEvent{}
+	stackFun.Uprobe = &uprobe
+	stackFun.StackEvent = event
 
-	switch event.Location {
-	case 0: // entry
-		level := 100
-		pid := 100000 + event.Goid
-		length := len(c.goEventStacks[event.TraceId][event.Goid])
-		//fmt.Println("StackProcess 函数入口开始处理 fun:", event.TraceId, uprobe.Funcname, length)
-		if length > 0 {
-			funEvent := c.goEventStacks[event.TraceId][event.Goid][length-1]
-			//fmt.Println("funEvent goEventStacks fun:", event.TraceId, funEvent.Uprobe.Funcname, funEvent.Nid, funEvent.Level)
-			lastEvent := funEvent.StackEvent
-			if lastEvent.Location == event.Location && lastEvent.Ip == event.Ip && lastEvent.Bp != event.CallerBp {
-				// duplicated entry event due to stack expansion/shrinkage
-				// log.Debugf("duplicated entry event: %+v", event)
-				//fmt.Println("GetUprobeGetUprobe duplicated entry event: %+v", event)
-				c.goEventStacks[event.TraceId][event.Goid][length-1].StackEvent = event
-				return
-			}
-			level = int(funEvent.Level)
-			pid = uint64(funEvent.Nid)
-		}
-		rand.Seed(time.Now().UnixNano())
-		// append new event
-		//fmt.Println("append goEventStacks fun:", event.TraceId, uprobe.Funcname, pid, level+1)
-		c.goEventStacks[event.TraceId][event.Goid] = append(c.goEventStacks[event.TraceId][event.Goid], ebpftracer.StackFunEvent{
-			StackEvent: event,
-			Uprobe:     &uprobe,
-			Level:      level + 1,
-			Pid:        int(pid),
-			Nid:        rand.Intn(100000000),
-		})
-		length = len(c.goEventStacks[event.TraceId][event.Goid])
-		//fmt.Println("append goEventStacks end:", event.TraceId, uprobe.Funcname, pid, level+1, length)
-	case 1: // ret
-		//// fmt.Println("StackProcess 函数出口开始处理 fun:", event.TraceId, uprobe.Funcname)
-		length := len(c.goEventStacks[event.TraceId][event.Goid])
-		//fmt.Println("StackProcess 函数出口开始处理 fun:", event.TraceId, uprobe.Funcname, length)
-		if length > 0 {
-			funEvent := c.goEventStacks[event.TraceId][event.Goid][length-1]
-			entFun := funEvent.StackEvent
-			apmTrace, ok := c.getTrace(event.TraceId)
-			//fmt.Println("StackProcess 函数出口处理 fun:", event.TraceId, funEvent.Uprobe.Funcname, length)
-			if ok {
-				//fmt.Println("append FuncTraceQuery fun:", event.TraceId, uprobe.Funcname, funEvent.Pid, funEvent.Level, funEvent.Nid)
-				duration := event.TimeNsEnd - entFun.TimeNsStart
-				c.goEventStacks[event.TraceId][event.Goid] = c.goEventStacks[event.TraceId][event.Goid][:length-1]
-				apmTrace.FuncTraceQuery(funEvent.Uprobe.Funcname, time.Duration(duration), funEvent.Level, funEvent.Pid, funEvent.Nid)
-			}
-		}
+	apmTrace, ok := c.getTrace(event.TraceId)
+	if ok {
+		apmTrace.FunAdd(stackFun)
 	}
-
 }
 
 // ResolveAddress returns the symbol(s) and offset of the given address.
@@ -371,15 +287,72 @@ func (c *Container) ResolveAddress(addr uint64, symbols []elf.Symbol) (syms []el
 	return syms, uint(addr - sym.Value), nil
 }
 
+type MemoryMap struct {
+	Start, End uint64
+}
+
+// ReadFirstLineOfMapsFile reads the first line of /proc/<pid>/maps file and return the memory map as a MemoryMap struct
+func ReadFirstLineOfMapsFile(pid string) (*MemoryMap, error) {
+	file, err := os.Open(fmt.Sprintf("/proc/%s/maps", pid))
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	if scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		addresses := strings.Split(fields[0], "-")
+		if len(addresses) != 2 {
+			return nil, errors.New("unexpected format in /proc/<pid>/maps")
+		}
+
+		start, err := strconv.ParseUint(addresses[0], 16, 64)
+		if err != nil {
+			return nil, err
+		}
+
+		end, err := strconv.ParseUint(addresses[1], 16, 64)
+		if err != nil {
+			return nil, err
+		}
+
+		return &MemoryMap{
+			Start: start,
+			End:   end,
+		}, nil
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return nil, errors.New("empty /proc/<pid>/maps")
+}
+
 func (c *Container) GetUprobe(event ebpftracer.StackEvent, tracer *ebpftracer.Tracer) (uprobe tracer.Uprobe, err error) {
 	//fmt.Println("GetUprobe entory:")
+
+	memoryMap, _ := ReadFirstLineOfMapsFile(strconv.Itoa(int(event.Pid)))
+	Address := event.Ip - memoryMap.Start
+	// fmt.Printf("memoryMap.Start: %x, event.Ip: %x, Address: %x\n", memoryMap.Start, event.Ip, Address)
+
+	for _, fun := range tracer.UprobesMap {
+		funAddress := fun.Address + fun.AbsOffset
+		// fmt.Printf("GetUprobeGetUprobeGetUprobe:fun.Address %x, fun.AbsOffset: %x\n", fun.Address, fun.AbsOffset)
+		if funAddress == Address {
+			// fmt.Printf("---GetUprobeGetUprobeGetUprobe: %x, event.Ip: %x\n", memoryMap.Start, event.Ip)
+			return fun, nil
+		}
+	}
+
 	syms, _, err := c.ResolveAddress(event.Ip, tracer.Symbols)
 	if err != nil {
 		return
 	}
 	for _, sym := range syms {
 		//fmt.Println("GetUprobeGetUprobeGetUprobe: %s+%d", sym.Name, offset)
-		uprobe, ok := tracer.UprobesMap[fmt.Sprintf("%s", sym.Name)]
+		uprobe, ok := tracer.UprobesMap[fmt.Sprintf("%s-%s", sym.Name, sym.Value)]
 		if ok {
 			return uprobe, nil
 		}

containers/registry.go

@@ -52,8 +52,6 @@ type Registry struct {
 }
 
 var (
-	// goEvents map[uint64][]ebpftracer.StackFunEvent // k=goid,v=[]event
-	// goEventStack map[uint64]uint64
 	uprobes    []tracer.Uprobe
 	uprobesMap map[string]tracer.Uprobe
 )
@@ -303,14 +301,13 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 				}
 
 				if c := r.containersByPid[uint32(e.StackEvent.Pid)]; c != nil {
-					//fmt.Println("e.EventTypeFunEnt: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
-					//fmt.Println("e.EventTypeFunEnt: FPid:%d, Nid:%d, Level:%d", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
+					// fmt.Printf("e.EventTypeFunEnt: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x\n", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
+					// fmt.Printf("e.EventTypeFunEnt: FPid:%x, Nid:%x, Level:%d\n", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
 					c.StackProcess(*e.StackEvent, r.tracer)
+				} else {
+					// fmt.Printf("e.EventTypeFunEnt ErrorError: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
+					// fmt.Printf("e.EventTypeFunEnt ErrorError: TraceId:%x, FPid:%x, Nid:%x, Level:%d\n", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
 				}
-				//else {
-				//	fmt.Println("e.EventTypeFunEnt ErrorError: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
-				//	fmt.Println("e.EventTypeFunEnt ErrorError: TraceId:%d, FPid:%d, Nid:%d, Level:%d", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
-				//}
 			}
 		}
 	}

ebpftracer/ebpf/utrace/go/net/stack.probe.bpf copy.c

@@ -0,0 +1,281 @@
+#include "arguments.h"
+#include "span_context.h"
+#include "go_context.h"
+#include "go_types.h"
+#include "uprobe.h"
+
+#define MAX_DATA_SIZE 64
+
+#define ENTPOINT 0
+#define RETPOINT 1
+
+// bpf write event data when enter/exit a function
+struct event
+{
+	__u64 pid;
+	__u64 trace_id;
+	__u64 goid;
+	__u64 ip;
+	__u64 bp;
+	__u64 caller_ip;
+	__u64 caller_bp;
+	__u64 time_ns_start;
+	__u64 time_ns_end;
+	__u64 nid;
+	__u64 fpid;
+	__u64 level;
+	__u8 location;
+}__attribute__((packed));
+
+struct trace_stack_entry_index_key_t {
+	__u32 trace_id;
+	__u64 goid;
+};
+
+struct trace_stack_entry_index_value_t {
+	__u64 fun_key;
+};
+
+struct trace_stack_entry_key_t {
+	__u32 trace_id;
+	__u64 goid;
+	__u64 fun_key;
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+    __uint(key_size, sizeof(int));
+    __uint(value_size, sizeof(int));
+} event_queue SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct trace_stack_entry_key_t));
+	__uint(value_size, sizeof(struct event));
+	__uint(max_entries, 32768);
+} trace_stack_entry SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct trace_stack_entry_index_key_t));
+	__uint(value_size, sizeof(__u64));
+	__uint(max_entries, 32768);
+} trace_stack_entry_index SEC(".maps");
+
+struct bpf_map_def SEC("maps") event_stack = {
+	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
+	.key_size = sizeof(__u32),
+	.value_size = sizeof(struct event),
+	.max_entries = 1,
+};
+
+struct bpf_map_def SEC("maps") should_trace_goid = {
+	.type = BPF_MAP_TYPE_HASH,
+	.key_size = sizeof(__u64),
+	.value_size = sizeof(bool),
+	.max_entries = 10000,
+};
+
+SEC("uprobe/ent")
+int ent(struct pt_regs *ctx)
+{
+    bpf_printk("[Go] [uprobe/ent] enter");
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+
+	__u64 trace_id = get_apm_trace_id(pid, pid_tgid);
+
+	__u32 key = 0;
+	struct event *e = bpf_map_lookup_elem(&event_stack, &key);
+	if (!e)
+	{
+		bpf_printk("[Go] [uprobe/ret]: conot get event");
+		return 0;
+	}
+	__builtin_memset(e, 0, sizeof(*e));
+
+	// e->goid = get_goid();
+	e->goid = get_current_goroutine();
+
+	// e->ip = ctx->ip;
+	e->ip = PT_REGS_IP(ctx); // 当前函数的地址
+	if (!bpf_map_lookup_elem(&should_trace_goid, &e->goid))
+	{
+		__u64 should_trace = true;
+		bpf_map_update_elem(&should_trace_goid, &e->goid, &should_trace, BPF_ANY);
+	}
+	bpf_printk("[Go] [uprobe/ent]: e->goid: %llu", e->goid);
+	bpf_printk("[Go] [uprobe/ent]: e->ip: %d", e->ip);
+	bpf_printk("[Go] [uprobe/ent]: yes");
+
+	e->pid = pid;
+	e->trace_id = trace_id;
+	e->location = ENTPOINT; // 标识入口还是出口
+	e->time_ns_start = bpf_ktime_get_ns();
+	e->bp = PT_REGS_SP(ctx) - 8; // 因为 ebpf 程序执行时实际上还没进行堆栈的切换，所以此处是需要在父函数中获取子函数的基地址，新的堆栈会将父函数的 rbp 入栈，一个 rbp 在 64位机器上占用 8 个字节，所以需要减 8，获取到子函数的基地址，低地址的需要减8，如果到高地址的cpu 上就得+8
+	e->caller_bp = PT_REGS_FP(ctx); // 父函数的基地址
+
+	void *ra;
+	ra = (void *)PT_REGS_SP(ctx); // 父函数的 sp，此时 sp 指向父函数的最后一个地址
+	bpf_probe_read_user(&e->caller_ip, sizeof(e->caller_ip), ra);
+
+	bpf_printk("[Go] [uprobe/ent]: goid: %llu", e->goid);
+
+	bpf_printk("[Go] [uprobe/ent]: event: location:%x,ip:%x,time_ns_start:%lld\n", e->location, e->ip, e->time_ns_start);
+	bpf_printk("[Go] [uprobe/ent]: event: bp:%x,caller_bp:%x,caller_ip:%x\n", e->bp,e->caller_bp,e->caller_ip);
+
+	struct trace_stack_entry_index_key_t trace_index_key = {};
+	trace_index_key.goid = e->goid;
+	trace_index_key.trace_id = trace_id;
+
+	bpf_printk("[Go] [uprobe/ent]: trace_stack_entry_index_key_t: goid:%d,trace_id:%lld\n", trace_index_key.goid,trace_index_key.trace_id);
+	__u64 *fun_key_p = bpf_map_lookup_elem(&trace_stack_entry_index, &trace_index_key);
+	__u64 fun_key = 1;
+	__u64 fpid = e->goid;
+	if (fun_key_p) {
+		bpf_printk("[Go] [uprobe/ent]: get fun_key: %d", *fun_key_p);
+		struct trace_stack_entry_key_t trace_key_parent = {};
+		trace_key_parent.goid = e->goid;
+		trace_key_parent.trace_id = trace_id;
+		trace_key_parent.fun_key = *fun_key_p;
+
+		struct event *event_parent = bpf_map_lookup_elem(&trace_stack_entry, &trace_key_parent);
+		if (!event_parent) {
+			bpf_printk("[Go] [uprobe/ent]: get event_parent Error Error: %d", *fun_key_p);
+		} else {
+			fpid = event_parent->nid;
+		}
+		fun_key = *fun_key_p + 1;
+	}
+
+	struct trace_stack_entry_key_t trace_key = {};
+	trace_key.goid = e->goid;
+	trace_key.trace_id = trace_id;
+	trace_key.fun_key = fun_key;
+
+	struct event event_current = {};
+	event_current.bp = e->bp;
+	event_current.caller_bp = e->caller_bp;
+	event_current.caller_ip = e->caller_ip;
+	event_current.pid = e->pid;
+	event_current.trace_id = e->trace_id;
+	event_current.goid = e->goid;
+	event_current.ip = e->ip;
+	event_current.time_ns_start = e->time_ns_start;
+	event_current.location = e->location;
+	event_current.nid = bpf_get_prandom_u32() & 0xFF;
+	event_current.fpid = fpid;
+	// 第一级是 appliction，二级是 协程
+	event_current.level = fun_key + 2;
+
+	bpf_printk("[Go] [uprobe/ent]: uodate fun_key: %d", fun_key);
+	bpf_map_update_elem(&trace_stack_entry, &trace_key, &event_current, BPF_ANY);
+	bpf_map_update_elem(&trace_stack_entry_index, &trace_index_key, &fun_key, BPF_ANY);
+
+	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
+	// bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, e, sizeof(*e));
+
+	bpf_printk("[Go] [uprobe/ent] end");
+	return 1;
+}
+
+SEC("uprobe/ret")
+int ret(struct pt_regs *ctx)
+{
+    bpf_printk("[Go] [uprobe/ret] enter");
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+
+	__u64 trace_id = get_apm_trace_id(pid, pid_tgid);
+
+	__u32 key = 0;
+	struct event *e = bpf_map_lookup_elem(&event_stack, &key);
+	if (!e)
+	{
+		bpf_printk("[Go] [uprobe/ret]: conot get event");
+		return 0;
+	}
+	__builtin_memset(e, 0, sizeof(*e));
+
+	// e->goid = get_goid();
+	e->goid = get_current_goroutine();
+	if (!bpf_map_lookup_elem(&should_trace_goid, &e->goid))
+	{
+		bpf_printk("[Go] [uprobe/ret]: conot get event: %llu", e->goid);
+		return 0;
+	}
+	bpf_printk("[Go] [uprobe/ret]: yes");
+	e->pid = pid;
+	e->trace_id = trace_id;
+	e->location = RETPOINT;
+	e->ip = PT_REGS_IP(ctx);
+	e->time_ns_end = bpf_ktime_get_ns();
+	e->bp = PT_REGS_FP(ctx);
+	bpf_printk("[Go] [uprobe/ret]: e->ip:%x,bp:%x", e->ip, e->bp);
+	bpf_printk("[Go] [uprobe/ret]: goid: %llu", e->goid);
+
+	bpf_printk("[Go] [uprobe/ret]: event: location:%x,ip:%x,time_ns_end:%lld\n", e->location, e->ip, e->time_ns_end);
+
+	struct trace_stack_entry_index_key_t trace_index_key = {};
+	trace_index_key.goid = e->goid;
+	trace_index_key.trace_id = trace_id;
+
+	__u64 *fun_key_p = bpf_map_lookup_elem(&trace_stack_entry_index, &trace_index_key);
+	__u64 fun_key = 1;
+	if (fun_key_p) {
+		bpf_printk("[Go] [uprobe/ret]: get fun_key: %d", *fun_key_p);
+		fun_key = *fun_key_p;
+	} else {
+		bpf_printk("[Go] [uprobe/ret]:ErrorErrorErrorError Not get fun_key");
+	}
+	struct trace_stack_entry_key_t trace_key = {};
+	trace_key.goid = e->goid;
+	trace_key.trace_id = trace_id;
+	trace_key.fun_key = fun_key;
+
+	struct event *event_p = bpf_map_lookup_elem(&trace_stack_entry, &trace_key);
+
+	if (!event_p) {
+		bpf_printk("[Go] [uprobe/ret]:ErrorErrorErrorError Not get funEntry");
+		return 0;
+	}
+
+	event_p->time_ns_end = e->time_ns_end;
+
+	bpf_printk("[Go] [uprobe/ret]: ent:event: location:%d,ip:%x,time_ns_start:%lld\n", event_p->location, event_p->ip, event_p->time_ns_start);
+	bpf_printk("[Go] [uprobe/ret]: ent:event: bp:%x,caller_bp:%x,caller_ip:%x\n", event_p->bp,event_p->caller_bp,event_p->caller_ip);
+	bpf_printk("[Go] [uprobe/ret]: ent:event: nid:%d,fpid:%d,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+
+	bpf_map_delete_elem(&trace_stack_entry, &trace_key);
+	__u64 fun_key_new = fun_key - 1;
+	bpf_map_update_elem(&trace_stack_entry_index, &trace_index_key, &fun_key_new, BPF_ANY);
+
+	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
+	// 最后一个函数结束，同时推送协程信息
+	if (fun_key == 1) {
+		struct event event_coroutine = {};
+		event_coroutine.bp = e->bp;
+		event_coroutine.caller_bp = e->caller_bp;
+		event_coroutine.caller_ip = e->caller_ip;
+		event_coroutine.pid = e->pid;
+		event_coroutine.trace_id = e->trace_id;
+		event_coroutine.goid = e->goid;
+		event_coroutine.ip = e->ip;
+		event_coroutine.time_ns_start = e->time_ns_start;
+		event_coroutine.location = 2;
+		event_coroutine.nid = e->goid;
+		event_coroutine.fpid = 1;
+		event_coroutine.level = 2;
+		bpf_printk("[Go] [uprobe/ret]: ent:event_coroutine push: nid:%d,fpid:%d,level:%d\n", event_coroutine.nid,event_coroutine.fpid,event_coroutine.level);
+		bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, &event_coroutine, sizeof(event_coroutine));
+	}
+	bpf_printk("[Go] [uprobe/ret]: ent:event_ret push: nid:%d,fpid:%d,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+	bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, event_p, sizeof(*e));
+
+	bpf_printk("[Go] [uprobe/ret] end");
+	return 1;
+}
+
+
+
+

ebpftracer/ebpf/utrace/go/net/stack.probe.bpf.c

@@ -27,25 +27,16 @@ struct event
 	__u8 location;
 }__attribute__((packed));
 
-struct trace_stack_entry_index_key_t {
-	__u32 trace_id;
-	__u64 goid;
-};
-
-struct trace_stack_entry_index_value_t {
-	__u64 fun_key;
-};
-
 struct trace_stack_entry_key_t {
-	__u32 trace_id;
-	__u64 goid;
-	__u64 fun_key;
+	__u64 caller_bp;
+	__u64 bp;
 };
 
 struct {
     __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
     __uint(key_size, sizeof(int));
     __uint(value_size, sizeof(int));
+	__uint(max_entries, 32768);
 } event_queue SEC(".maps");
 
 struct {
@@ -55,21 +46,6 @@ struct {
 	__uint(max_entries, 32768);
 } trace_stack_entry SEC(".maps");
 
-struct {
-	__uint(type, BPF_MAP_TYPE_LRU_HASH);
-	__uint(key_size, sizeof(struct trace_stack_entry_index_key_t));
-	__uint(value_size, sizeof(__u64));
-	__uint(max_entries, 32768);
-} trace_stack_entry_index SEC(".maps");
-
-// struct bpf_map_def SEC("maps") event_queue = {
-// 	.type = BPF_MAP_TYPE_QUEUE,
-// 	.key_size = 0,
-// 	.value_size = sizeof(struct event),
-// 	.max_entries = 10000,
-// };
-
-
 struct bpf_map_def SEC("maps") event_stack = {
 	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
 	.key_size = sizeof(__u32),
@@ -78,10 +54,10 @@ struct bpf_map_def SEC("maps") event_stack = {
 };
 
 struct bpf_map_def SEC("maps") should_trace_goid = {
-	.type = BPF_MAP_TYPE_HASH,
+	.type = BPF_MAP_TYPE_LRU_HASH,
 	.key_size = sizeof(__u64),
 	.value_size = sizeof(bool),
-	.max_entries = 10000,
+	.max_entries = 32768,
 };
 
 SEC("uprobe/ent")
@@ -129,58 +105,35 @@ int ent(struct pt_regs *ctx)
 
 	cw_bpf_debug("[Go] [uprobe/ent]: goid: %llu", e->goid);
 
-	cw_bpf_debug("[Go] [uprobe/ent]: event: location:%x,ip:%x,time_ns_start:%lld\n", e->location, e->ip, e->time_ns_start);
+	cw_bpf_debug("[Go] [uprobe/ent]: event: location:%x,ip:%lx,time_ns_start:%lld\n", e->location, e->ip, e->time_ns_start);
 	cw_bpf_debug("[Go] [uprobe/ent]: event: bp:%x,caller_bp:%x,caller_ip:%x\n", e->bp,e->caller_bp,e->caller_ip);
 
-	struct trace_stack_entry_index_key_t trace_index_key = {};
-	trace_index_key.goid = e->goid;
-	trace_index_key.trace_id = trace_id;
-
-	cw_bpf_debug("[Go] [uprobe/ent]: trace_stack_entry_index_key_t: goid:%d,trace_id:%lld\n", trace_index_key.goid,trace_index_key.trace_id);
-	__u64 *fun_key_p = bpf_map_lookup_elem(&trace_stack_entry_index, &trace_index_key);
-	__u64 fun_key = 1;
-	__u64 fpid = e->goid;
-	if (fun_key_p) {
-		cw_bpf_debug("[Go] [uprobe/ent]: get fun_key: %d", *fun_key_p);
-		struct trace_stack_entry_key_t trace_key_parent = {};
-		trace_key_parent.goid = e->goid;
-		trace_key_parent.trace_id = trace_id;
-		trace_key_parent.fun_key = *fun_key_p;
-
-		struct event *event_parent = bpf_map_lookup_elem(&trace_stack_entry, &trace_key_parent);
-		if (!event_parent) {
-			cw_bpf_debug("[Go] [uprobe/ent]: get event_parent Error Error: %d", *fun_key_p);
-		} else {
-			fpid = event_parent->nid;
-		}
-		fun_key = *fun_key_p + 1;
-	}
-
-	struct trace_stack_entry_key_t trace_key = {};
-	trace_key.goid = e->goid;
-	trace_key.trace_id = trace_id;
-	trace_key.fun_key = fun_key;
-
-	struct event event_current = {};
-	event_current.bp = e->bp,
-	event_current.caller_bp = e->caller_bp,
-	event_current.caller_ip = e->caller_ip,
-	event_current.pid = e->pid,
-	event_current.trace_id = e->trace_id,
-	event_current.goid = e->goid,
-	event_current.ip = e->ip,
-	event_current.time_ns_start = e->time_ns_start,
-	event_current.location = e->location,
-	event_current.nid = bpf_get_prandom_u32() & 0xFF,
-	event_current.fpid = fpid,
-	event_current.level = fun_key + 2, // 第一级是 appliction，二级是 协程
-
-	cw_bpf_debug("[Go] [uprobe/ent]: uodate fun_key: %d", fun_key);
-	bpf_map_update_elem(&trace_stack_entry, &trace_key, &event_current, BPF_ANY);
-	bpf_map_update_elem(&trace_stack_entry_index, &trace_index_key, &fun_key, BPF_ANY);
+	// struct trace_stack_entry_key_t trace_key = {};
+	// trace_key.caller_bp = e->caller_bp % 0x800 + (e->goid << 12);
+	// trace_key.bp = e->bp % 0x800 + (e->goid << 12);
+
+	// trace_key.caller_bp = e->caller_bp;
+	// trace_key.bp = e->bp;
+	// cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: bp: %x", trace_key.bp);
+	// cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x", trace_key.caller_bp, trace_key.bp);
+
+	// struct event event_current = {};
+	// event_current.bp = e->bp,
+	// event_current.caller_bp = e->caller_bp,
+	// event_current.caller_ip = e->caller_ip,
+	// event_current.pid = e->pid,
+	// event_current.trace_id = e->trace_id,
+	// event_current.goid = e->goid,
+	// event_current.ip = e->ip,
+	// event_current.time_ns_start = e->time_ns_start,
+	// event_current.location = e->location,
+	// event_current.nid = ((e->ip % 0x1000) << 24) + e->bp % 0x800 + (e->goid << 12),
+	// event_current.fpid = e->caller_bp % 0x800 + (e->goid << 12),
+
+	// bpf_map_update_elem(&trace_stack_entry, &trace_key, &event_current, BPF_ANY);
 
 	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
-	// bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, e, sizeof(*e));
+	bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, e, sizeof(*e));
 
 	cw_bpf_debug("[Go] [uprobe/ent] end");
 	return 1;
@@ -217,67 +170,42 @@ int ret(struct pt_regs *ctx)
 	e->location = RETPOINT;
 	e->ip = PT_REGS_IP(ctx);
 	e->time_ns_end = bpf_ktime_get_ns();
-	e->bp = PT_REGS_FP(ctx);
-	cw_bpf_debug("[Go] [uprobe/ret]: e->ip:%x,bp:%x", e->ip, e->bp);
-	cw_bpf_debug("[Go] [uprobe/ret]: goid: %llu", e->goid);
+	e->bp = PT_REGS_SP(ctx) - 8;
+	e->caller_bp = PT_REGS_FP(ctx);
+
+	cw_bpf_debug("[Go] [uprobe/ret]: e->ip:%lx,bp:%x,caller_bp:%x", e->ip, e->bp, e->caller_bp);
+	cw_bpf_debug("[Go] [uprobe/ret]: sp:%x, goid: %llu, goid:0x:%x", PT_REGS_SP(ctx), e->goid, e->goid);
 
 	cw_bpf_debug("[Go] [uprobe/ret]: event: location:%x,ip:%x,time_ns_end:%lld\n", e->location, e->ip, e->time_ns_end);
 
-	struct trace_stack_entry_index_key_t trace_index_key = {};
-	trace_index_key.goid = e->goid;
-	trace_index_key.trace_id = trace_id;
-
-	__u64 *fun_key_p = bpf_map_lookup_elem(&trace_stack_entry_index, &trace_index_key);
-	__u64 fun_key = 1;
-	if (fun_key_p) {
-		cw_bpf_debug("[Go] [uprobe/ret]: get fun_key: %d", *fun_key_p);
-		fun_key = *fun_key_p;
-	} else {
-		cw_bpf_debug("[Go] [uprobe/ret]:ErrorErrorErrorError Not get fun_key");
-	}
-	struct trace_stack_entry_key_t trace_key = {};
-	trace_key.goid = e->goid;
-	trace_key.trace_id = trace_id;
-	trace_key.fun_key = fun_key;
+	// struct trace_stack_entry_key_t trace_key = {};
+	// trace_key.caller_bp = caller_bp % 0x800 + (e->goid << 12);
+	// trace_key.bp = e->bp % 0x800 + (e->goid << 12);
 
-	struct event *event_p = bpf_map_lookup_elem(&trace_stack_entry, &trace_key);
+	// trace_key.caller_bp = caller_bp;
+	// trace_key.bp = e->bp;
 
-	if (!event_p) {
-		cw_bpf_debug("[Go] [uprobe/ret]:ErrorErrorErrorError Not get funEntry");
-		return 0;
-	}
+	// cw_bpf_debug("[Go] [uprobe/ret]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x", trace_key.caller_bp, trace_key.bp);
+	// cw_bpf_debug("[Go] [uprobe/ret]: trace_keytrace_keytrace_key: bp: %x", trace_key.bp);
+
+	// struct event *event_p = bpf_map_lookup_elem(&trace_stack_entry, &trace_key);
 
-	event_p->time_ns_end = e->time_ns_end;
+	// if (!event_p) {
+	// 	cw_bpf_debug("[Go] [uprobe/ret]:ErrorErrorErrorError Not get funEntry");
+	// 	return 0;
+	// }
 
-	cw_bpf_debug("[Go] [uprobe/ret]: ent:event: location:%d,ip:%x,time_ns_start:%lld\n", event_p->location, event_p->ip, event_p->time_ns_start);
-	cw_bpf_debug("[Go] [uprobe/ret]: ent:event: bp:%x,caller_bp:%x,caller_ip:%x\n", event_p->bp,event_p->caller_bp,event_p->caller_ip);
-	cw_bpf_debug("[Go] [uprobe/ret]: ent:event: nid:%d,fpid:%d,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+	// event_p->time_ns_end = e->time_ns_end;
 
-	bpf_map_delete_elem(&trace_stack_entry, &trace_key);
-	__u64 fun_key_new = fun_key - 1;
-	bpf_map_update_elem(&trace_stack_entry_index, &trace_index_key, &fun_key_new, BPF_ANY);
+	// cw_bpf_debug("[Go] [uprobe/ret]: ent:event: location:%d,ip:%x,time_ns_start:%lld\n", event_p->location, event_p->ip, event_p->time_ns_start);
+	// cw_bpf_debug("[Go] [uprobe/ret]: ent:event: bp:%x,caller_bp:%x,caller_ip:%x\n", event_p->bp,event_p->caller_bp,event_p->caller_ip);
+	// cw_bpf_debug("[Go] [uprobe/ret]: ent:event: nid:%llx,fpid:%llx,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+
+	// bpf_map_delete_elem(&trace_stack_entry, &trace_key);
 
 	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
-	// 最后一个函数结束，同时推送协程信息
-	if (fun_key == 1) {
-		struct event event_coroutine = {};
-		event_coroutine.bp = e->bp,
-		event_coroutine.caller_bp = e->caller_bp,
-		event_coroutine.caller_ip = e->caller_ip,
-		event_coroutine.pid = e->pid,
-		event_coroutine.trace_id = e->trace_id,
-		event_coroutine.goid = e->goid,
-		event_coroutine.ip = e->ip,
-		event_coroutine.time_ns_start = e->time_ns_start,
-		event_coroutine.location = 2,
-		event_coroutine.nid = e->goid,
-		event_coroutine.fpid = 1,
-		event_coroutine.level = 2,
-		cw_bpf_debug("[Go] [uprobe/ret]: ent:event_coroutine push: nid:%d,fpid:%d,level:%d\n", event_coroutine.nid,event_coroutine.fpid,event_coroutine.level);
-		bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, &event_coroutine, sizeof(event_coroutine));
-	}
-	cw_bpf_debug("[Go] [uprobe/ret]: ent:event_ret push: nid:%d,fpid:%d,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
-	bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, event_p, sizeof(*e));
+	// cw_bpf_debug("[Go] [uprobe/ret]: ent:event_ret push: nid:%llx,fpid:%llx,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+	bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, e, sizeof(*e));
 
 	cw_bpf_debug("[Go] [uprobe/ret] end");
 	return 1;
@@ -285,4 +213,87 @@ int ret(struct pt_regs *ctx)
 
 
 
+SEC("uprobe/dotnetent")
+int dotnetent(struct pt_regs *ctx)
+{
+    cw_bpf_debug("[Go] [uprobe/ent] enter");
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+
+	__u64 trace_id = get_apm_trace_id(pid, pid_tgid);
+
+	__u32 key = 0;
+	struct event *e = bpf_map_lookup_elem(&event_stack, &key);
+	if (!e)
+	{
+		cw_bpf_debug("[Go] [uprobe/ret]: conot get event");
+		return 0;
+	}
+	__builtin_memset(e, 0, sizeof(*e));
+
+	// e->goid = get_goid();
+	e->goid = get_current_goroutine();
+
+	// e->ip = ctx->ip;
+	e->ip = PT_REGS_IP(ctx); // 当前函数的地址
+	if (!bpf_map_lookup_elem(&should_trace_goid, &e->goid))
+	{
+		__u64 should_trace = true;
+		bpf_map_update_elem(&should_trace_goid, &e->goid, &should_trace, BPF_ANY);
+	}
+	cw_bpf_debug("[Go] [uprobe/ent]: e->goid: %llu", e->goid);
+	cw_bpf_debug("[Go] [uprobe/ent]: e->ip: %d", e->ip);
+	cw_bpf_debug("[Go] [uprobe/ent]: yes");
+
+	__u64 asm_;
+	bpf_probe_read_user(&asm_, sizeof(asm_), (void*)e->ip);
 
+	asm_ = asm_ >> 32;
+	cw_bpf_debug("[Go] [uprobe/ent]: e->asm: %x", asm_);
+
+	e->pid = pid;
+	e->trace_id = trace_id;
+	e->location = ENTPOINT; // 标识入口还是出口
+	e->time_ns_start = bpf_ktime_get_ns();
+	e->bp = PT_REGS_SP(ctx) + asm_; // 因为 ebpf 程序执行时实际上还没进行堆栈的切换，所以此处是需要在父函数中获取子函数的基地址，新的堆栈会将父函数的 rbp 入栈，一个 rbp 在 64位机器上占用 8 个字节，所以需要减 8，获取到子函数的基地址，低地址的需要减8，如果到高地址的cpu 上就得+8
+	e->caller_bp = PT_REGS_FP(ctx); // 父函数的基地址
+
+	void *ra;
+	ra = (void *)PT_REGS_SP(ctx); // 父函数的 sp，此时 sp 指向父函数的最后一个地址
+	bpf_probe_read_user(&e->caller_ip, sizeof(e->caller_ip), ra);
+
+	cw_bpf_debug("[Go] [uprobe/ent]: goid: %llu", e->goid);
+
+	cw_bpf_debug("[Go] [uprobe/ent]: event: location:%x,ip:%lx,time_ns_start:%lld\n", e->location, e->ip, e->time_ns_start);
+	cw_bpf_debug("[Go] [uprobe/ent]: event: bp:%x,caller_bp:%x,caller_ip:%x\n", e->bp,e->caller_bp,e->caller_ip);
+
+	// struct trace_stack_entry_key_t trace_key = {};
+	// trace_key.caller_bp = e->caller_bp % 0x800 + (e->goid << 12);
+	// trace_key.bp = e->bp % 0x800 + (e->goid << 12);
+
+	// trace_key.caller_bp = e->caller_bp;
+	// trace_key.bp = e->bp;
+	// cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: bp: %x", trace_key.bp);
+	// cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x", trace_key.caller_bp, trace_key.bp);
+
+	// struct event event_current = {};
+	// event_current.bp = e->bp,
+	// event_current.caller_bp = e->caller_bp,
+	// event_current.caller_ip = e->caller_ip,
+	// event_current.pid = e->pid,
+	// event_current.trace_id = e->trace_id,
+	// event_current.goid = e->goid,
+	// event_current.ip = e->ip,
+	// event_current.time_ns_start = e->time_ns_start,
+	// event_current.location = e->location,
+	// event_current.nid = ((e->ip % 0x1000) << 24) + e->bp % 0x800 + (e->goid << 12),
+	// event_current.fpid = e->caller_bp % 0x800 + (e->goid << 12),
+
+	// bpf_map_update_elem(&trace_stack_entry, &trace_key, &event_current, BPF_ANY);
+
+	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
+	bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, e, sizeof(*e));
+
+	cw_bpf_debug("[Go] [uprobe/ent] end");
+	return 1;
+}

ebpftracer/stack.go

@@ -2,6 +2,7 @@ package ebpftracer
 
 import (
 	"context"
+	"debug/dwarf"
 	"debug/elf"
 	debugelf "debug/elf"
 	"fmt"
@@ -16,47 +17,225 @@ import (
 	"github.com/cilium/ebpf/link"
 	"github.com/coroot/coroot-node-agent/ebpftracer/tracer"
 	"github.com/coroot/coroot-node-agent/proc"
+	"golang.org/x/arch/arm64/arm64asm"
+	"golang.org/x/arch/x86/x86asm"
 	"golang.org/x/sync/semaphore"
 )
 
+type uprobesDef struct {
+	Name       string
+	Offset     uint64
+	EntAddress uint64
+	RetAddress uint64
+}
+
 func (t *Tracer) stack() error {
-	// uprobes := []tracer.Uprobe{}
 	if t.disableL7Tracing {
 		return nil
 	}
 
+	binType := "dotnet"
+	MatchString := ".*HandleFunc|.*main.*|testfun.*|.*serverHandler.*|.*ServeHTTP.*"
+	dbgpath := ""
+
 	ENV_PID := os.Getenv("FILTER_PID")
 	WHITE_LIST := os.Getenv("WHITE_LIST")
+	BIN_TYPE := os.Getenv("BIN_TYPE")
+	DBG_PATH := os.Getenv("DBG_PATH")
+
 	if ENV_PID == "" {
 		return nil
 	}
 
-	MatchString := ".*HandleFunc|.*main.*|testfun.*|.*serverHandler.*|.*ServeHTTP.*"
-
 	if WHITE_LIST != "" {
 		MatchString = WHITE_LIST
 	}
 
-	fmt.Println("UprobesMatchString::: init", MatchString)
+	if DBG_PATH != "" {
+		dbgpath = DBG_PATH
+	}
+
+	if BIN_TYPE != "" {
+		binType = BIN_TYPE
+	}
+
+	fmt.Println("UprobesMatchString:::init", MatchString)
 
 	pid, _ := strconv.ParseInt(ENV_PID, 10, 32)
 	path := proc.Path(uint32(pid), "exe")
 
-	t.Uprobes, _ = t.getUprobes(path, MatchString)
+	if dbgpath != "" {
+		t.Uprobes, _ = t.getJavaAOTUprobes(binType, path, dbgpath, MatchString)
+	} else {
+		t.Uprobes, _ = t.getUprobes(path, MatchString)
+	}
+
 	t.UprobesMap = map[string]tracer.Uprobe{}
-	fmt.Println("UprobesMap::: init")
+	fmt.Println("UprobesMap:::init")
 	for _, up := range t.Uprobes {
-		fmt.Println("UprobesMap:::", up.Funcname)
-		t.UprobesMap[fmt.Sprintf("%s", up.Funcname)] = up
+		fmt.Println("UprobesMap:::", up.Funcname, up.Address, up.AbsOffset)
+		t.UprobesMap[fmt.Sprintf("%s-%s", up.Funcname, up.Address+up.AbsOffset)] = up
 	}
 	links := t.attachUprobes(path, t.Uprobes)
 	t.links = append(t.links, links...)
 
 	// defer t.detachUprobes(links)
-
 	return nil
 }
 
+func (t *Tracer) getJavaAOTUprobes(binType, path string, dbgpath string, MatchString string) ([]tracer.Uprobe, error) {
+	uprobes := []tracer.Uprobe{}
+
+	elfFile, err := elf.Open(path)
+
+	funSection := ".text"
+
+	if binType == "dotnet" {
+		funSection = "__managedcode"
+	}
+
+	textSection := elfFile.Section(funSection)
+	if textSection == nil {
+		fmt.Println("no text section", nil)
+		return nil, nil
+	}
+	textSectionData, err := textSection.Data()
+	if err != nil {
+		fmt.Println("failed to read text section", err)
+		return nil, nil
+	}
+	textSectionLen := uint64(len(textSectionData) - 1)
+
+	dwarfFile, err := elf.Open(dbgpath)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	dwarfData, err := dwarfFile.DWARF()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	entryReader := dwarfData.Reader()
+
+	// var targetAddress uint64
+
+	listEntry := make(map[dwarf.Offset]uprobesDef)
+	SpecListEntry := []dwarf.Entry{}
+
+	for {
+		entry, err := entryReader.Next()
+		if err == io.EOF {
+			// We've reached the end of DWARF entries
+			break
+		}
+		if err != nil {
+			log.Fatalf("Error reading entry: %v", err)
+		}
+		if entry == nil {
+			log.Println("Warning: a nil entry was returned with no error")
+			break
+		}
+		if entry.Tag == dwarf.TagSubprogram {
+			// fmt.Printf("entry address: %x, %d\n", entry.Offset, entry.Children)
+			funName, _ := entry.Val(dwarf.AttrName).(string)
+			found, _ := regexp.MatchString(MatchString, funName)
+			if found {
+				entAddress, _ := entry.Val(dwarf.AttrLowpc).(uint64)
+				retAddress, _ := entry.Val(dwarf.AttrHighpc).(uint64)
+				// fmt.Printf("Function %s address: %x, %x\n", funName, address, entry.Offset)
+				uprobes := uprobesDef{}
+				uprobes.EntAddress = entAddress
+				uprobes.RetAddress = retAddress
+				uprobes.Offset = uint64(entry.Offset)
+				uprobes.Name = funName
+
+				listEntry[entry.Offset] = uprobes
+			}
+
+			specAddr, _ := entry.Val(dwarf.AttrSpecification).(dwarf.Offset)
+			lowpc := entry.Val(dwarf.AttrLowpc)
+			if lowpc != nil && specAddr > 0 && lowpc.(uint64) > 0 {
+				// fmt.Printf("AttrSpecification address: %x, %x\n", specAddr, entry.Offset)
+				SpecListEntry = append(SpecListEntry, *entry)
+			}
+		}
+	}
+
+	for _, v := range SpecListEntry {
+		specAddr, _ := v.Val(dwarf.AttrSpecification).(dwarf.Offset)
+		// fmt.Printf("SpecListEntrySpecListEntrySpecListEntry Attach Function: %x\n", specAddr)
+		_, ok := listEntry[specAddr]
+		if ok {
+			vv := listEntry[specAddr]
+			entAddr := v.Val(dwarf.AttrLowpc)
+			if entAddr != nil {
+				vv.EntAddress = entAddr.(uint64)
+			}
+			retAddr := v.Val(dwarf.AttrHighpc)
+			if retAddr != nil {
+				switch retAddr.(type) {
+				case uint64:
+					vv.RetAddress = uint64(retAddr.(uint64))
+				case int64:
+					vv.RetAddress = uint64(retAddr.(int64))
+				default:
+					fmt.Println("Unknown type")
+				}
+			}
+			listEntry[specAddr] = vv
+		}
+	}
+
+	for _, v := range listEntry {
+		fmt.Printf("Need Attach Function %s address: %x, %x\n", v.Name, v.EntAddress, v.RetAddress)
+
+		sStart := v.EntAddress - textSection.Addr
+		sSize := v.RetAddress
+		if v.RetAddress > v.EntAddress {
+			sSize = v.RetAddress - v.EntAddress
+		}
+		sEnd := sStart + sSize
+		if sEnd > textSectionLen {
+			continue
+		}
+		sBytes := textSectionData[sStart:sEnd]
+		rbpOffsets := getRbpEnterOffsets(elfFile.Machine, sBytes)
+		returnOffsets := getReturnOffsets(elfFile.Machine, sBytes)
+
+		if rbpOffsets != 0 {
+			uprobes = append(uprobes, tracer.Uprobe{
+				Funcname:  v.Name,               // 函数名
+				Location:  tracer.AtDotNetEntry, // 入口
+				Address:   v.EntAddress,         // 函数地址
+				AbsOffset: uint64(rbpOffsets),   // 函数相对 ELF 偏移
+				RelOffset: 0,                    // 函数真实偏移
+			})
+		} else {
+			// 函数入口加入待 attach 列表
+			uprobes = append(uprobes, tracer.Uprobe{
+				Funcname:  v.Name,         // 函数名
+				Location:  tracer.AtEntry, // 入口
+				Address:   v.EntAddress,   // 函数地址
+				AbsOffset: 0,              // 函数相对 ELF 偏移
+				RelOffset: 0,              // 函数真实偏移
+			})
+		}
+
+		for _, offset := range returnOffsets {
+			uprobes = append(uprobes, tracer.Uprobe{
+				Funcname:  v.Name,
+				Location:  tracer.AtRet,
+				Address:   v.EntAddress,
+				AbsOffset: uint64(offset),
+				RelOffset: 0,
+			})
+		}
+	}
+	return uprobes, nil
+}
+
 func (t *Tracer) getUprobes(path string, MatchString string) ([]tracer.Uprobe, error) {
 	uprobes := []tracer.Uprobe{}
 
@@ -109,7 +288,7 @@ func (t *Tracer) getUprobes(path string, MatchString string) ([]tracer.Uprobe, e
 
 		if found {
 			// 匹配到了加入 attachFuncs 列表
-			fmt.Println("Fuck This: ", symbol.Name)
+			fmt.Printf("Fuck This: %s, %x\n", symbol.Name, symbol.Value)
 			// attachFuncs = append(attachFuncs, symbol.Name)
 			// 根据函数名拿到当前函数的符号结构体
 			sym := symnames[symbol.Name]
@@ -169,28 +348,25 @@ func (t *Tracer) attachUprobes(path string, uprobes []tracer.Uprobe) []link.Link
 	if err != nil {
 		return nil
 	}
-	fmt.Println("AttachAttachAttach", path)
-
-	for i, up := range t.uprobes {
-		fmt.Println("attachingERROR---", i, up)
-	}
+	fmt.Println("Attach Start", path)
 
 	for i, up := range uprobes {
-		fmt.Printf("attaching %d -> %d -> %s -> 0x%x -> 0x%x\n", i, len(uprobes), up.Funcname, up.AbsOffset, up.Address)
+		fmt.Printf("attaching %d -> %d -> %s -> 0x%x -> 0x%x -> 0x%x\n", i, len(uprobes), up.Funcname, up.AbsOffset, up.Address, up.AbsOffset+up.Address)
 		var prog *ebpf.Program
 		switch up.Location {
 		case tracer.AtEntry:
 			prog = t.uprobes["ent"]
 		case tracer.AtRet:
 			prog = t.uprobes["ret"]
+		case tracer.AtDotNetEntry:
+			prog = t.uprobes["dotnetent"]
 		}
-		fmt.Println("progprogprogprogprogprog---", prog)
-		up, err := ex.Uprobe(up.Funcname, prog, &link.UprobeOptions{Address: up.Address, Offset: up.AbsOffset})
+		uplink, err := ex.Uprobe(up.Funcname, prog, &link.UprobeOptions{Address: up.Address, Offset: up.AbsOffset})
 		if err != nil {
-			fmt.Println("attachingERROR", err)
+			fmt.Printf("attachingERROR:%v, %v, %v\n", err, up, uplink)
 			// return nil
 		} else {
-			links = append(links, up)
+			links = append(links, uplink)
 		}
 	}
 	return links
@@ -208,3 +384,26 @@ func (t *Tracer) detachUprobes(links []link.Link) {
 	}
 	fmt.Println()
 }
+
+func getRbpEnterOffsets(machine elf.Machine, instructions []byte) int {
+	switch machine {
+	case elf.EM_X86_64:
+		for i := 0; i < len(instructions); {
+			ins, err := x86asm.Decode(instructions[i:], 64)
+			if err == nil && ins.Op == x86asm.LEA && ins.Args[0].String() == "RBP" {
+				fmt.Printf("getRbpEnterOffsets: %v, %s, %s\n", ins, ins.Args[0].String(), ins.Args[1].String())
+				return i
+			}
+			i += ins.Len
+		}
+	case elf.EM_AARCH64:
+		for i := 0; i < len(instructions); {
+			ins, err := arm64asm.Decode(instructions[i:])
+			if err == nil && ins.Op == arm64asm.RET {
+				return i
+			}
+			i += 4
+		}
+	}
+	return 0
+}

ebpftracer/tracer.go

@@ -488,9 +488,6 @@ type StackEvent struct {
 type StackFunEvent struct {
 	StackEvent StackEvent
 	Uprobe     *tracer.Uprobe
-	Level      int
-	Pid        int
-	Nid        int
 }
 
 func runEventsReader(name string, r *perf.Reader, ch chan<- Event, typ perfMapType) {

ebpftracer/tracer/uprobe.go

@@ -5,6 +5,7 @@ type UprobeLocation int
 const (
 	AtEntry UprobeLocation = iota
 	AtRet
+	AtDotNetEntry
 	AtGoroutineExit
 	AtGoroutineNewproc1
 	AtGoroutineNewproc1Ret

pkg/go.opentelemetry.io/otel/exporters/otlp/otlptrace/apm_exporter.go

@@ -4,14 +4,16 @@ import (
 	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
 
+	"time"
+
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/internal/tracetransform"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
 	tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
-	"time"
 )
 
 const (
@@ -93,6 +95,7 @@ type MapInfoT struct {
 	ServiceName    string   `json:"service_name"`
 	ServiceType    string   `json:"service_type"`
 	StartTime      uint64   `json:"start_time"`
+	EndTime        uint64   `json:"end_time"`
 	WallTime       uint64   `json:"wall_time"`
 	Schema         string   `json:"schema,omitempty"`
 	AssumedAppId   int64    `json:"assumed_app_id,omitempty"`
@@ -142,6 +145,7 @@ func tracetransformData(sdl []tracesdk.ReadOnlySpan) []RootDataT {
 	sendData := []RootDataT{}
 	for traceId, v := range TraceRootMap {
 		if v.TheEnd {
+			buildLevel(v)
 			sendData = append(sendData, v.RootData)
 			delete(TraceRootMap, traceId)
 			//fmt.Println("the end!")
@@ -159,6 +163,71 @@ func tracetransformData(sdl []tracesdk.ReadOnlySpan) []RootDataT {
 	return sendData
 }
 
+type TimeMap struct {
+	Time uint64
+	Type int
+	Map  *MapInfoT
+}
+
+func buildLevel(sdl *TraceMapT) {
+	nidMap := make(map[int]*MapInfoT)
+
+	mapSlice := []TimeMap{}
+
+	for i, v := range sdl.RootData.Maps {
+		if v.ServiceType == "APPLICATION" {
+			continue
+		}
+		nidMap[v.Nid] = &sdl.RootData.Maps[i]
+		timeStartMap := TimeMap{
+			Time: v.StartTime,
+			Type: 0,
+			Map:  &sdl.RootData.Maps[i],
+		}
+		mapSlice = append(mapSlice, timeStartMap)
+		timeEndMap := TimeMap{
+			Time: v.EndTime,
+			Type: 1,
+			Map:  &sdl.RootData.Maps[i],
+		}
+		mapSlice = append(mapSlice, timeEndMap)
+	}
+	sort.Slice(mapSlice, func(i, j int) bool {
+		return mapSlice[i].Time < mapSlice[j].Time
+	})
+
+	funStack := []TimeMap{}
+
+	currentNid := 1
+	Nid := 2
+	level := 2
+
+	for _, v := range mapSlice {
+		// fmt.Println("SliceSliceindex", k, "value", v.Time, v.Type, v.Map.MethodName, v.Map.Nid)
+		if v.Type == 0 {
+			// 函数入口
+			funStack = append(funStack, v)
+			v.Map.Pid = currentNid
+			v.Map.Level = level
+			v.Map.Nid = Nid
+			currentNid = Nid
+			level += 1
+			Nid += 1
+		} else if v.Type == 1 {
+			// 函数出口
+			len := len(funStack)
+			funStack = funStack[:len-1]
+			if (len - 2) < 0 {
+				currentNid = 1
+			} else {
+				currentNid = funStack[len-2].Map.Nid
+			}
+
+			level -= 1
+		}
+	}
+}
+
 func initRootData(traceId string) RootDataT {
 	data := RootDataT{
 		AccountId:      110,
@@ -222,7 +291,8 @@ func initMapNode(spanSd *tracepb.Span) (MapInfoT, string) {
 	mNode.MethodName = spanSd.Name
 	mNode.PureTime = (spanSd.EndTimeUnixNano - spanSd.StartTimeUnixNano) / 1e3
 	mNode.WallTime = mNode.PureTime
-	mNode.StartTime = spanSd.StartTimeUnixNano / 1e6
+	mNode.StartTime = spanSd.StartTimeUnixNano
+	mNode.EndTime = spanSd.EndTimeUnixNano
 
 	for _, attr := range spanSd.GetAttributes() {
 		switch attr.Key {
@@ -253,7 +323,7 @@ func buildAndAssemblyMap(sd apmTraceSpan, traceRoot *TraceMapT) MapInfoT {
 		buildRedisMap(&mNode, sd)
 	}
 	if mapType != "" {
-		// mNode.Nid = traceRoot.Index
+		mNode.Nid = traceRoot.Index
 		traceRoot.RootData.Maps = append(traceRoot.RootData.Maps, mNode)
 	}
 	return mNode

run.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+pid=`ps aux | grep ebpfdemo81 | grep -v grep | awk '{print $2}'`
+echo $pid
+TRACES_ENDPOINT=http://10.2.31.73:8099/docp/api/v2/data/receive FILTER_PID=$pid WHITE_LIST=".*HandleFunc|.*main.*|.*serverHandler.*|.*ServeHTTP.*" ./euspace --listen="0.0.0.0:8123"

tracing/apm_tracing.go

@@ -3,15 +3,18 @@ package tracing
 import (
 	"context"
 	"fmt"
+	"sort"
 	"time"
 
+	"strconv"
+
+	"github.com/coroot/coroot-node-agent/ebpftracer"
 	"github.com/coroot/coroot-node-agent/ebpftracer/l7"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	semconv "go.opentelemetry.io/otel/semconv/v1.18.0"
 	"go.opentelemetry.io/otel/trace"
 	"inet.af/netaddr"
-	"strconv"
 )
 
 /**
@@ -30,6 +33,79 @@ func (t *Trace) setSpan(span trace.Span) {
 	t.span = span
 }
 
+type TimeMap struct {
+	Time uint64
+	Type int
+	Map  *ebpftracer.StackFunEvent
+}
+
+func (t *Trace) buildFun() {
+	mapSlice := []TimeMap{}
+	for i, v := range t.stack {
+		timeStartMap := TimeMap{}
+		if v.StackEvent.Location == 0 {
+			timeStartMap = TimeMap{
+				Time: v.StackEvent.TimeNsStart,
+				Type: 0,
+				Map:  &t.stack[i],
+			}
+		} else {
+			timeStartMap = TimeMap{
+				Time: v.StackEvent.TimeNsEnd,
+				Type: 1,
+				Map:  &t.stack[i],
+			}
+		}
+		mapSlice = append(mapSlice, timeStartMap)
+	}
+	sort.Slice(mapSlice, func(i, j int) bool {
+		return mapSlice[i].Time < mapSlice[j].Time
+	})
+
+	funStack := []TimeMap{}
+
+	currentfunNum := 1
+
+	// for k, v := range mapSlice {
+	// 	fmt.Println("---SliceSliceindex", k, "value", v.Time, v.Type, v.Map.Uprobe.Funcname, v.Map.StackEvent.Nid)
+	// }
+
+	mapSliceLen := len(mapSlice)
+	for k, v := range mapSlice {
+		// fmt.Println("SliceSliceindex", k, "value", v.Time, v.Type, v.Map.Uprobe.Funcname, v.Map.StackEvent.Nid)
+		if v.Type == 0 {
+			// 函数入口
+			funStack = append(funStack, v)
+		} else if v.Type == 1 {
+			// 函数出口
+			len := len(funStack)
+			if len < 1 {
+				fmt.Printf("buildFun ErrorError return before enter: %v\n", v)
+				continue
+			}
+			currnt := funStack[len-1]
+			if currnt.Map.StackEvent.Location != 0 {
+				fmt.Printf("currnt StackEvent ErrorError is not enter: %v\n", v)
+				continue
+			}
+			if k < mapSliceLen-1 && len >= 2 {
+				nextfun := mapSlice[k+1]
+				preCurrnt := funStack[len-2]
+				// // 处理 .netcore 多次 returun
+				// 下一个事件是 return 并且函数名跟当前事件是一样的，且上一个函数不是当前函数
+				if nextfun.Map.StackEvent.Location == 1 && nextfun.Map.Uprobe.Funcname == currnt.Map.Uprobe.Funcname && preCurrnt.Map.Uprobe.Funcname != currnt.Map.Uprobe.Funcname {
+					currentfunNum++
+					continue
+				}
+			}
+			funStack = funStack[:len-1]
+			duration := v.Map.StackEvent.TimeNsEnd - currnt.Map.StackEvent.TimeNsStart
+			t.FuncTraceQuery(currnt.Map.Uprobe.Funcname, time.Duration(duration), currnt.Map.StackEvent.TimeNsStart, v.Map.StackEvent.TimeNsEnd, currentfunNum)
+			currentfunNum = 1
+		}
+	}
+}
+
 func (t *Trace) TraceStart(method, path string, status l7.Status, duration time.Duration) {
 	if t == nil || method == "" {
 		return
@@ -68,6 +144,10 @@ func (t *Trace) TraceEnd(r *l7.RequestData) {
 	if r.ParentSpanContext.SpanIdFrom != "0000000000000000" {
 		t.span.SetAttributes(attribute.String("server.span_id_from", r.ParentSpanContext.SpanIdFrom))
 	}
+	// for _, v := range t.stack {
+	// 	fmt.Printf("TraceEndTraceEndTraceEnd%s\n", v)
+	// }
+	t.buildFun()
 	t.span.End(trace.WithTimestamp(time.Now()))
 }
 
@@ -146,13 +226,24 @@ func (t *Trace) HttpTraceRequest(method, path, ip string, port uint16, r *l7.Req
 	)
 }
 
-func (t *Trace) FuncTraceQuery(funcname string, duration time.Duration, level int, pid int, nid int) {
+func (t *Trace) FuncTraceQuery(funcname string, duration time.Duration, start uint64, end uint64, num int) {
 	if t == nil || funcname == "" {
 		return
 	}
-	t.createTraceSpan(funcname, duration, false,
-		attribute.Int("level", level),
-		attribute.Int("pid", pid),
-		attribute.Int("nid", nid),
-	)
+	t.createTraceSpanNoTime(funcname, duration, false, start, end, attribute.Int("num", num))
+}
+
+func (t *Trace) createTraceSpanNoTime(name string, duration time.Duration, error bool, start uint64, end uint64, attrs ...attribute.KeyValue) {
+	// end := time.Now()
+	// start := end.Add(-duration)
+	startTime := time.Unix(0, int64(start))
+	endTime := time.Unix(0, int64(end))
+	//fmt.Println("createTraceSpan:", t.ctx)
+	_, span := tracer(t.containerId).Start(t.ctx, name, trace.WithTimestamp(startTime), trace.WithSpanKind(trace.SpanKindClient))
+	span.SetAttributes(t.commonAttrs...)
+	span.SetAttributes(attrs...)
+	if error {
+		span.SetStatus(codes.Error, "")
+	}
+	span.End(trace.WithTimestamp(endTime))
 }

tracing/tracing.go

@@ -4,7 +4,11 @@ import (
 	"context"
 	"fmt"
 
+	"sync"
+	"time"
+
 	"github.com/coroot/coroot-node-agent/common"
+	"github.com/coroot/coroot-node-agent/ebpftracer"
 	"github.com/coroot/coroot-node-agent/ebpftracer/l7"
 	"github.com/coroot/coroot-node-agent/flags"
 	"go.opentelemetry.io/otel/attribute"
@@ -17,8 +21,6 @@ import (
 	"go.opentelemetry.io/otel/trace"
 	"inet.af/netaddr"
 	"k8s.io/klog/v2"
-	"sync"
-	"time"
 )
 
 const (
@@ -78,6 +80,7 @@ type Trace struct {
 	ctx         context.Context
 	span        trace.Span
 	lock        sync.RWMutex
+	stack       []ebpftracer.StackFunEvent
 }
 
 func NewTrace(containerId string, destination netaddr.IPPort) *Trace {
@@ -193,3 +196,7 @@ func (t *Trace) RedisQuery(cmd, args string, error bool, duration time.Duration)
 		semconv.DBStatement(statement),
 	)
 }
+
+func (t *Trace) FunAdd(stackFun ebpftracer.StackFunEvent) {
+	t.stack = append(t.stack, stackFun)
+}