Fixed #TASK_GK-2944 Go 纵向堆栈内核层拼接函数出入库

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "files.associations": {
+        "*.toml": "toml",
+        "numbers": "c"
+    }
+}

Makefile2

@@ -14,16 +14,16 @@ endif
 debug:
 	exit
 build:
-	CGO_ENABLED=1 go build -buildvcs=false -o euspace
+	CGO_ENABLED=1 go build -gcflags="all=-N -l" -buildvcs=false -o euspace
 c:
 	#docker exec -it 9d928d96d4d0 sh -c 'cd /opt/github/euspace/ebpftracer && sh build.sh${PARAMS}'
-	docker exec -it 62d0676aa0b7 sh -c 'cd /opt/github/euspace/ebpftracer && sh build_linux.sh${PARAMS}'
+	docker exec -it 3c1a4817a7ed sh -c 'cd /opt/github/euspace/ebpftracer && sh build_linux.sh ${PARAMS}'
 c-build: c
 
 go-build:
 	#ssh [email protected] 'export https_proxy=http://10.0.22.50:4780 && source ~/.g/env && cd /opt/github/euspace && make -f Makefile2 build'
-	docker exec -it 62d0676aa0b7 bash -c 'cd /opt/github/euspace && source ~/.g/env && make -f Makefile2 build'
-go: go-build run
+	docker exec -it 3c1a4817a7ed bash -c 'cd /opt/github/euspace && source ~/.g/env && make -f Makefile2 build'
+go: go-build
 
 run:
 	ssh [email protected] 'cd /opt/github/euspace && TRACES_ENDPOINT=http://10.0.12.192:18080/docp/api/v2/data/receive ${FILTER} ./euspace --listen="0.0.0.0:8123"'

build.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+rm ./euspace
+make -f Makefile2 all debug=1 pid=1121
+pid=`ps aux | grep ebpfdemo81 | grep -v grep | awk '{print $2}'`
+echo $pid
+TRACES_ENDPOINT=http://10.2.31.73:8099/docp/api/v2/data/receive SEND=1 FILTER_PID=$pid WHITE_LIST=".*HandleFunc|.*main.*|.*serverHandler.*|.*ServeHTTP.*" ./euspace  --listen="0.0.0.0:8123"

cbuild.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+rm ./euspace
+make -f Makefile2 c debug=1 pid=1121
+# pid=`ps aux | grep ebpfdemo81 | grep -v grep | awk '{print $2}'`
+# echo $pid
+# TRACES_ENDPOINT=http://10.2.29.1:8099/docp/api/v2/data/receive FILTER_PID=$pid ./euspace --listen="0.0.0.0:8123"

cmd

@@ -0,0 +1,2 @@
+make -f Makefile2 all debug=1 pid=1121
+TRACES_ENDPOINT=http://10.2.29.1:8099/docp/api/v2/data/receive FILTER_PID=493419 ./euspace --listen="0.0.0.0:8123"

containers/container_apm.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math/rand"
 	"sort"
-	"strconv"
 	"time"
 
 	"github.com/coroot/coroot-node-agent/ebpftracer"
@@ -205,32 +204,24 @@ func (c *Container) StackProcess(event ebpftracer.StackEvent, tracer *ebpftracer
 	case 0: // ret
 		uprobe, err := c.GetUprobe(event, tracer)
 		if err != nil {
-			//fmt.Println("GetUprobeGetUprobe errer: %v", err)
+			fmt.Println("GetUprobeGetUprobe errer: %v", err)
 			// log.Errorf("failed to get uprobe for event %+v: %+v", event, err)
 			return
 		}
 
 		if event.TraceId <= 0 {
-			//fmt.Println("StackProcess TraceId id 0")
+			fmt.Println("StackProcess TraceId id 0")
 			// log.Errorf("failed to get uprobe for event %+v: %+v", event, err)
 			return
 		}
 
-		//fmt.Println("StackProcess 函数入口开始处理 fun:", event.TraceId, uprobe.Funcname, event.TimeNsEnd-event.TimeNsStart)
+		fmt.Println("StackProcess 函数入口开始处理 fun:", event.TraceId, uprobe.Funcname, event.TimeNsEnd-event.TimeNsStart)
 		apmTrace, ok := c.getTrace(event.TraceId)
 		if ok {
-			//fmt.Println("append FuncTraceQuery fun:", event.TraceId, uprobe.Funcname, event.Pid)
+			fmt.Println("append FuncTraceQuery fun:", event.TraceId, uprobe.Funcname, event.Pid)
 			duration := event.TimeNsEnd - event.TimeNsStart
 			apmTrace.FuncTraceQuery(uprobe.Funcname, time.Duration(duration), int(event.Level), int(event.Fpid), int(event.Nid))
 		}
-	case 2: // coroutine
-		//fmt.Println("StackProcess 协程入口开始处理 fun:", event.TraceId, event.Goid, event.TimeNsEnd-event.TimeNsStart)
-		apmTrace, ok := c.getTrace(event.TraceId)
-		if ok {
-			//fmt.Println("append FuncTraceQuery fun:", event.TraceId, "coroutine"+strconv.FormatUint(event.Goid, 10), event.Pid, event.Fpid)
-			duration := event.TimeNsEnd - event.TimeNsStart
-			apmTrace.FuncTraceQuery("coroutine"+strconv.FormatUint(event.Goid, 10), time.Duration(duration), int(event.Level), int(event.Fpid), int(event.Nid))
-		}
 	}
 }
 

containers/registry.go

@@ -300,14 +300,13 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 				}
 
 				if c := r.containersByPid[uint32(e.StackEvent.Pid)]; c != nil {
-					//fmt.Println("e.EventTypeFunEnt: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
-					//fmt.Println("e.EventTypeFunEnt: FPid:%d, Nid:%d, Level:%d", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
+					fmt.Println("e.EventTypeFunEnt: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
+					fmt.Println("e.EventTypeFunEnt: FPid:%d, Nid:%d, Level:%d", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
 					c.StackProcess(*e.StackEvent, r.tracer)
+				} else {
+					fmt.Println("e.EventTypeFunEnt ErrorError: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
+					fmt.Println("e.EventTypeFunEnt ErrorError: TraceId:%d, FPid:%d, Nid:%d, Level:%d", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
 				}
-				//else {
-				//	fmt.Println("e.EventTypeFunEnt ErrorError: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
-				//	fmt.Println("e.EventTypeFunEnt ErrorError: TraceId:%d, FPid:%d, Nid:%d, Level:%d", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
-				//}
 			}
 		}
 	}

ebpftracer/ebpf/utrace/go/net/stack.probe.bpf copy.c

@@ -0,0 +1,281 @@
+#include "arguments.h"
+#include "span_context.h"
+#include "go_context.h"
+#include "go_types.h"
+#include "uprobe.h"
+
+#define MAX_DATA_SIZE 64
+
+#define ENTPOINT 0
+#define RETPOINT 1
+
+// bpf write event data when enter/exit a function
+struct event
+{
+	__u64 pid;
+	__u64 trace_id;
+	__u64 goid;
+	__u64 ip;
+	__u64 bp;
+	__u64 caller_ip;
+	__u64 caller_bp;
+	__u64 time_ns_start;
+	__u64 time_ns_end;
+	__u64 nid;
+	__u64 fpid;
+	__u64 level;
+	__u8 location;
+}__attribute__((packed));
+
+struct trace_stack_entry_index_key_t {
+	__u32 trace_id;
+	__u64 goid;
+};
+
+struct trace_stack_entry_index_value_t {
+	__u64 fun_key;
+};
+
+struct trace_stack_entry_key_t {
+	__u32 trace_id;
+	__u64 goid;
+	__u64 fun_key;
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+    __uint(key_size, sizeof(int));
+    __uint(value_size, sizeof(int));
+} event_queue SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct trace_stack_entry_key_t));
+	__uint(value_size, sizeof(struct event));
+	__uint(max_entries, 32768);
+} trace_stack_entry SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct trace_stack_entry_index_key_t));
+	__uint(value_size, sizeof(__u64));
+	__uint(max_entries, 32768);
+} trace_stack_entry_index SEC(".maps");
+
+struct bpf_map_def SEC("maps") event_stack = {
+	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
+	.key_size = sizeof(__u32),
+	.value_size = sizeof(struct event),
+	.max_entries = 1,
+};
+
+struct bpf_map_def SEC("maps") should_trace_goid = {
+	.type = BPF_MAP_TYPE_HASH,
+	.key_size = sizeof(__u64),
+	.value_size = sizeof(bool),
+	.max_entries = 10000,
+};
+
+SEC("uprobe/ent")
+int ent(struct pt_regs *ctx)
+{
+    bpf_printk("[Go] [uprobe/ent] enter");
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+
+	__u64 trace_id = get_apm_trace_id(pid, pid_tgid);
+
+	__u32 key = 0;
+	struct event *e = bpf_map_lookup_elem(&event_stack, &key);
+	if (!e)
+	{
+		bpf_printk("[Go] [uprobe/ret]: conot get event");
+		return 0;
+	}
+	__builtin_memset(e, 0, sizeof(*e));
+
+	// e->goid = get_goid();
+	e->goid = get_current_goroutine();
+
+	// e->ip = ctx->ip;
+	e->ip = PT_REGS_IP(ctx); // 当前函数的地址
+	if (!bpf_map_lookup_elem(&should_trace_goid, &e->goid))
+	{
+		__u64 should_trace = true;
+		bpf_map_update_elem(&should_trace_goid, &e->goid, &should_trace, BPF_ANY);
+	}
+	bpf_printk("[Go] [uprobe/ent]: e->goid: %llu", e->goid);
+	bpf_printk("[Go] [uprobe/ent]: e->ip: %d", e->ip);
+	bpf_printk("[Go] [uprobe/ent]: yes");
+
+	e->pid = pid;
+	e->trace_id = trace_id;
+	e->location = ENTPOINT; // 标识入口还是出口
+	e->time_ns_start = bpf_ktime_get_ns();
+	e->bp = PT_REGS_SP(ctx) - 8; // 因为 ebpf 程序执行时实际上还没进行堆栈的切换，所以此处是需要在父函数中获取子函数的基地址，新的堆栈会将父函数的 rbp 入栈，一个 rbp 在 64位机器上占用 8 个字节，所以需要减 8，获取到子函数的基地址，低地址的需要减8，如果到高地址的cpu 上就得+8
+	e->caller_bp = PT_REGS_FP(ctx); // 父函数的基地址
+
+	void *ra;
+	ra = (void *)PT_REGS_SP(ctx); // 父函数的 sp，此时 sp 指向父函数的最后一个地址
+	bpf_probe_read_user(&e->caller_ip, sizeof(e->caller_ip), ra);
+
+	bpf_printk("[Go] [uprobe/ent]: goid: %llu", e->goid);
+
+	bpf_printk("[Go] [uprobe/ent]: event: location:%x,ip:%x,time_ns_start:%lld\n", e->location, e->ip, e->time_ns_start);
+	bpf_printk("[Go] [uprobe/ent]: event: bp:%x,caller_bp:%x,caller_ip:%x\n", e->bp,e->caller_bp,e->caller_ip);
+
+	struct trace_stack_entry_index_key_t trace_index_key = {};
+	trace_index_key.goid = e->goid;
+	trace_index_key.trace_id = trace_id;
+
+	bpf_printk("[Go] [uprobe/ent]: trace_stack_entry_index_key_t: goid:%d,trace_id:%lld\n", trace_index_key.goid,trace_index_key.trace_id);
+	__u64 *fun_key_p = bpf_map_lookup_elem(&trace_stack_entry_index, &trace_index_key);
+	__u64 fun_key = 1;
+	__u64 fpid = e->goid;
+	if (fun_key_p) {
+		bpf_printk("[Go] [uprobe/ent]: get fun_key: %d", *fun_key_p);
+		struct trace_stack_entry_key_t trace_key_parent = {};
+		trace_key_parent.goid = e->goid;
+		trace_key_parent.trace_id = trace_id;
+		trace_key_parent.fun_key = *fun_key_p;
+
+		struct event *event_parent = bpf_map_lookup_elem(&trace_stack_entry, &trace_key_parent);
+		if (!event_parent) {
+			bpf_printk("[Go] [uprobe/ent]: get event_parent Error Error: %d", *fun_key_p);
+		} else {
+			fpid = event_parent->nid;
+		}
+		fun_key = *fun_key_p + 1;
+	}
+
+	struct trace_stack_entry_key_t trace_key = {};
+	trace_key.goid = e->goid;
+	trace_key.trace_id = trace_id;
+	trace_key.fun_key = fun_key;
+
+	struct event event_current = {};
+	event_current.bp = e->bp;
+	event_current.caller_bp = e->caller_bp;
+	event_current.caller_ip = e->caller_ip;
+	event_current.pid = e->pid;
+	event_current.trace_id = e->trace_id;
+	event_current.goid = e->goid;
+	event_current.ip = e->ip;
+	event_current.time_ns_start = e->time_ns_start;
+	event_current.location = e->location;
+	event_current.nid = bpf_get_prandom_u32() & 0xFF;
+	event_current.fpid = fpid;
+	// 第一级是 appliction，二级是 协程
+	event_current.level = fun_key + 2;
+
+	bpf_printk("[Go] [uprobe/ent]: uodate fun_key: %d", fun_key);
+	bpf_map_update_elem(&trace_stack_entry, &trace_key, &event_current, BPF_ANY);
+	bpf_map_update_elem(&trace_stack_entry_index, &trace_index_key, &fun_key, BPF_ANY);
+
+	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
+	// bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, e, sizeof(*e));
+
+	bpf_printk("[Go] [uprobe/ent] end");
+	return 1;
+}
+
+SEC("uprobe/ret")
+int ret(struct pt_regs *ctx)
+{
+    bpf_printk("[Go] [uprobe/ret] enter");
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+
+	__u64 trace_id = get_apm_trace_id(pid, pid_tgid);
+
+	__u32 key = 0;
+	struct event *e = bpf_map_lookup_elem(&event_stack, &key);
+	if (!e)
+	{
+		bpf_printk("[Go] [uprobe/ret]: conot get event");
+		return 0;
+	}
+	__builtin_memset(e, 0, sizeof(*e));
+
+	// e->goid = get_goid();
+	e->goid = get_current_goroutine();
+	if (!bpf_map_lookup_elem(&should_trace_goid, &e->goid))
+	{
+		bpf_printk("[Go] [uprobe/ret]: conot get event: %llu", e->goid);
+		return 0;
+	}
+	bpf_printk("[Go] [uprobe/ret]: yes");
+	e->pid = pid;
+	e->trace_id = trace_id;
+	e->location = RETPOINT;
+	e->ip = PT_REGS_IP(ctx);
+	e->time_ns_end = bpf_ktime_get_ns();
+	e->bp = PT_REGS_FP(ctx);
+	bpf_printk("[Go] [uprobe/ret]: e->ip:%x,bp:%x", e->ip, e->bp);
+	bpf_printk("[Go] [uprobe/ret]: goid: %llu", e->goid);
+
+	bpf_printk("[Go] [uprobe/ret]: event: location:%x,ip:%x,time_ns_end:%lld\n", e->location, e->ip, e->time_ns_end);
+
+	struct trace_stack_entry_index_key_t trace_index_key = {};
+	trace_index_key.goid = e->goid;
+	trace_index_key.trace_id = trace_id;
+
+	__u64 *fun_key_p = bpf_map_lookup_elem(&trace_stack_entry_index, &trace_index_key);
+	__u64 fun_key = 1;
+	if (fun_key_p) {
+		bpf_printk("[Go] [uprobe/ret]: get fun_key: %d", *fun_key_p);
+		fun_key = *fun_key_p;
+	} else {
+		bpf_printk("[Go] [uprobe/ret]:ErrorErrorErrorError Not get fun_key");
+	}
+	struct trace_stack_entry_key_t trace_key = {};
+	trace_key.goid = e->goid;
+	trace_key.trace_id = trace_id;
+	trace_key.fun_key = fun_key;
+
+	struct event *event_p = bpf_map_lookup_elem(&trace_stack_entry, &trace_key);
+
+	if (!event_p) {
+		bpf_printk("[Go] [uprobe/ret]:ErrorErrorErrorError Not get funEntry");
+		return 0;
+	}
+
+	event_p->time_ns_end = e->time_ns_end;
+
+	bpf_printk("[Go] [uprobe/ret]: ent:event: location:%d,ip:%x,time_ns_start:%lld\n", event_p->location, event_p->ip, event_p->time_ns_start);
+	bpf_printk("[Go] [uprobe/ret]: ent:event: bp:%x,caller_bp:%x,caller_ip:%x\n", event_p->bp,event_p->caller_bp,event_p->caller_ip);
+	bpf_printk("[Go] [uprobe/ret]: ent:event: nid:%d,fpid:%d,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+
+	bpf_map_delete_elem(&trace_stack_entry, &trace_key);
+	__u64 fun_key_new = fun_key - 1;
+	bpf_map_update_elem(&trace_stack_entry_index, &trace_index_key, &fun_key_new, BPF_ANY);
+
+	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
+	// 最后一个函数结束，同时推送协程信息
+	if (fun_key == 1) {
+		struct event event_coroutine = {};
+		event_coroutine.bp = e->bp;
+		event_coroutine.caller_bp = e->caller_bp;
+		event_coroutine.caller_ip = e->caller_ip;
+		event_coroutine.pid = e->pid;
+		event_coroutine.trace_id = e->trace_id;
+		event_coroutine.goid = e->goid;
+		event_coroutine.ip = e->ip;
+		event_coroutine.time_ns_start = e->time_ns_start;
+		event_coroutine.location = 2;
+		event_coroutine.nid = e->goid;
+		event_coroutine.fpid = 1;
+		event_coroutine.level = 2;
+		bpf_printk("[Go] [uprobe/ret]: ent:event_coroutine push: nid:%d,fpid:%d,level:%d\n", event_coroutine.nid,event_coroutine.fpid,event_coroutine.level);
+		bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, &event_coroutine, sizeof(event_coroutine));
+	}
+	bpf_printk("[Go] [uprobe/ret]: ent:event_ret push: nid:%d,fpid:%d,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+	bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, event_p, sizeof(*e));
+
+	bpf_printk("[Go] [uprobe/ret] end");
+	return 1;
+}
+
+
+
+

ebpftracer/ebpf/utrace/go/net/stack.probe.bpf.c

@@ -27,25 +27,16 @@ struct event
 	__u8 location;
 }__attribute__((packed));
 
-struct trace_stack_entry_index_key_t {
-	__u32 trace_id;
-	__u64 goid;
-};
-
-struct trace_stack_entry_index_value_t {
-	__u64 fun_key;
-};
-
 struct trace_stack_entry_key_t {
-	__u32 trace_id;
-	__u64 goid;
-	__u64 fun_key;
+	__u64 caller_bp;
+	__u64 bp;
 };
 
 struct {
     __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
     __uint(key_size, sizeof(int));
     __uint(value_size, sizeof(int));
+	__uint(max_entries, 32768);
 } event_queue SEC(".maps");
 
 struct {
@@ -55,21 +46,6 @@ struct {
 	__uint(max_entries, 32768);
 } trace_stack_entry SEC(".maps");
 
-struct {
-	__uint(type, BPF_MAP_TYPE_LRU_HASH);
-	__uint(key_size, sizeof(struct trace_stack_entry_index_key_t));
-	__uint(value_size, sizeof(__u64));
-	__uint(max_entries, 32768);
-} trace_stack_entry_index SEC(".maps");
-
-// struct bpf_map_def SEC("maps") event_queue = {
-// 	.type = BPF_MAP_TYPE_QUEUE,
-// 	.key_size = 0,
-// 	.value_size = sizeof(struct event),
-// 	.max_entries = 10000,
-// };
-
-
 struct bpf_map_def SEC("maps") event_stack = {
 	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
 	.key_size = sizeof(__u32),
@@ -78,10 +54,10 @@ struct bpf_map_def SEC("maps") event_stack = {
 };
 
 struct bpf_map_def SEC("maps") should_trace_goid = {
-	.type = BPF_MAP_TYPE_HASH,
+	.type = BPF_MAP_TYPE_LRU_HASH,
 	.key_size = sizeof(__u64),
 	.value_size = sizeof(bool),
-	.max_entries = 10000,
+	.max_entries = 32768,
 };
 
 SEC("uprobe/ent")
@@ -124,7 +100,7 @@ int ent(struct pt_regs *ctx)
 	e->caller_bp = PT_REGS_FP(ctx); // 父函数的基地址
 
 	void *ra;
-	ra = (void *)PT_REGS_SP(ctx); // 父函数的 sp，此时 sp 指向父函数的最后一个地址
+	ra = (void *)PT_REGS_FP(ctx); // 父函数的 sp，此时 sp 指向父函数的最后一个地址
 	bpf_probe_read_user(&e->caller_ip, sizeof(e->caller_ip), ra);
 
 	cw_bpf_debug("[Go] [uprobe/ent]: goid: %llu", e->goid);
@@ -132,34 +108,11 @@ int ent(struct pt_regs *ctx)
 	cw_bpf_debug("[Go] [uprobe/ent]: event: location:%x,ip:%x,time_ns_start:%lld\n", e->location, e->ip, e->time_ns_start);
 	cw_bpf_debug("[Go] [uprobe/ent]: event: bp:%x,caller_bp:%x,caller_ip:%x\n", e->bp,e->caller_bp,e->caller_ip);
 
-	struct trace_stack_entry_index_key_t trace_index_key = {};
-	trace_index_key.goid = e->goid;
-	trace_index_key.trace_id = trace_id;
-
-	cw_bpf_debug("[Go] [uprobe/ent]: trace_stack_entry_index_key_t: goid:%d,trace_id:%lld\n", trace_index_key.goid,trace_index_key.trace_id);
-	__u64 *fun_key_p = bpf_map_lookup_elem(&trace_stack_entry_index, &trace_index_key);
-	__u64 fun_key = 1;
-	__u64 fpid = e->goid;
-	if (fun_key_p) {
-		cw_bpf_debug("[Go] [uprobe/ent]: get fun_key: %d", *fun_key_p);
-		struct trace_stack_entry_key_t trace_key_parent = {};
-		trace_key_parent.goid = e->goid;
-		trace_key_parent.trace_id = trace_id;
-		trace_key_parent.fun_key = *fun_key_p;
-
-		struct event *event_parent = bpf_map_lookup_elem(&trace_stack_entry, &trace_key_parent);
-		if (!event_parent) {
-			cw_bpf_debug("[Go] [uprobe/ent]: get event_parent Error Error: %d", *fun_key_p);
-		} else {
-			fpid = event_parent->nid;
-		}
-		fun_key = *fun_key_p + 1;
-	}
-
 	struct trace_stack_entry_key_t trace_key = {};
-	trace_key.goid = e->goid;
-	trace_key.trace_id = trace_id;
-	trace_key.fun_key = fun_key;
+	trace_key.caller_bp = e->caller_bp % 0x800 + (e->goid << 12);
+	trace_key.bp = e->bp % 0x800 + (e->goid << 12);
+	// cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: bp: %x", trace_key.bp);
+	cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x", trace_key.caller_bp, trace_key.bp);
 
 	struct event event_current = {};
 	event_current.bp = e->bp,
@@ -171,13 +124,10 @@ int ent(struct pt_regs *ctx)
 	event_current.ip = e->ip,
 	event_current.time_ns_start = e->time_ns_start,
 	event_current.location = e->location,
-	event_current.nid = bpf_get_prandom_u32() & 0xFF,
-	event_current.fpid = fpid,
-	event_current.level = fun_key + 2, // 第一级是 appliction，二级是 协程
+	event_current.nid = ((e->ip % 0x1000) << 24) + e->bp % 0x800 + (e->goid << 12),
+	event_current.fpid = e->caller_bp % 0x800 + (e->goid << 12),
 
-	cw_bpf_debug("[Go] [uprobe/ent]: uodate fun_key: %d", fun_key);
 	bpf_map_update_elem(&trace_stack_entry, &trace_key, &event_current, BPF_ANY);
-	bpf_map_update_elem(&trace_stack_entry_index, &trace_index_key, &fun_key, BPF_ANY);
 
 	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
 	// bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, e, sizeof(*e));
@@ -218,27 +168,19 @@ int ret(struct pt_regs *ctx)
 	e->ip = PT_REGS_IP(ctx);
 	e->time_ns_end = bpf_ktime_get_ns();
 	e->bp = PT_REGS_FP(ctx);
+
+	__u64 caller_bp = PT_REGS_SP(ctx) - 8;
 	cw_bpf_debug("[Go] [uprobe/ret]: e->ip:%x,bp:%x", e->ip, e->bp);
-	cw_bpf_debug("[Go] [uprobe/ret]: goid: %llu", e->goid);
+	cw_bpf_debug("[Go] [uprobe/ret]: goid: %llu, goid:0x:%x", e->goid, e->goid);
 
 	cw_bpf_debug("[Go] [uprobe/ret]: event: location:%x,ip:%x,time_ns_end:%lld\n", e->location, e->ip, e->time_ns_end);
 
-	struct trace_stack_entry_index_key_t trace_index_key = {};
-	trace_index_key.goid = e->goid;
-	trace_index_key.trace_id = trace_id;
-
-	__u64 *fun_key_p = bpf_map_lookup_elem(&trace_stack_entry_index, &trace_index_key);
-	__u64 fun_key = 1;
-	if (fun_key_p) {
-		cw_bpf_debug("[Go] [uprobe/ret]: get fun_key: %d", *fun_key_p);
-		fun_key = *fun_key_p;
-	} else {
-		cw_bpf_debug("[Go] [uprobe/ret]:ErrorErrorErrorError Not get fun_key");
-	}
 	struct trace_stack_entry_key_t trace_key = {};
-	trace_key.goid = e->goid;
-	trace_key.trace_id = trace_id;
-	trace_key.fun_key = fun_key;
+	trace_key.bp = caller_bp % 0x800 + (e->goid << 12);
+	trace_key.caller_bp = e->bp % 0x800 + (e->goid << 12);
+
+	cw_bpf_debug("[Go] [uprobe/ret]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x", trace_key.caller_bp, trace_key.bp);
+	// cw_bpf_debug("[Go] [uprobe/ret]: trace_keytrace_keytrace_key: bp: %x", trace_key.bp);
 
 	struct event *event_p = bpf_map_lookup_elem(&trace_stack_entry, &trace_key);
 
@@ -251,32 +193,12 @@ int ret(struct pt_regs *ctx)
 
 	cw_bpf_debug("[Go] [uprobe/ret]: ent:event: location:%d,ip:%x,time_ns_start:%lld\n", event_p->location, event_p->ip, event_p->time_ns_start);
 	cw_bpf_debug("[Go] [uprobe/ret]: ent:event: bp:%x,caller_bp:%x,caller_ip:%x\n", event_p->bp,event_p->caller_bp,event_p->caller_ip);
-	cw_bpf_debug("[Go] [uprobe/ret]: ent:event: nid:%d,fpid:%d,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+	cw_bpf_debug("[Go] [uprobe/ret]: ent:event: nid:%llx,fpid:%llx,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
 
 	bpf_map_delete_elem(&trace_stack_entry, &trace_key);
-	__u64 fun_key_new = fun_key - 1;
-	bpf_map_update_elem(&trace_stack_entry_index, &trace_index_key, &fun_key_new, BPF_ANY);
 
 	// return bpf_map_push_elem(&event_queue, e, BPF_EXIST);
-	// 最后一个函数结束，同时推送协程信息
-	if (fun_key == 1) {
-		struct event event_coroutine = {};
-		event_coroutine.bp = e->bp,
-		event_coroutine.caller_bp = e->caller_bp,
-		event_coroutine.caller_ip = e->caller_ip,
-		event_coroutine.pid = e->pid,
-		event_coroutine.trace_id = e->trace_id,
-		event_coroutine.goid = e->goid,
-		event_coroutine.ip = e->ip,
-		event_coroutine.time_ns_start = e->time_ns_start,
-		event_coroutine.location = 2,
-		event_coroutine.nid = e->goid,
-		event_coroutine.fpid = 1,
-		event_coroutine.level = 2,
-		cw_bpf_debug("[Go] [uprobe/ret]: ent:event_coroutine push: nid:%d,fpid:%d,level:%d\n", event_coroutine.nid,event_coroutine.fpid,event_coroutine.level);
-		bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, &event_coroutine, sizeof(event_coroutine));
-	}
-	cw_bpf_debug("[Go] [uprobe/ret]: ent:event_ret push: nid:%d,fpid:%d,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
+	cw_bpf_debug("[Go] [uprobe/ret]: ent:event_ret push: nid:%llx,fpid:%llx,level:%d\n", event_p->nid,event_p->fpid,event_p->level);
 	bpf_perf_event_output(ctx, &event_queue, BPF_F_CURRENT_CPU, event_p, sizeof(*e));
 
 	cw_bpf_debug("[Go] [uprobe/ret] end");

pkg/go.opentelemetry.io/otel/exporters/otlp/otlptrace/apm_exporter.go

@@ -8,10 +8,11 @@ import (
 	"strings"
 	"sync"
 
+	"time"
+
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/internal/tracetransform"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
 	tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
-	"time"
 )
 
 const (
@@ -142,6 +143,7 @@ func tracetransformData(sdl []tracesdk.ReadOnlySpan) []RootDataT {
 	sendData := []RootDataT{}
 	for traceId, v := range TraceRootMap {
 		if v.TheEnd {
+			buildLevel(v)
 			sendData = append(sendData, v.RootData)
 			delete(TraceRootMap, traceId)
 			//fmt.Println("the end!")
@@ -158,6 +160,45 @@ func tracetransformData(sdl []tracesdk.ReadOnlySpan) []RootDataT {
 	fmt.Println(len(sdl))
 	return sendData
 }
+func buildLevelRecursion(parentMap map[int]map[int]*MapInfoT, pid int, level int) {
+	for i, v := range parentMap[pid] {
+		parentMap[pid][i].Level = level
+		// fmt.Printf("buildLevelRecursion: %x, %x\n", v.Nid, v.Nid%0x1000000)
+		buildLevelRecursion(parentMap, v.Nid%0x1000000, level+1)
+	}
+}
+
+func buildLevel(sdl *TraceMapT) {
+	parentMap := make(map[int]map[int]*MapInfoT)
+	nidMap := make(map[int]*MapInfoT)
+
+	for i, v := range sdl.RootData.Maps {
+		nidMap[v.Nid%0x1000000] = &sdl.RootData.Maps[i]
+	}
+	// 没有 pid 的放到 application 上
+	for i, v := range sdl.RootData.Maps {
+		_, ok := nidMap[v.Pid]
+		if !ok && v.Pid != 0 {
+			sdl.RootData.Maps[i].Pid = 1
+		}
+	}
+	// fmt.Println("nidMapnidMapnidMapnidMap--------------")
+	// fmt.Println(nidMap)
+	// fmt.Println(sdl.RootData.Maps)
+	// 构建 pid 映射map
+	for i, v := range sdl.RootData.Maps {
+		_, ok := parentMap[v.Pid]
+		if !ok {
+			parentMap[v.Pid] = make(map[int]*MapInfoT)
+		}
+		parentMap[v.Pid][v.Nid] = &sdl.RootData.Maps[i]
+	}
+	// fmt.Println(parentMap)
+	// fmt.Println("nidMapnidMapnidMapnidMap--------------||||")
+	// 构建层级
+	level := 1
+	buildLevelRecursion(parentMap, 0, level)
+}
 
 func initRootData(traceId string) RootDataT {
 	data := RootDataT{
@@ -265,6 +306,7 @@ func buildAppMap(mNode *MapInfoT, traceRoot *TraceMapT, sd apmTraceSpan) {
 	mNode.MethodName = "net/http.(*Transport).roundTrip()"
 	mNode.Level = 1
 	mNode.Pid = 0
+	mNode.Nid = 1
 	// 构建root节点
 	traceRoot.RootData.RespTime = mNode.PureTime
 	traceRoot.RootData.CollTime = mNode.StartTime

run.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+pid=`ps aux | grep ebpfdemo81 | grep -v grep | awk '{print $2}'`
+echo $pid
+TRACES_ENDPOINT=http://10.2.31.73:8099/docp/api/v2/data/receive FILTER_PID=$pid WHITE_LIST=".*HandleFunc|.*main.*|.*serverHandler.*|.*ServeHTTP.*" ./euspace --listen="0.0.0.0:8123"