Fixed #TASK_QT-9810 goroutine优化

ebpftracer/ebpf/ebpf.c

@@ -53,7 +53,7 @@
 //#include "l7/openssl.c"
 #include "utrace/go/net/server.probe.bpf.c"
 #include "utrace/go/net/client.probe.bpf.c"
-#include "utrace/go/net/stack.probe.bpf.c"
+//#include "utrace/go/net/stack.probe.bpf.c"
 
 #include "utrace/java/net/server.probe.bpf.c"
 #include "utrace/java/net/client.probe.bpf.c"

ebpftracer/ebpf/include/apm_trace.h

@@ -55,6 +55,11 @@ struct thread_trace_key_t {
 	__u64 pid_tgid;
 };
 
+struct goid_trace_key_t {
+	__u32 tgid;
+	__u64 goid;
+};
+
 
 //struct apm_span_context {
 //	unsigned char TraceID[APM_TRACE_ID_SIZE];
@@ -83,12 +88,20 @@ struct apm_trace_info_t {
 	//	__u64 thread_trace_id; // 线程追踪ID
 	//	__u64 socket_id; // Records the socket associated when tracing was created (记录创建追踪时关联的socket)
 	__u64 trace_id;
+
 	struct apm_trace_key_t trace_key;
 
 	struct fd_trace_key_t fd_trace_key;
 
 	struct thread_trace_key_t thread_trace_key;
 
+	struct goid_trace_key_t goid_trace_key;
+
+	bool routine_reuse;
+
+	__u8 type;
+
+//	struct goid_trace_key_t goid_trace_key;
 //	struct apm_span_context psc;
 
 //	struct apm_span_context sc;

ebpftracer/ebpf/l7/apm_trace.c

@@ -37,6 +37,20 @@ struct {
 	__uint(max_entries, 32768);
 } thread_trace_info_heap SEC(".maps");
 
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct goid_trace_key_t));
+	__uint(value_size, sizeof(struct apm_trace_info_t));
+	__uint(max_entries, 32768);
+} goid_trace_info_heap SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(key_size, sizeof(struct goid_trace_key_t));
+	__uint(value_size, sizeof(__u64));
+	__uint(max_entries, 32768);
+} goid_trace_info_heap2 SEC(".maps");
+
 struct {
 	__uint(type, BPF_MAP_TYPE_HASH);
 	__uint(key_size, sizeof(__u64));
@@ -71,6 +85,13 @@ struct apm_trace_key_t get_apm_trace_key(__u64 timeout, bool is_socket_io) {
 	return key;
 }
 
+static __inline __attribute__((__always_inline__))
+struct thread_trace_key_t get_thread_trace_key(__u64 id) {
+	struct thread_trace_key_t trace_key = {};
+	trace_key.pid_tgid = id;
+	return trace_key;
+}
+
 static __inline __attribute__((__always_inline__))
 __u64 get_apm_trace_id(__u32 pid, __u32 tid) {
 	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
@@ -83,6 +104,257 @@ __u64 get_apm_trace_id(__u32 pid, __u32 tid) {
 	return 0;
 }
 
+//
+//static __inline __u64 get_trace_id_from_runq_map(__u32 tgid, __u64 goid) {
+//
+//	__u64 ancestor = goid;
+//	int idx = 0;
+//#pragma unroll
+//	for (idx = 0; idx < 6; ++idx) {
+//		bpf_printk("[start][%d] goid------>%llu -----", idx, ancestor);
+//		struct go_key key = {.tgid = tgid, .goid = ancestor};
+//		__u64 *newancestor =
+//				bpf_map_lookup_elem(&go_ancerstor_by_runq_map, &key);
+//		if (!newancestor) {
+//			break;
+//		}
+//		ancestor = *newancestor;
+//		struct apm_trace_key_t trace_key = {.tgid = tgid, .goid = ancestor};
+//		struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key);
+//		if (trace_info) {
+//			bpf_printk("---------get_trace_id_from_runq_map info_trace_id:%llu", trace_info->trace_id);
+//			return trace_info->trace_id;
+//		}
+//		bpf_printk("[end][%d] goid------>%llu -----\n", idx, ancestor);
+//
+//	}
+//
+//	return 0;
+//}
+
+//	    find Parent from go_ancerstor_by_runq_map and find in  go_ancerstor_map == 0
+//	    find by tgid_pid == 0
+//	    find by tgid_goid == 0
+//	    any idea?
+static __inline __attribute__((__always_inline__))
+__u64 get_apm_trace_id_v2(__u64 tgid_pid, __u32 tgid, __u32 pid) {
+	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+	__u64 trace_id = 0;
+	struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key);
+	if (trace_info) {
+		cw_bpf_debug("info_trace_id:%llu", trace_info->trace_id);
+		trace_id = trace_info->trace_id;
+	}
+
+
+	if (trace_id == 0) {
+		__u64 goid = get_current_goroutine();
+//		__u32 tgid = (__u32) (bpf_get_current_pid_tgid() >> 32);
+		struct go_key key = {.tgid = tgid, .goid = goid};
+		__u64 *goid_in_runq_map =
+				bpf_map_lookup_elem(&go_ancerstor_by_runq_map, &key);
+		if (goid_in_runq_map) {
+//			bpf_printk("-----[Kernel Enter][HTTP]:has 1 goid_in_runq_map1 :%llu", *goid_in_runq_map);
+			struct go_key goid_in_runq_map_key = {.tgid = tgid, .goid = *goid_in_runq_map};
+
+//			bpf_printk("-----[Kernel Enter][HTTP]: only one in go_ancerstor_by_runq_map. Now find it from go_ancerstor_map");
+			__u64 *newancestor3 = bpf_map_lookup_elem(&go_ancerstor_map, &goid_in_runq_map_key);
+			if (newancestor3) {
+//				bpf_printk("-----[Kernel Enter][HTTP]:has a newancestor  in go_ancerstor_map :%llu", *newancestor3);
+				struct apm_trace_key_t trace_key2 = {.tgid = tgid, .goid = *newancestor3};
+				trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key2);
+				if (trace_info) {
+					bpf_printk("[get_apm_trace_id_v2] [find goid in go_ancerstor_map] [key:%d-%llu] -->%llu", tgid,
+					           goid, trace_info->trace_id);
+					struct goid_trace_key_t k = {.tgid = tgid, .goid = get_current_goroutine()};
+					if (trace_info->routine_reuse == true) {
+						bpf_printk("[get_apm_trace_id_v2] [routine_reuse already set] [key:%d-%llu] -->%llu", tgid,
+						           goid, trace_info->trace_id);
+					}
+					if (trace_info->routine_reuse == false) {
+						trace_id = trace_info->trace_id;
+						trace_info->routine_reuse = true;
+//						bpf_map_update_elem(&trace_info_heap, &key, trace_info, BPF_EXIST);
+						bpf_printk(
+								"[get_apm_trace_id_v2] [save routine_reuse into trace_info_heap] [key:%d-%llu] -->%llu",
+								tgid, goid, trace_info->trace_id);
+					}
+
+//					trace_id = trace_info->trace_id;
+
+
+
+//					long aaa = bpf_map_update_elem(&goid_trace_info_heap, &k, &trace_id, BPF_ANY);
+//					if (aaa==0){
+//						bpf_printk("save ok");
+//						__u64  *watch_trace_id = bpf_map_lookup_elem(&goid_trace_info_heap, &k);
+//						if (watch_trace_id) {
+//							bpf_printk("watch [key:%d-%llu] -> trace_id:%llu",k.tgid ,k.goid,*watch_trace_id);
+//						}
+//					}
+
+//					 todo 保存 gocache
+//						bpf_printk("ok tid:%d-goid:%d-%llu",key.tgid,key.goid, trace_info2->trace_id);
+//						struct goid_trace_key_t go_key = {.tgid = tgid, .goid = goid};
+//					bpf_printk("111find by goid_trace_key_t tgid:%d-goid:%llu", k.tgid, k.goid);
+//
+//					struct apm_trace_info_t *aa = bpf_map_lookup_elem(&goid_trace_info_heap, &k);
+//					if (aa) {
+//						bpf_printk("111get[%llu]", aa->trace_id);
+//					}
+//					return trace_id;
+				}
+			}
+//			 todo 保存 gocache
+//			trace_id = get_trace_id_from_runq_map(tgid, goid);
+		}
+	}
+
+//	    find by tgid_pid
+//	if (trace_id == 0) {
+//		bpf_printk("find by tgid_pid");
+//		struct thread_trace_key_t t_key = get_thread_trace_key(tgid_pid);
+//		struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&thread_trace_info_heap, &t_key);
+//		if (trace_info) {
+//
+//			// todo 保存 gocache
+////			struct goid_trace_key_t key = {.tgid = tgid, .goid = get_current_goroutine()};
+////			bpf_map_update_elem(&goid_trace_info_heap, &key, trace_info, BPF_ANY);
+//
+//			trace_id = trace_info->trace_id;
+//			bpf_printk("ttttttttttttt trace_id:%llu", trace_info->trace_id);
+//		}
+//	}
+
+	// save
+
+
+//	if (trace_id == 0) {
+//		struct goid_trace_key_t go_key = {.tgid = tgid, .goid = get_current_goroutine()};
+//		bpf_printk("find by goid_trace_key_t tgid:%d-goid:%llu", go_key.tgid, go_key.goid);
+//		struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&goid_trace_info_heap, &go_key);
+//		if (trace_info) {
+//			bpf_printk("get trace_id from goid_trace_key_t pid:[%d]|trace_info:[%llu]", pid, trace_info->trace_id);
+//			trace_id = trace_info->trace_id;
+//			bpf_printk("---------[from] goid_trace_key_t trace_id:%llu", trace_info->trace_id);
+//		}
+//	}
+
+	return trace_id;
+}
+
+static __inline __attribute__((__always_inline__))
+struct apm_trace_info_t *get_apm_trace_info_v2(__u64 tgid_pid, __u32 tgid, __u32 pid) {
+	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+//	__u64 trace_id = 0;
+	struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key);
+	if (trace_info) {
+		bpf_printk("info_trace_id:%llu", trace_info->trace_id);
+//		return trace_info;
+		return trace_info;
+	}
+
+	__u64 goid = get_current_goroutine();
+	struct go_key key = {.tgid = tgid, .goid = goid};
+	__u64 *goid_in_runq_map =
+			bpf_map_lookup_elem(&go_ancerstor_by_runq_map, &key);
+	if (goid_in_runq_map) {
+//			bpf_printk("-----[Kernel Enter][HTTP]:has 1 goid_in_runq_map1 :%llu", *goid_in_runq_map);
+		struct go_key goid_in_runq_map_key = {.tgid = tgid, .goid = *goid_in_runq_map};
+
+//			bpf_printk("-----[Kernel Enter][HTTP]: only one in go_ancerstor_by_runq_map. Now find it from go_ancerstor_map");
+		__u64 *newancestor3 = bpf_map_lookup_elem(&go_ancerstor_map, &goid_in_runq_map_key);
+		if (newancestor3) {
+//				bpf_printk("-----[Kernel Enter][HTTP]:has a newancestor  in go_ancerstor_map :%llu", *newancestor3);
+			struct apm_trace_key_t trace_key2 = {.tgid = tgid, .goid = *newancestor3};
+			trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key2);
+			if (trace_info) {
+				bpf_printk("[get_apm_trace_info_v2] [find goid in go_ancerstor_map] [key:%d-%llu] -->%llu", tgid, goid,
+				           trace_info->trace_id);
+
+				struct goid_trace_key_t k = {.tgid = tgid, .goid = get_current_goroutine()};
+				if (trace_info->routine_reuse == true) {
+					bpf_printk("[get_apm_trace_info_v2] [routine_reuse already set] [key:%d-%llu] -->%llu", tgid, goid,
+					           trace_info->trace_id);
+					return NULL;
+				}
+				if (trace_info->routine_reuse == false) {
+//						trace_id = trace_info->trace_id;
+					trace_info->routine_reuse = true;
+//						bpf_map_update_elem(&trace_info_heap, &key, trace_info, BPF_EXIST);
+					bpf_printk("[sssset routine_reuse true] [key:%d-%llu] -->%llu", tgid, goid, trace_info->trace_id);
+					return trace_info;
+				}
+			}
+		}
+//			 todo 保存 gocache
+//			trace_id = get_trace_id_from_runq_map(tgid, goid);
+	}
+//	}
+
+	return NULL;
+}
+
+static __inline __attribute__((__always_inline__))
+struct apm_trace_info_t *get_apm_trace_info_v3(__u64 tgid_pid, __u32 tgid, __u32 pid) {
+	struct apm_trace_info_t *trace_info = get_apm_trace_info_v2(tgid_pid, tgid, pid);
+
+	struct goid_trace_key_t goid_trace_key = {.tgid = tgid, .goid = get_current_goroutine()};
+
+	if (trace_info != NULL) {
+		trace_info->type = 1;
+//		    trace_info = get_apm_trace_info_v2(id, pid, tid);
+//		trace_id = trace_info->trace_id;
+		bpf_map_update_elem(&goid_trace_info_heap, &goid_trace_key, trace_info, BPF_ANY);
+//		    if (trace_info!=NULL)
+//		    bpf_printk("%llu",trace_info->trace_id);
+	    bpf_printk("[get_apm_trace_info_v3 save] [key:%d-%llu] -> trace_id:%llu",goid_trace_key.tgid,goid_trace_key.goid ,trace_info->trace_id);
+	}
+
+	if (trace_info == NULL) {
+		bpf_printk("[get_apm_trace_info_v3]  find by goid_trace_info_heap2 ");
+		//find
+		struct apm_trace_info_t *goid_trace_info = bpf_map_lookup_elem(&goid_trace_info_heap, &goid_trace_key);
+		if (goid_trace_info != NULL) {
+//		    trace_id = *ttt;
+			bpf_printk(" [get_apm_trace_info_v3 find ooooooooooook] [key:%d-%llu] -> trace_id:%llu", goid_trace_key.tgid,
+			           goid_trace_key.goid, goid_trace_info->trace_id);
+			// 将key保存在trace_info用于后续清除
+
+			goid_trace_info->goid_trace_key = goid_trace_key;
+			trace_info = bpf_map_lookup_elem(&trace_info_heap, &goid_trace_info->trace_key);
+			if (trace_info) {
+				trace_info->goid_trace_key = goid_trace_key;
+				trace_info->type = 2;
+			}
+//		        bpf_printk(" v2v2v2v2v2 [key:%d-%llu] -> trace_id:%llu",.tgid,kkk.goid ,*val2);
+
+
+//				trace_info->goid_trace_key = go_id_key;
+
+//		    bpf_printk(" v2v2v2v2v2 [key:%d-%llu] -> trace_id:%llu",kkk.tgid,kkk.goid ,*val2);
+
+//			    trace_id = *val2;
+//			type = 2;
+//			trace_id = goid_trace_info->trace_id;
+		}
+	}
+
+	if (trace_info == NULL) {
+		bpf_printk("[get_apm_trace_info_v3] find by tgid_pid");
+		struct thread_trace_key_t t_key = get_thread_trace_key(tgid_pid);
+		trace_info = bpf_map_lookup_elem(&thread_trace_info_heap, &t_key);
+		if (trace_info != NULL) {
+			bpf_printk("ttttttttttttt trace_id:%llu", trace_info->trace_id);
+			trace_info->type = 3;
+		}
+	}
+
+	return trace_info;
+
+
+}
+
 static __inline __attribute__((__always_inline__))
 struct apm_trace_info_t *get_apm_trace_info(__u32 pid, __u32 tid) {
 	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
@@ -90,7 +362,7 @@ struct apm_trace_info_t *get_apm_trace_info(__u32 pid, __u32 tid) {
 	if (trace_info) {
 		return trace_info;
 	}
-	return 0;
+	return NULL;
 }
 
 static __inline __attribute__((__always_inline__))
@@ -101,13 +373,6 @@ struct fd_trace_key_t get_fd_trace_key(__u32 pid, __u64 fd) {
 	return trace_key;
 }
 
-static __inline __attribute__((__always_inline__))
-struct thread_trace_key_t get_thread_trace_key(__u64 id) {
-	struct thread_trace_key_t trace_key = {};
-	trace_key.pid_tgid = id;
-	return trace_key;
-}
-
 
 static __inline __attribute__((__always_inline__))
 __u64 get_fd_trace_id(__u32 pid, __u32 fd) {
@@ -192,6 +457,13 @@ __u64 cw_clear_trace(__u32 tgid, __u32 pid, __u32 fd) {
 	struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key);
 	if (trace_info) {
 		bpf_map_delete_elem(&thread_trace_info_heap, &trace_info->thread_trace_key);
+
+//		bpf_map_delete_elem(&trace_info_heap, &trace_info_by_fd->trace_key);
+//		bpf_map_delete_elem(&goid_trace_info_heap, &trace_info_by_fd->goid_trace_key);
+//		bpf_printk("clean  trace_info_heap ok %d",trace_info_by_fd->trace_key.goid);
+
+		bpf_map_delete_elem(&goid_trace_info_heap, &trace_info->goid_trace_key);
+		bpf_printk("clean goid_trace_info_heap ok %d", trace_info->goid_trace_key.goid);
 	}
 	// 清除trace信息
 	bpf_map_delete_elem(&trace_info_heap, &trace_key);
@@ -259,7 +531,7 @@ __u32 cw_add_event_count(__u64 trace_id) {
 static __inline __attribute__((__always_inline__))
 __u32 cw_get_event_count(__u64 trace_id) {
 	__u32 *event_count = bpf_map_lookup_elem(&trace_event_count_heap, &trace_id);
-	if (event_count != NULL){
+	if (event_count != NULL) {
 		return *event_count;
 	}
 	return 0;
@@ -290,6 +562,13 @@ struct apm_trace_info_t cw_save_trace_info(__u64 id, __u32 pid, __u64 fd) {
 	bpf_map_update_elem(&trace_info_heap, &trace_info.trace_key, &trace_info, BPF_NOEXIST);
 	bpf_map_update_elem(&fd_trace_info_heap, &trace_info.fd_trace_key, &trace_info, BPF_NOEXIST);
 	bpf_map_update_elem(&thread_trace_info_heap, &trace_info.thread_trace_key, &trace_info, BPF_NOEXIST);
+
+
+//
+//	struct goid_trace_key_t key = {.tgid = id, .goid = get_current_goroutine()};
+//	bpf_map_update_elem(&goid_trace_info_heap, &key, &trace_info, BPF_NOEXIST);
+
+
 	// 事件个数
 	__u32 event_count = 0;
 	bpf_map_update_elem(&trace_event_count_heap, &trace_info.trace_id, &event_count, BPF_NOEXIST);

ebpftracer/ebpf/l7/l7.c

@@ -140,6 +140,7 @@ struct l7_request {
     __u8 partial;
     __u8 request_type;
     __s32 request_id;
+	__u64 trace_id;
     __u64 payload_size;
     char payload[MAX_PAYLOAD_SIZE];
 };
@@ -294,7 +295,7 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
 //        __u64 trace_id = get_apm_trace_id(pid, fd);
         __u64 trace_id = get_fd_trace_id(pid, fd);
 	    __u32 event_count = cw_get_event_count(trace_id);
-        cw_bpf_debug("[Trace End in l7][Response][HTTP] pid:%d,fd:%d,trace_id:%llu", tid, fd, trace_id);
+        bpf_printk("[Trace End in l7][Response][HTTP] pid:%d,fd:%d,trace_id:%llu", tid, fd, trace_id);
         cw_bpf_debug("[Trace End in l7][Response][HTTP] event_count:%d", event_count);
         // 清除trace信息
         cw_clear_trace(pid, tid, fd);
@@ -312,7 +313,6 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
             return 0;
         }
 	    e->start_at = req->ns;
-	    bpf_printk("req->ns:%llu",req->ns);
 	    e->end_at = bpf_ktime_get_ns();
         e->duration = e->end_at - e->start_at;
 //        cw_bpf_debug("[Response][HTTP]:duration->ns:%d\n",e->duration);
@@ -359,19 +359,161 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
 
 //        cw_bpf_debug("off->task__files_offset:%x", off->task__files_offset);
 //        cw_bpf_debug("e->test_id1111:%d", ttt->test_id);
-        cw_bpf_debug("HTTP_END");
+	    bpf_printk("HTTP_END");
         return 0;
     }
 
     if (is_http_request(payload)) {
-	    cw_bpf_debug("");
-	    cw_bpf_debug("-----[Kernel Enter][HTTP]:pid:[%d]|CURRENT-GOID:[%llu]|FD:[%d]", tid, get_current_goroutine(), k.fd);
+//	    get_apm_trace_id ==0
+//	    find Parent from go_ancerstor_by_runq_map and find in  go_ancerstor_map == 0
+//	    find by tgid_pid == 0
+//	    find by tgid_goid == 0
+//	    any idea?
+
+	    bpf_printk("");
+	    bpf_printk("-----[Kernel Enter][HTTP]:pid:[%d]|CURRENT-GOID:[%llu]|FD:[%d]", tid, get_current_goroutine(), k.fd);
+		int type = 0;
+//	    __u64 trace_id = get_apm_trace_id(pid, tid);
+//	    if (trace_id==0){
+//		    trace_id = get_apm_trace_id_v2(id, pid, tid);
+//			type = 1;
+//	    }
+
+/*info*/
+	    __u64 trace_id = 0;
+
+	    struct apm_trace_info_t * trace_info = get_apm_trace_info(pid, tid);
+
+
+	    if (trace_info== NULL) {
+		    trace_info = get_apm_trace_info_v3(id, pid, tid);
+		}
+
+        if (trace_info) {
+		    bpf_printk("%llu",trace_info->trace_id);
+			trace_id = trace_info->trace_id;
+			type = trace_info->type;
+	    }
 
-	    req->protocol = PROTOCOL_HTTP;
-//        struct apm_trace_key_t trace_key = {0};
-//        trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
-//        cw_bpf_debug("[Enter] [HTTP]:goid:%llu",trace_key.goid);
 
+//	    struct goid_trace_key_t goid_trace_key = {.tgid = pid, .goid = get_current_goroutine()};
+
+	    //  保存数据到 goid_trace_info_heap 用于后续goid关联
+//	    if (trace_info != NULL){
+//		    bpf_printk("save goid_trace_info_heap %llu",trace_info->trace_id);
+//		    type = 1;
+////		    trace_info = get_apm_trace_info_v2(id, pid, tid);
+//		    trace_id = trace_info->trace_id;
+//		    bpf_map_update_elem(&goid_trace_info_heap, &goid_trace_key, trace_info, BPF_ANY);
+////		    if (trace_info!=NULL)
+////		    bpf_printk("%llu",trace_info->trace_id);
+////		    bpf_printk(" v1v1v1v1v1 [key:%d-%llu] -> trace_id:%llu",goid_trace_key.tgid,goid_trace_key.goid ,trace_info->trace_id);
+//	    }
+
+
+//	     从 goid 关联 中获取数据
+//	    if (trace_info == NULL) {
+//		    bpf_printk("aaaaaaaa find by goid_trace_info_heap2 ");
+//		    //find
+//		    struct apm_trace_info_t * goid_trace_info = bpf_map_lookup_elem(&goid_trace_info_heap, &goid_trace_key);
+//		    if (goid_trace_info != NULL) {
+////		    trace_id = *ttt;
+//			    bpf_printk(" v2v2v2v2v2 [key:%d-%llu] -> trace_id:%llu",goid_trace_key.tgid,goid_trace_key.goid ,goid_trace_info->trace_id);
+//			    // 将key保存在trace_info用于后续清除
+//
+//			    goid_trace_info->goid_trace_key = goid_trace_key;
+//			    trace_info = bpf_map_lookup_elem(&trace_info_heap, &goid_trace_info->trace_key);
+//			    if (trace_info != NULL){
+//				    bpf_printk("11111111111");
+//				    trace_info->goid_trace_key = goid_trace_key;
+//				}
+////		        bpf_printk(" v2v2v2v2v2 [key:%d-%llu] -> trace_id:%llu",.tgid,kkk.goid ,*val2);
+//
+//
+////				trace_info->goid_trace_key = go_id_key;
+//
+////		    bpf_printk(" v2v2v2v2v2 [key:%d-%llu] -> trace_id:%llu",kkk.tgid,kkk.goid ,*val2);
+//
+////			    trace_id = *val2;
+//			    type = 2;
+//			    trace_id = goid_trace_info->trace_id;
+//		    }
+//	    }
+		// 最后从线程关联
+//	    if (trace_info== NULL) {
+//		    bpf_printk("find by tgid_pid");
+//		    struct thread_trace_key_t t_key = get_thread_trace_key(id);
+//		    trace_info = bpf_map_lookup_elem(&thread_trace_info_heap, &t_key);
+//		    if (trace_info != NULL) {
+//			    trace_id = trace_info->trace_id;
+//			    bpf_printk("ttttttttttttt trace_id:%llu", trace_info->trace_id);
+//			    type = 3;
+//		    }
+//	    }
+/*info*/
+
+	    /*idididididididididididididididididid
+	    struct goid_trace_key_t kkk = {.tgid = pid, .goid = get_current_goroutine()};
+	    // save 保存数据到goid_trace_info_heap2
+	    if (trace_id!=0){
+		    __u64 trace_id_test= trace_id;
+		    bpf_map_update_elem(&goid_trace_info_heap2, &kkk, &trace_id_test, BPF_ANY);
+		    bpf_printk("save goid [%d-%llu]-[%llu]",kkk.tgid,kkk.goid,trace_id_test);
+//		    struct apm_trace_info_t *ttt =get_apm_trace_info_v2(id, pid, tid);
+//		    ttt.trace_id = trace_id_test;
+//		    if (ttt){
+//			    bpf_map_update_elem(&goid_trace_info_heap, &kkk, ttt, BPF_ANY);
+//			    bpf_printk("save goid [%llu]",ttt->trace_id);
+//		    }
+	    }
+
+        // 从 goid_trace_info_heap2 中获取数据
+	    if (trace_id ==0 ){
+		    bpf_printk("aaaaaaaa find by goid_trace_info_heap2 ");
+		    //find
+		    __u64  *val2 = bpf_map_lookup_elem(&goid_trace_info_heap2, &kkk);
+		    if (val2 != NULL) {
+//		    trace_id = *ttt;
+			    bpf_printk(" v2v2v2v2v2 [key:%d-%llu] -> trace_id:%llu",kkk.tgid,kkk.goid ,*val2);
+//		    bpf_printk(" v2v2v2v2v2 [key:%d-%llu] -> trace_id:%llu",kkk.tgid,kkk.goid ,*val2);
+
+			    trace_id = *val2;
+			    type = 2;
+
+		    }
+	    }
+
+
+	    if (trace_id == 0) {
+		    bpf_printk("find by tgid_pid");
+		    struct thread_trace_key_t t_key = get_thread_trace_key(id);
+		    struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&thread_trace_info_heap, &t_key);
+		    if (trace_info) {
+
+			    // todo 保存 gocache
+//			struct goid_trace_key_t key = {.tgid = tgid, .goid = get_current_goroutine()};
+//			bpf_map_update_elem(&goid_trace_info_heap, &key, trace_info, BPF_ANY);
+
+			    trace_id = trace_info->trace_id;
+			    bpf_printk("ttttttttttttt trace_id:%llu", trace_info->trace_id);
+			    type = 3;
+		    }
+	    }
+	    idididididididididididididididididid*/
+
+//	    bpf_printk("-----[Kernel Enter][HTTP]:trace_id:%llu",trace_id);
+
+
+	    req->protocol = PROTOCOL_HTTP;
+//		req->trace_id = trace_id;
+//	    if (trace_info != NULL) {
+//		    trace_id = trace_info->trace_id;
+//		}
+		req->trace_id = trace_id;
+//	    struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+//	    bpf_printk("[Enter] [HTTP]:trace_key.goid:%llu",trace_key.goid);
+// 0默认，1
+	    bpf_printk("[Kernel Enter][end] req-payload:%s [traceid:%llu][type:%d]",payload,trace_id,type);
 
     } else if (is_postgres_query(payload, size, &req->request_type)) {
         if (req->request_type == POSTGRES_FRAME_CLOSE) {
@@ -584,8 +726,8 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 	    struct apm_trace_info_t trace_info = cw_save_trace_info(id,pid, k.fd);
 //	    __u64 uid_base = bpf_ktime_get_ns();
 //	    trace_info.trace_id = bpf_get_current_pid_tgid() + uid_base;
-	    cw_bpf_debug("\n");
-	    cw_bpf_debug("[Trace Start in l7][Receive][HTTP]pid:[%d]|GOID:[%d]|FD:%d", tid, trace_info.trace_key.goid,k.fd);
+	    bpf_printk("\n");
+	    bpf_printk("[Trace Start in l7][Receive][HTTP]pid:[%d]|GOID:[%d]|trace_id:%llu", tid, trace_info.trace_key.goid,trace_info.trace_id);
 
         e->trace_start = 1;
         e->trace_end = 0;
@@ -656,7 +798,7 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 
     bpf_map_delete_elem(&active_l7_requests, &k);
     if (e->protocol == PROTOCOL_HTTP) {
-        __u64 trace_id = get_apm_trace_id(pid, tid);
+        __u64 trace_id =req->trace_id;
         e->trace_id = trace_id;
 	    struct  apm_span_context * sc = cw_get_current_tracking_span();
 	    if (sc) {
@@ -669,7 +811,7 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 		//        cw_bpf_debug("[Response][HTTP222]:thread_id:%d|type:%s|FD:%d\n",k.pid,"",k.fd);
 //        cw_bpf_debug("[Response][HTTP222] trace_id:%llu", trace_id);
 //        // 请求报文
-//        cw_bpf_debug("[Response][HTTP222] req-payload:%s",e->payload);
+//	    bpf_printk("[Response][HTTP222] req-payload:%s",e->payload);
 //        // 响应报文
 //        cw_bpf_debug("[Response][HTTP222] resp-payload:%s",payload);
 
@@ -679,19 +821,20 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 //		    cw_bpf_debug("[Kernel End] go_id:%llu",goid);
 //
 //		}
-	    if (e->trace_id == 0){
-		    struct thread_trace_key_t t_key = get_thread_trace_key(id);
-		    struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&thread_trace_info_heap, &t_key);
-		    if (trace_info) {
-			    if (trace_info->trace_id == 0){
-				    cw_bpf_debug("get trace_id from enter pid:[%d]|thread_trace_key_t:[%llu]", tid,trace_info->trace_id);
-				    e->trace_id=trace_info->trace_id;
-				}
-		    } else {
-				// 从子事件的头获取线程id
-			}
-	    }
-	    cw_bpf_debug("[Kernel End][HTTP]:pid:[%d]|CURRENT-GOID:[%llu]|trace_id:[%llu]---------\n", tid, get_current_goroutine(),e->trace_id);
+//	    if (e->trace_id == 0){
+//		    struct thread_trace_key_t t_key = get_thread_trace_key(id);
+//		    struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&thread_trace_info_heap, &t_key);
+//		    if (trace_info) {
+//			    if (trace_info->trace_id == 0){
+//				    cw_bpf_debug("get trace_id from enter pid:[%d]|thread_trace_key_t:[%llu]", tid,trace_info->trace_id);
+//				    e->trace_id=trace_info->trace_id;
+//				}
+//		    } else {
+//				// 从子事件的头获取线程id
+//			}
+//	    }
+
+	    bpf_printk("[Kernel End][HTTP]:pid:[%d]|CURRENT-GOID:[%llu]|trace_id:[%llu]---------\n", tid, get_current_goroutine(),e->trace_id);
 
     } else if (e->protocol == PROTOCOL_POSTGRES) {
         response = is_postgres_response(payload, ret, &e->status);
@@ -707,9 +850,9 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
     } else if (e->protocol == PROTOCOL_MEMCACHED) {
         response = is_memcached_response(payload, ret, &e->status);
     } else if (e->protocol == PROTOCOL_MYSQL) {
-        cw_bpf_debug("[Response][Mysql]:thread_id:%d\n",tid);
+//        bpf_printk("[Response][Mysql]:thread_id:%d\n",tid);
         __u64 trace_id = get_apm_trace_id(pid, tid);
-//        cw_bpf_debug("[Mysql] trace_id:%llu", trace_id);
+        bpf_printk("[Mysql] trace_id:%llu goid %llu", trace_id,get_current_goroutine());
         e->trace_id = trace_id;
         response = is_mysql_response(payload, ret, req->request_type, &e->statement_id, &e->status);
         if (req->request_type == MYSQL_COM_STMT_PREPARE) {

ebpftracer/ebpf/uprobe_base_bpf.c

@@ -67,6 +67,19 @@ struct bpf_map_def SEC("maps") go_ancerstor_map = {
 	.max_entries = HASH_ENTRIES_MAX,
 };
 
+
+// The mapping of coroutines to ancestors, the map is updated when a new
+// coroutine is created
+// key : current gorouting (struct go_key)
+// value : ancerstor goid
+struct bpf_map_def SEC("maps") go_ancerstor_by_runq_map = {
+		.type = BPF_MAP_TYPE_LRU_HASH,
+		.key_size = sizeof(struct go_key),
+		.value_size = sizeof(__u64),
+		.max_entries = HASH_ENTRIES_MAX,
+};
+
+
 // Used to determine the timeout, as a termination condition for finding
 // ancestors.
 // key : current gorouting (struct go_key)
@@ -230,6 +243,20 @@ static __inline bool is_final_ancestor(__u32 tgid, __u64 goid, __u64 now,
 	return now < *ts + timeout;
 }
 
+static __inline bool is_final_ancestor2(__u32 tgid, __u64 goid, __u64 now,
+                                      __u64 timeout)
+{
+	struct go_key key = { .tgid = tgid, .goid = goid };
+
+	__u64 *ts = bpf_map_lookup_elem(&go_rw_ts_map, &key);
+	if (!ts) {
+		bpf_printk("no ts");
+		return false;
+	}
+	bpf_printk("%llu-:%llu=*ts + timeout %llu",now, *ts,*ts + timeout);
+	return now < *ts + timeout;
+}
+
 // Try to find an ancestor coroutine that can represent this request.
 // The ancestor coroutine needs to meet two conditions:
 //  1. There have been socket read or write operations in the recent period of time
@@ -458,7 +485,7 @@ int enter_runtime_newproc1(struct pt_regs *ctx)
 	if (!goid) {
 		return 0;
 	}
-	debug("[Go] [runtime.newproc1] goid:%llu",goid);
+	bpf_printk("[Go] [runtime.newproc1] goid:%llu",goid);
 
 	struct go_newproc_caller caller = {
 			.goid = goid,
@@ -517,17 +544,93 @@ int exit_runtime_newproc1(struct pt_regs *ctx)
 		bpf_map_delete_elem(&pid_tgid_callerid_map, &pid_tgid);
 		return 0;
 	}
-	debug("[Go] [runtime.newproc1.exit] current->goid:%llu",goid);
+	bpf_printk("[Go] [runtime.newproc1.exit] current->goid:%llu",goid);
 	// 生成当前协程key
 	struct go_key key = { .tgid = tgid, .goid = goid };
 	goid = caller->goid;
-	debug("[Go] [runtime.newproc1.exit] caller->goid:%llu",goid);	goid = caller->goid;
+	bpf_printk("[Go] [runtime.newproc1.exit] caller->goid:%llu",goid);
+	goid = caller->goid;
 	bpf_map_update_elem(&go_ancerstor_map, &key, &goid, BPF_ANY);
 
 	bpf_map_delete_elem(&pid_tgid_callerid_map, &pid_tgid);
 	return 0;
 }
 
+// [schedule][runqget from inner GetGoroutineID0	[schedule][runqget from inner--------------------]: goidcache=[14]|goidcacheend=[17]|goid=[23]
+// [runtime.newproc1 end]: newg goid=13
+// runqget goid -> goidcache-1
+// runtime/proc.go:5957
+SEC("uprobe/enter_runtime.runqget")
+int enter_runtime_runqget(struct pt_regs *ctx)
+{
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+	__u32 tgid = pid_tgid >> 32;
+
+	int offset_g_goid = 152;
+
+	int offset_p_goidcache = 384;
+
+	int offset_p_runnext = 2456;
+
+	struct ebpf_proc_info *info =
+			bpf_map_lookup_elem(&proc_info_map, &tgid);
+	if (!info) {
+		return 0;
+	}
+
+	if(info->version < GO_VERSION(1, 15, 0)){
+		return 0;
+	}
+	void * sp = (void *)PT_REGS_SP(ctx);
+
+	void *p_ptr;// _p_ *p
+	if (is_register_based_call(info)) {
+		p_ptr = (void *)PT_GO_REGS_PARM1(ctx);
+	} else {
+		if (info->version >= GO_VERSION(1, 18, 0)) {
+			bpf_probe_read(&p_ptr, sizeof(p_ptr), sp + 32);
+		} else {
+			bpf_probe_read(&p_ptr, sizeof(p_ptr), sp + 48);
+		}
+	}
+	__s64 goidcache = 0;
+	bpf_probe_read(&goidcache, sizeof(goidcache), p_ptr + offset_p_goidcache);
+//	__u64 goid = get_current_goroutine();
+
+	void *runnext_ptr;
+	bpf_probe_read(&runnext_ptr, sizeof(runnext_ptr), p_ptr + offset_p_runnext);
+
+	if (runnext_ptr!= 0){
+		__s64 next_goid = 0;
+		bpf_probe_read(&next_goid, sizeof(next_goid), runnext_ptr + offset_g_goid);
+//		bpf_printk("[Go] [runtime.runqget.enter] [next_goid:%llu -----> goidcache:%llu]", next_goid, goidcache);
+		struct go_key key = { .tgid = tgid, .goid = next_goid };
+		// caller id
+		__u64 caller_goid = goidcache - 1;
+		bpf_printk("[Go] [runtime.runqget.enter] [save] [(caller_goid):%llu ----->:%llu(next_goid)]", caller_goid,
+		           next_goid);
+		if (caller_goid == next_goid)
+			return 0;
+		bpf_map_update_elem(&go_ancerstor_by_runq_map, &key, &caller_goid, BPF_ANY);
+	}
+
+
+//	__s64 goidcache = 0;
+//	bpf_probe_read(&goidcache, sizeof(goidcache), runnext_ptr + offset_p_goidcache);
+
+//	bpf_printk("[Go] [runtime.runqget.enter] goidcache:%llu [goid:%llu]",goidcache,);
+
+	return 0;
+
+//	__s64 goid = 0;
+
+
+
+
+//	bpf_map_update_elem(&go_ancerstor_map, &key, &caller_goid, BPF_ANY);
+
+}
+
 // /sys/kernel/debug/tracing/events/sched/sched_process_exit/format
 SEC("tracepoint/sched/sched_process_exit")
 int bpf_func_sched_process_exit(struct sched_comm_exit_ctx *ctx)

ebpftracer/ebpf/utrace/go/net/client.probe.bpf.c

@@ -369,7 +369,7 @@ static __always_inline long cw_inject_header2(void* headers_ptr, struct apm_span
 // func net/http/transport.roundTrip(req *Request) (*Response, error)
 SEC("uprobe/Transport_roundTrip")
 int uprobe_Transport_roundTrip(struct pt_regs *ctx) {
-	cw_bpf_debug("--------[Uprobe HTTP Client] start");
+	bpf_printk("--------[Uprobe HTTP Client] start %llu",get_current_goroutine());
 
     u64 request_pos = 2;
     void *req_ptr = get_argument(ctx, request_pos);
@@ -504,6 +504,7 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) {
 		cw_bpf_debug("uprobe_Transport_roundTrip: Failed to inject header");
 	}
 
+	bpf_printk("pppppppppppppp %s",httpReq->path);
     // Write event
 //    bpf_map_update_elem(&http_events, &key, httpReq, 0);
 //    start_tracking_span(context_ptr_val, &httpReq->sc);
@@ -514,7 +515,7 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) {
 // func net/http/transport.roundTrip(req *Request) (*Response, error)
 SEC("uprobe/Transport_roundTrip")
 int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) {
-	cw_bpf_debug("[Uprobe HTTP Client] start Return----------");
+	bpf_printk("[Uprobe HTTP Client] start Return----------%llu",get_current_goroutine());
 	clear_current_span_context();
     u64 end_time = bpf_ktime_get_ns();
     void *req_ctx_ptr = get_Go_context(ctx, 2, ctx_ptr_pos, false);
@@ -536,7 +537,9 @@ int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) {
     http_req_span->end_time = end_time;
 
 //    bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, http_req_span, sizeof(*http_req_span));
-    stop_tracking_span(&http_req_span->sc, &http_req_span->psc);
+	bpf_printk("goid:%llu path",get_current_goroutine());
+
+	stop_tracking_span(&http_req_span->sc, &http_req_span->psc);
 
     bpf_map_delete_elem(&http_events, &key);
     return 0;

ebpftracer/ebpf/utrace/go/net/stack.probe.bpf.c

@@ -114,7 +114,7 @@ int ent(struct pt_regs *ctx)
 	// trace_key.caller_bp = e->caller_bp;
 	// trace_key.bp = e->bp;
 	// cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: bp: %x", trace_key.bp);
-	bpf_printk("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x, %lld", trace_key.caller_bp, trace_key.bp, trace_key.trace_id);
+	cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x, %lld", trace_key.caller_bp, trace_key.bp, trace_key.trace_id);
 
 	struct event *event_p = bpf_map_lookup_elem(&trace_stack_entry, &trace_key);
 

ebpftracer/tls.go

@@ -28,6 +28,7 @@ const (
 	goTlsReadSymbol       = "crypto/tls.(*Conn).Read"
 	goExecute             = "runtime.execute"
 	goNewproc1            = "runtime.newproc1"
+	goRunqget             = "runtime.runqget"
 	goServeHTTP           = "net/http.serverHandler.ServeHTTP"
 	goTransport           = "net/http.(*Transport).roundTrip"
 )
@@ -190,7 +191,15 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32, insID utils.ID) []link.Link {
 	}
 
 	offset, ok := tracer.GetOffset(tracer.NewID("std", "runtime", "g", "goid"), path)
+	//pOffset, ok2 := tracer.GetOffset(tracer.NewID("std", "runtime", "p", "goidcache"), path)
+	//runnextOffset, ok3 := tracer.GetOffset(tracer.NewID("std", "runtime", "p", "runnext"), path)
+	//if ok3 {
+	//
+	//}
+
 	fmt.Println(offset, ok, version)
+	//fmt.Println(runnextOffset, ok3, version)
+	//os.Exit(1)
 	if ok {
 		realVersion := strings.Replace(bi.GoVersion, "go", "", 1)
 		parts := strings.Split(realVersion, ".")
@@ -243,7 +252,7 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32, insID utils.ID) []link.Link {
 		}
 		switch s.Name {
 		//case goTlsWriteSymbol, goTlsReadSymbol:
-		case goExecute, goNewproc1, goServeHTTP, goTransport:
+		case goExecute, goNewproc1, goRunqget, goServeHTTP, goTransport:
 		default:
 			continue
 		}
@@ -298,6 +307,33 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32, insID utils.ID) []link.Link {
 				}
 				links = append(links, l)
 			}
+
+		case goRunqget:
+			l, err := exe.Uprobe(s.Name, t.uprobes["enter_runtime_runqget"], &link.UprobeOptions{Address: address})
+			if err != nil {
+				log("failed to attach newproc1 uprobe", err)
+				return nil
+			}
+			links = append(links, l)
+			//sStart := s.Value - textSection.Addr
+			//sEnd := sStart + s.Size
+			//if sEnd > textSectionLen {
+			//	continue
+			//}
+			//sBytes := textSectionData[sStart:sEnd]
+			//returnOffsets := getReturnOffsets(ef.Machine, sBytes)
+			//if len(returnOffsets) == 0 {
+			//	log("failed to attach enter_runtime_newproc1 uprobe", fmt.Errorf("no return offsets found"))
+			//	return nil
+			//}
+			//for _, offset := range returnOffsets {
+			//	l, err := exe.Uprobe(s.Name, t.uprobes["exit_runtime_newproc1"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
+			//	if err != nil {
+			//		log("failed to attach exit_runtime_newproc1 uprobe", err)
+			//		return nil
+			//	}
+			//	links = append(links, l)
+			//}
 		case goServeHTTP:
 			l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_HandlerFunc_ServeHTTP"], &link.UprobeOptions{Address: address})
 			if err != nil {

ebpftracer/tracer.go

@@ -696,16 +696,16 @@ func runEventsReader(name string, r *perf.Reader, ch chan<- Event, typ perfMapTy
 				Timestamp: v.Timestamp,
 			}
 		case perfMapTypeEventQueue:
-			v := &StackEvent{}
-			if err := binary.Read(bytes.NewBuffer(rec.RawSample), binary.LittleEndian, v); err != nil {
-				klog.Warningln("failed to read msg:", err)
-				continue
-			}
-
-			event = Event{
-				Type:       EventTypeFunEnt,
-				StackEvent: v,
-			}
+			//v := &StackEvent{}
+			//if err := binary.Read(bytes.NewBuffer(rec.RawSample), binary.LittleEndian, v); err != nil {
+			//	klog.Warningln("failed to read msg:", err)
+			//	continue
+			//}
+			//
+			//event = Event{
+			//	Type:       EventTypeFunEnt,
+			//	StackEvent: v,
+			//}
 		default:
 			continue
 		}

pkg/go.opentelemetry.io/otel/exporters/otlp/otlptrace/apm_exporter.go

@@ -144,8 +144,8 @@ func tracetransformData(sdl []tracesdk.ReadOnlySpan) []RootDataT {
 		buildAppMapFromEvent(&rootData, sd)
 		// 构建maps
 		for _, event := range sd.Events() {
-			aaa, _ := json.Marshal(event)
-			fmt.Println("event.info", string(aaa))
+			//aaa, _ := json.Marshal(event)
+			//fmt.Println("event.info", string(aaa))
 			mNode := buildMapNodeFromEvent(event)
 			switch event.EventType {
 			// stack
@@ -307,8 +307,8 @@ func buildLevelFromEvent(sdl *RootDataT) {
 	Nid := 2
 	level := 2
 
-	for k, v := range mapSlice {
-		fmt.Println("SliceSliceindex", k, "value", v.Time, v.Type, v.Map.MethodName, v.Map.Nid)
+	for _, v := range mapSlice {
+		//fmt.Println("SliceSliceindex", k, "value", v.Time, v.Type, v.Map.MethodName, v.Map.Nid)
 		if v.Type == 0 {
 			// 函数入口
 			funStack = append(funStack, v)
@@ -481,7 +481,7 @@ func buildMapNodeFromEvent(event tracesdk.Event) MapInfoT {
 	//mNode.StartTime = spanSd.StartTimeUnixNano
 	//mNode.EndTime = spanSd.EndTimeUnixNano
 	for _, attr := range event.Attributes {
-		fmt.Println(event.Name, "--->buildMapNodeFromEvent--->", attr.Key, ":", attr.Value.AsInterface())
+		//fmt.Println(event.Name, "--->buildMapNodeFromEvent--->", attr.Key, ":", attr.Value.AsInterface())
 		switch attr.Key {
 		case "nid":
 			mNode.Nid = int(attr.Value.AsInt64())
@@ -609,7 +609,7 @@ func buildAppMapFromEvent(traceRoot *RootDataT, sd apmTraceSpan) {
 	//traceRoot.RespTime = mNode.PureTimex
 	//traceRoot.CollTime = mNode.StartTime
 	for _, attr := range sd.Attributes() {
-		fmt.Println("Appmap:", attr.Key, ":", attr.Value.AsInterface())
+		//fmt.Println("Appmap:", attr.Key, ":", attr.Value.AsInterface())
 		switch attr.Key {
 		case "http.uri":
 			traceRoot.Uri = attr.Value.AsString()
@@ -683,7 +683,7 @@ func buildHttpMapFromEvent(mNode *MapInfoT, event tracesdk.Event) {
 	mNode.MethodName = "net/http.serverHandler.ServeHTTP()"
 	//var descAddr string
 	for _, attr := range event.Attributes {
-		fmt.Println("HTTP--->", attr.Key, ":", attr.Value.AsInterface())
+		//fmt.Println("HTTP--->", attr.Key, ":", attr.Value.AsInterface())
 		switch attr.Key {
 		case "http.ip":
 			mNode.Ip = attr.Value.AsString()