Fixed #TASK_QT-9810 goroutine优化(dev)

containers/container.go

@@ -1096,6 +1096,7 @@ func (c *Container) revalidateListens(now time.Time, actualListens map[netaddr.I
 }
 
 func (c *Container) attachUprobes(tracer *ebpftracer.Tracer, pid uint32) error {
+	klog.Infoln("[attach] attachUprobes start")
 	if tracer.DisableL7Tracing() {
 		return nil
 	}

containers/registry.go

@@ -201,7 +201,7 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 								klog.WithField("pid", pid).Infoln("[registry] Attach uprobes success!")
 							}
 							klog.WithField("pid", pid).Infoln("[registry] Attach app stack.")
-							err = c.stackTrace(r.tracer, pid)
+							err = c.StackTrace(r.tracer, pid)
 							if err != nil {
 								klog.WithField("pid", pid).WithError(err).Errorf("[registry][end] Failed attach stack trace!")
 							} else {
@@ -362,7 +362,7 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 							continue
 						}
 						c.attachUprobes(r.tracer, e.Pid)
-						err = c.stackTrace(r.tracer, e.Pid)
+						err = c.StackTrace(r.tracer, e.Pid)
 						if err != nil {
 							klog.Errorf("Stack trace error", err)
 						}

ebpftracer/ebpf/common/common.h

@@ -1,4 +1,5 @@
 #include "load_const.h"
+
 #define __uint(name, val) int (*name)[val]
 #define __type(name, val) typeof(val) *name
 #define __array(name, val) typeof(val) *name[]
@@ -7,7 +8,7 @@
 //    char ____fmt[] = fmt;                                      \
 //    bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
 //})
-// #ifdef _DEBUG_MODE
+#ifdef _DEBUG_MODE
 #define cw_bpf_debug(fmt, ...)                                 \
 ({                                                             \
     char ____fmt[] = fmt;                                      \
@@ -18,22 +19,22 @@
     char ____fmt[] = fmt;                                      \
     bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
 })
-// #else
-// #define cw_bpf_debug(fmt, ...)                                  \
-// ({                                                              \
-//                                                                 \
-// })
-// #define debug(fmt, ...)                                         \
-// ({                                                              \
-//                                                                 \
-// })
-// #endif
+#else
+#define cw_bpf_debug(fmt, ...)                                  \
+ ({                                                              \
+																 \
+ })
+#define debug(fmt, ...)                                         \
+ ({                                                              \
+																 \
+ })
+#endif
 
 
 /* Helper macro to print out debug messages */
-#define bpf_printk(fmt, ...)				\
-({							\
-	char ____fmt[] = fmt;				\
-	bpf_trace_printk(____fmt, sizeof(____fmt),	\
-			 ##__VA_ARGS__);		\
+#define bpf_printk(fmt, ...)                \
+({                            \
+    char ____fmt[] = fmt;                \
+    bpf_trace_printk(____fmt, sizeof(____fmt),    \
+             ##__VA_ARGS__);        \
 })

ebpftracer/ebpf/include/apm_trace.h

@@ -55,6 +55,11 @@ struct thread_trace_key_t {
 	__u64 pid_tgid;
 };
 
+struct goid_trace_key_t {
+	__u32 tgid;
+	__u64 goid;
+};
+
 
 //struct apm_span_context {
 //	unsigned char TraceID[APM_TRACE_ID_SIZE];
@@ -71,34 +76,33 @@ struct apm_span_context {
 	unsigned char assumed_app_id[APM_ASSUMED_APP_ID_SIZE];
 	unsigned char span_id[APM_SPAN_ID_SIZE];
 };
-
-struct apm_trace_info_t {
-	/*
+/*
 	 * Whether traceID is zero ?
 	 * For the client to actively send request, set traceID to zero.
 	 */
-	//	bool is_trace_id_zero;
-	//	__u32 update_time; // 从系统开机开始到创建/更新时的间隔时间单位是秒
-	//	__u32 peer_fd;	   // 用于socket之间的关联
-	//	__u64 thread_trace_id; // 线程追踪ID
-	//	__u64 socket_id; // Records the socket associated when tracing was created (记录创建追踪时关联的socket)
+//	bool is_trace_id_zero;
+//	__u32 update_time; // 从系统开机开始到创建/更新时的间隔时间单位是秒
+//	__u32 peer_fd;	   // 用于socket之间的关联
+//	__u64 thread_trace_id; // 线程追踪ID
+//	__u64 socket_id; // Records the socket associated when tracing was created (记录创建追踪时关联的socket)
+struct apm_trace_info_t {
 	__u64 trace_id;
+	__u8 type;
+	bool routine_reuse;
 	struct apm_trace_key_t trace_key;
-
 	struct fd_trace_key_t fd_trace_key;
-
 	struct thread_trace_key_t thread_trace_key;
+	struct goid_trace_key_t goid_trace_key;
 
 //	struct apm_span_context psc;
 
 //	struct apm_span_context sc;
 };
 
-static __always_inline void cw_copy_byte_arrays(unsigned char *src, unsigned char *dst, __u32 size)
-{
-	for (int i = 0; i < size; i++)
-	{
+static __always_inline void cw_copy_byte_arrays(unsigned char *src, unsigned char *dst, __u32 size) {
+	for (int i = 0; i < size; i++) {
 		dst[i] = src[i];
 	}
 }
+
 #endif //EUSPACES_APM_TRACE_H

ebpftracer/ebpf/l7/apm_trace.c

@@ -37,6 +37,13 @@ struct {
 	__uint(max_entries, 32768);
 } thread_trace_info_heap SEC(".maps");
 
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct goid_trace_key_t));
+	__uint(value_size, sizeof(struct apm_trace_info_t));
+	__uint(max_entries, 32768);
+} goid_trace_info_heap SEC(".maps");
+
 struct {
 	__uint(type, BPF_MAP_TYPE_HASH);
 	__uint(key_size, sizeof(__u64));
@@ -97,6 +104,13 @@ struct apm_trace_key_t get_apm_trace_key(__u64 timeout, bool is_socket_io) {
 	return key;
 }
 
+static __inline __attribute__((__always_inline__))
+struct thread_trace_key_t get_thread_trace_key(__u64 id) {
+	struct thread_trace_key_t trace_key = {};
+	trace_key.pid_tgid = id;
+	return trace_key;
+}
+
 static __inline __attribute__((__always_inline__))
 __u64 get_apm_trace_id(__u32 pid, __u32 tid) {
 	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
@@ -109,6 +123,112 @@ __u64 get_apm_trace_id(__u32 pid, __u32 tid) {
 	return 0;
 }
 
+//	find Parent from go_ancerstor_by_runq_map and find in  go_ancerstor_map == 0
+//	find by tgid_pid == 0
+//	find by tgid_goid == 0
+static __inline __attribute__((__always_inline__))
+struct apm_trace_info_t *get_apm_trace_info_v2(__u64 tgid_pid, __u32 tgid, __u32 pid) {
+	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+	struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key);
+	if (trace_info) {
+//		bpf_printk("info_trace_id:%llu", trace_info->trace_id);
+		return trace_info;
+	}
+
+	__u64 goid = get_current_goroutine();
+	struct go_key key = {.tgid = tgid, .goid = goid};
+
+	//	find Parent from go_ancerstor_by_runq_map and find in  go_ancerstor_map == 0
+	__u64 *runq_map_goid = bpf_map_lookup_elem(&go_ancerstor_by_runq_map, &key);
+	if (runq_map_goid) {
+//			bpf_printk("-----[Kernel Enter][HTTP]:has 1 runq_map_goid1 :%llu", *runq_map_goid);
+		struct go_key runq_map_goid_key = {.tgid = tgid, .goid = *runq_map_goid};
+//			bpf_printk("-----[Kernel Enter][HTTP]: only one in go_ancerstor_by_runq_map. Now find it from go_ancerstor_map");
+		//	find Parent from go_ancerstor_map
+		__u64 *ancestor_goid = bpf_map_lookup_elem(&go_ancerstor_map, &runq_map_goid_key);
+		if (ancestor_goid) {
+//				bpf_printk("-----[Kernel Enter][HTTP]:has a newancestor  in go_ancerstor_map :%llu", *newancestor3);
+			struct apm_trace_key_t trace_key2 = {.tgid = tgid, .goid = *ancestor_goid};
+			trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key2);
+			if (trace_info) {
+//				bpf_printk("[get_apm_trace_info_v2] [find goid in go_ancerstor_map] [key:%d-%llu] -->%llu", tgid, goid,trace_info->trace_id);
+
+				struct goid_trace_key_t k = {.tgid = tgid, .goid = get_current_goroutine()};
+				// 祖先协程被复用
+				if (trace_info->routine_reuse == true) {
+//					bpf_printk("[get_apm_trace_info_v2] [routine_reuse already set] [key:%d-%llu] -->%llu", tgid, goid,	trace_info->trace_id);
+					return NULL;
+				}
+				if (trace_info->routine_reuse == false) {
+//						trace_id = trace_info->trace_id;
+					trace_info->routine_reuse = true;
+//						bpf_map_update_elem(&trace_info_heap, &key, trace_info, BPF_EXIST);
+//					bpf_printk("[sssset routine_reuse true] [key:%d-%llu] -->%llu", tgid, goid, trace_info->trace_id);
+					return trace_info;
+				}
+			}
+		}
+//			 todo 保存 gocache
+//			trace_id = get_trace_id_from_runq_map(tgid, goid);
+	}
+//	}
+
+	return NULL;
+}
+
+static __inline __attribute__((__always_inline__))
+struct apm_trace_info_t *get_apm_trace_info_v3(__u64 tgid_pid, __u32 tgid, __u32 pid) {
+	struct apm_trace_info_t *trace_info = get_apm_trace_info_v2(tgid_pid, tgid, pid);
+
+	struct goid_trace_key_t key_goid = {.tgid = tgid, .goid = get_current_goroutine()};
+
+	if (trace_info != NULL) {
+		trace_info->type = 1;
+//		    trace_info = get_apm_trace_info_v2(id, pid, tid);
+//		trace_id = trace_info->trace_id;
+		trace_info->goid_trace_key = key_goid;
+		// [save] [goid_trace_key_t => trace_info]  (goid_trace_info_heap)]
+		// 保存goid -> trace数据
+		bpf_map_update_elem(&goid_trace_info_heap, &key_goid, trace_info, BPF_ANY);
+//		    if (trace_info!=NULL)
+//		    bpf_printk("%llu",trace_info->trace_id);
+//	    bpf_printk("[get_apm_trace_info_v3 save] [key:%d-%llu] -> trace_id:%llu",key_goid.tgid,key_goid.goid ,trace_info->trace_id);
+	}
+
+	if (trace_info == NULL) {
+//		bpf_printk("[get_apm_trace_info_v3]  find by goid_trace_info_heap2 ");
+		// Second or multiple times
+		// [find] [goid_trace_key => tmp_trace_info] = [apm_trace_info_t (goid_trace_info_heap)]
+		struct apm_trace_info_t *tmp_goid_trace_info = bpf_map_lookup_elem(&goid_trace_info_heap, &key_goid);
+		if (tmp_goid_trace_info != NULL) {
+//		    trace_id = *ttt;
+//			bpf_printk(" [get_apm_trace_info_v3 find ooooooooooook] [key:%d-%llu] -> trace_id:%llu", key_goid.tgid,key_goid.goid, tmp_goid_trace_info->trace_id);
+			// todo do not use
+			tmp_goid_trace_info->goid_trace_key = key_goid;
+
+			// [find] [tmp_trace_info.apm_trace_key => trace_info] = [apm_trace_info_t (trace_info_heap)]
+			trace_info = bpf_map_lookup_elem(&trace_info_heap, &tmp_goid_trace_info->trace_key);
+			if (trace_info) {
+				// 将key保存在trace_info用于后续清除
+				trace_info->goid_trace_key = key_goid;
+				// Second or multiple times
+				trace_info->type = 2;
+			}
+		}
+	}
+
+	if (trace_info == NULL) {
+//		bpf_printk("[get_apm_trace_info_v3] find by tgid_pid");
+		struct thread_trace_key_t t_key = get_thread_trace_key(tgid_pid);
+		trace_info = bpf_map_lookup_elem(&thread_trace_info_heap, &t_key);
+		if (trace_info != NULL) {
+//			bpf_printk("ttttttttttttt trace_id:%llu", trace_info->trace_id);
+			trace_info->type = 3;
+		}
+	}
+	return trace_info;
+}
+
 static __inline __attribute__((__always_inline__))
 struct apm_trace_info_t *get_apm_trace_info(__u32 pid, __u32 tid) {
 	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
@@ -116,7 +236,7 @@ struct apm_trace_info_t *get_apm_trace_info(__u32 pid, __u32 tid) {
 	if (trace_info) {
 		return trace_info;
 	}
-	return 0;
+	return NULL;
 }
 
 static __inline __attribute__((__always_inline__))
@@ -127,14 +247,6 @@ struct fd_trace_key_t get_fd_trace_key(__u32 pid, __u64 fd) {
 	return trace_key;
 }
 
-static __inline __attribute__((__always_inline__))
-struct thread_trace_key_t get_thread_trace_key(__u64 id) {
-	struct thread_trace_key_t trace_key = {};
-	trace_key.pid_tgid = id;
-	return trace_key;
-}
-
-
 static __inline __attribute__((__always_inline__))
 __u64 get_fd_trace_id(__u32 pid, __u32 fd) {
 	struct fd_trace_key_t trace_key = get_fd_trace_key(pid, fd);
@@ -214,10 +326,13 @@ static __inline __attribute__((__always_inline__))
 __u64 cw_clear_trace(__u32 tgid, __u32 pid, __u32 fd) {
 	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
 	struct fd_trace_key_t fd_trace_key = get_fd_trace_key(tgid, fd);
-	// 清除trace信息
+	// 清除 trace_info_heap
 	struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key);
 	if (trace_info) {
 		bpf_map_delete_elem(&thread_trace_info_heap, &trace_info->thread_trace_key);
+		// 清除 goid_trace_info_heap
+		bpf_map_delete_elem(&goid_trace_info_heap, &trace_info->goid_trace_key);
+//		bpf_printk("clean goid_trace_info_heap ok %d", trace_info->goid_trace_key.goid);
 	}
 	// 清除trace信息
 	bpf_map_delete_elem(&trace_info_heap, &trace_key);
@@ -230,17 +345,27 @@ __u64 cw_clear_trace(__u32 tgid, __u32 pid, __u32 fd) {
 static __inline __attribute__((__always_inline__))
 void cw_save_current_tracking_span(struct apm_span_context *sc) {
 	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+//	bpf_printk("[save cw sc]%d",trace_key.goid);
 	bpf_map_update_elem(&apm_current_span_context_map, &trace_key, sc, BPF_ANY);
 }
 
 
 static __inline __attribute__((__always_inline__))
-struct apm_span_context *cw_get_current_tracking_span() {
+struct apm_span_context *cw_get_current_tracking_span(struct apm_trace_info_t *trace_info) {
 	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+//	bpf_printk("[get cw sc]%d goid:%d",trace_key.goid,get_current_goroutine());
 	struct apm_span_context *apm_sc = {0};
 	struct apm_span_context *span_contexts = bpf_map_lookup_elem(&apm_current_span_context_map, &trace_key);
 //	bpf_printk("-------");
 
+	if (span_contexts == NULL && trace_info) {
+		if (trace_info->routine_reuse == true) {
+//			bpf_printk("[get cw sc bbb] :%llu", trace_info->trace_key.goid);
+			struct apm_trace_key_t ancestor_trace_key = trace_info->trace_key;
+			span_contexts = bpf_map_lookup_elem(&apm_current_span_context_map, &ancestor_trace_key);
+		}
+	}
+
 	if (span_contexts) {
 		/*for (int i = 0; i < APM_TYPE_FROM_SIZE; i++) {
 			bpf_printk("cw_get_current_tracking_span-type_from[%d] = %02x", i, span_contexts->type_from[i]);
@@ -286,7 +411,7 @@ __u32 cw_add_event_count(__u64 trace_id) {
 static __inline __attribute__((__always_inline__))
 __u32 cw_get_event_count(__u64 trace_id) {
 	__u32 *event_count = bpf_map_lookup_elem(&trace_event_count_heap, &trace_id);
-	if (event_count != NULL){
+	if (event_count != NULL) {
 		return *event_count;
 	}
 	return 0;
@@ -317,6 +442,12 @@ struct apm_trace_info_t cw_save_trace_info(__u64 id, __u32 pid, __u64 fd) {
 	bpf_map_update_elem(&trace_info_heap, &trace_info.trace_key, &trace_info, BPF_NOEXIST);
 	bpf_map_update_elem(&fd_trace_info_heap, &trace_info.fd_trace_key, &trace_info, BPF_NOEXIST);
 	bpf_map_update_elem(&thread_trace_info_heap, &trace_info.thread_trace_key, &trace_info, BPF_NOEXIST);
+
+
+//	struct goid_trace_key_t key = {.tgid = id, .goid = get_current_goroutine()};
+//	bpf_map_update_elem(&goid_trace_info_heap, &key, &trace_info, BPF_NOEXIST);
+
+
 	// 事件个数
 	__u32 event_count = 0;
 	bpf_map_update_elem(&trace_event_count_heap, &trace_info.trace_id, &event_count, BPF_NOEXIST);

ebpftracer/ebpf/l7/l7.c

@@ -142,6 +142,9 @@ struct l7_request {
     __u8 partial;
     __u8 request_type;
     __s32 request_id;
+	__u64 trace_id;
+	unsigned char assumed_app_id[APM_ASSUMED_APP_ID_SIZE];
+	unsigned char span_id[APM_SPAN_ID_SIZE];
     __u64 payload_size;
     char payload[MAX_PAYLOAD_SIZE];
 };
@@ -234,7 +237,7 @@ void send_event(void *ctx, struct l7_event *e, __u32 pid, __u64 fd) {
     __u64 *timestamp = bpf_map_lookup_elem(&connection_timestamps, &sk);
     if (timestamp) {
         if (*timestamp == 0) {
-	        bpf_printk("timestamp=0");
+//	        cw_bpf_debug("timestamp=0");
             return;
         }
         e->connection_timestamp = *timestamp;
@@ -398,7 +401,7 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
 		    cw_copy_byte_arrays(cw_psc->app_id, e->app_id_from, APM_APP_ID_SIZE);
 		    cw_copy_byte_arrays(cw_psc->span_id, e->span_id_from, APM_SPAN_ID_SIZE);
 //		    for (int i = 0; i < APM_TRACE_ID_SIZE; i++) {
-//			    bpf_printk("trace_enter_write - span_id_from = %02x", e->span_id_from[i]);
+//			    cw_bpf_debug("trace_enter_write - span_id_from = %02x", e->span_id_from[i]);
 //		    }
 	    }
         struct l7_request *req = bpf_map_lookup_elem(&active_l7_requests, &k);
@@ -445,19 +448,41 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
 
 //        cw_bpf_debug("off->task__files_offset:%x", off->task__files_offset);
 //        cw_bpf_debug("e->test_id1111:%d", ttt->test_id);
-        cw_bpf_debug("HTTP_END");
+	    cw_bpf_debug("HTTP_END");
         return 0;
     }
 
     if (is_http_request(payload)) {
 	    cw_bpf_debug("");
-	    cw_bpf_debug("-----[Kernel Enter][HTTP]:pid:[%d]|CURRENT-GOID:[%llu]|FD:[%d]", tid, get_current_goroutine(), k.fd);
+	    cw_bpf_debug("-----[Kernel HTTP Enter]:pid:[%d]|CURRENT-GOID:[%llu]|FD:[%d]", tid, get_current_goroutine(), k.fd);
+	    __u8 type =  0;
+	    __u64 trace_id = 0;
+	    struct apm_trace_info_t * trace_info = get_apm_trace_info(pid, tid);
 
-	    req->protocol = PROTOCOL_HTTP;
-//        struct apm_trace_key_t trace_key = {0};
-//        trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
-//        cw_bpf_debug("[Enter] [HTTP]:goid:%llu",trace_key.goid);
+	    if (trace_info== NULL) {
+		    trace_info = get_apm_trace_info_v3(id, pid, tid);
+		}
 
+        if (trace_info) {
+//		    cw_bpf_debug("%llu",trace_info->trace_id);
+			trace_id = trace_info->trace_id;
+			type = trace_info->type;
+	    }
+	    req->protocol = PROTOCOL_HTTP;
+		req->trace_id = trace_id;
+	    struct  apm_span_context * sc = cw_get_current_tracking_span(trace_info);
+	    if (sc) {
+		    cw_copy_byte_arrays(sc->assumed_app_id, req->assumed_app_id, APM_ASSUMED_APP_ID_SIZE);
+		    cw_copy_byte_arrays(sc->span_id, req->span_id, APM_SPAN_ID_SIZE);
+//		    for (int i = 0; i < APM_ASSUMED_APP_ID_SIZE; i++) {
+//			    cw_bpf_debug("assumed_app_id-assumed_app_id[%d] = %02x", i, req->assumed_app_id[i]);
+//		    }
+//		    for (int i = 0; i < APM_SPAN_ID_SIZE; i++) {
+//			    cw_bpf_debug("cw_get_current_tracking_span-span_id[%d] = %02x", i, req->span_id[i]);
+//		    }
+		}
+		// 0默认，1
+	    cw_bpf_debug("[Kernel HTTP Enter]req-payload:%s [traceid:%llu][type:%d]",payload,trace_id,type);
 
     } else if (is_postgres_query(payload, size, &req->request_type)) {
         if (req->request_type == POSTGRES_FRAME_CLOSE) {
@@ -498,7 +523,7 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
         }
         req->protocol = PROTOCOL_MYSQL;
     } else if (is_dm_query(payload, size,&req->request_type)) {
-//        bpf_printk("[Request][DM] start -------->");
+//        cw_bpf_debug("[Request][DM] start -------->");
         req->protocol = PROTOCOL_DM;
 
         struct l7_request_dm_ctx *dm_ctx ;
@@ -511,7 +536,7 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
            dm_ctx->req_start_at_ns = 0 ;
            dm_ctx->status = 0;
            bpf_map_update_elem(&active_l7_requests_dm_ctx, &k, dm_ctx, BPF_NOEXIST);
-//           bpf_printk("[Request][DM]  init active_l7_requests_dm_ctx,request_type <0x%x> [%d]",req->request_type,req->request_type);
+//           cw_bpf_debug("[Request][DM]  init active_l7_requests_dm_ctx,request_type <0x%x> [%d]",req->request_type,req->request_type);
        }
 
        if (req->request_type == DM_QUERY) {
@@ -528,8 +553,8 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
             bpf_map_update_elem(&active_l7_requests, &k, req, BPF_ANY);
             return 0 ;
         }
-//        bpf_printk("[Request][DM] is request ,request_type <0x%x> [%d]",req->request_type,req->request_type);
-//        bpf_printk("[Request][DM] end <--------");
+//        cw_bpf_debug("[Request][DM] is request ,request_type <0x%x> [%d]",req->request_type,req->request_type);
+//        cw_bpf_debug("[Request][DM] end <--------");
     } else if (is_mongo_query(payload, size)) {
         req->protocol = PROTOCOL_MONGO;
     } else if (is_rabbitmq_produce(payload, size)) {
@@ -657,7 +682,6 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
     e->payload_size = 0;
 	__u8 b[8];
 	bpf_read(payload, b);
-	bpf_printk("payload44444:[0x%x]",b[4]);
 //    __u32 k0 = 0;
 //    struct member_fields_offset *offset = members_offset__lookup(&k0);
 //    if (!offset)
@@ -745,9 +769,9 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
     struct l7_request *req = bpf_map_lookup_elem(&active_l7_requests, &k);
     int response = 0;
     if (!req) {
-	    bpf_printk("no req? 6:[0x%x] k.pid:%d, k.fd:%d",b[4],k.pid,k.fd);
+//	    cw_bpf_debug("no req? 6:[0x%x] k.pid:%d, k.fd:%d",b[4],k.pid,k.fd);
         if (is_dns_response(payload, ret, &k.stream_id, &e->status)) {
-	        bpf_printk("dns");
+//	        cw_bpf_debug("dns");
             req = bpf_map_lookup_elem(&active_l7_requests, &k);
             if (!req) {
                 return 0;
@@ -775,7 +799,7 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
             send_event(ctx, e, k.pid, k.fd);
             return 0;
         } else {
-	        bpf_printk("bb 6:[0x%x] k.pid:%d, k.fd:%d",b[4],k.pid,k.fd);
+//	        cw_bpf_debug("bb 6:[0x%x] k.pid:%d, k.fd:%d",b[4],k.pid,k.fd);
             return 0;
         }
     }
@@ -785,90 +809,76 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
     COPY_PAYLOAD(e->payload, req->payload_size, req->payload);
 
     bpf_map_delete_elem(&active_l7_requests, &k);
-	bpf_printk("delete req--------:[0x%x] k.pid:%d, k.fd:%d",b[4],k.pid,k.fd);
-    if (e->protocol == PROTOCOL_HTTP) {
-        __u64 trace_id = get_apm_trace_id(pid, tid);
-        e->trace_id = trace_id;
-	    struct  apm_span_context * sc = cw_get_current_tracking_span();
-	    if (sc) {
-		    cw_copy_byte_arrays(sc->assumed_app_id, e->assumed_app_id, APM_ASSUMED_APP_ID_SIZE);
-		    cw_copy_byte_arrays(sc->span_id, e->span_id, APM_SPAN_ID_SIZE);
+//	cw_bpf_debug("delete req--------:[0x%x] k.pid:%d, k.fd:%d",b[4],k.pid,k.fd);
+	if (e->protocol == PROTOCOL_HTTP) {
+		__u64 trace_id = req->trace_id;
+		e->trace_id = trace_id;
+//	    struct  apm_span_context * sc = cw_get_current_tracking_span();
+//	    if (sc) {
+		cw_copy_byte_arrays(req->assumed_app_id, e->assumed_app_id, APM_ASSUMED_APP_ID_SIZE);
+		cw_copy_byte_arrays(req->span_id, e->span_id, APM_SPAN_ID_SIZE);
 //		    for (int i = 0; i < APM_ASSUMED_APP_ID_SIZE; i++) {
-//			    bpf_printk("assumed_app_id-assumed_app_id[%d] = %02x", i, sc->assumed_app_id[i]);
+//			    cw_bpf_debug("assumed_app_id-assumed_app_id[%d] = %02x", i, req->assumed_app_id[i]);
 //		    }
-	    }
+//	    for (int i = 0; i < APM_SPAN_ID_SIZE; i++) {
+//			    cw_bpf_debug("cw_get_current_tracking_span-span_id[%d] = %02x", i, req->span_id[i]);
+//		    }
+//	    }
 		//        cw_bpf_debug("[Response][HTTP222]:thread_id:%d|type:%s|FD:%d\n",k.pid,"",k.fd);
 //        cw_bpf_debug("[Response][HTTP222] trace_id:%llu", trace_id);
 //        // 请求报文
-//        cw_bpf_debug("[Response][HTTP222] req-payload:%s",e->payload);
+//	    cw_bpf_debug("[Response][HTTP222] req-payload:%s",e->payload);
 //        // 响应报文
 //        cw_bpf_debug("[Response][HTTP222] resp-payload:%s",payload);
 
-        response = is_http_response(payload, &e->status);
-//	    if (trace_id == 0) {
-//		    __u64 goid = get_current_goroutine();
-//		    cw_bpf_debug("[Kernel End] go_id:%llu",goid);
-//
-//		}
-	    if (e->trace_id == 0){
-		    struct thread_trace_key_t t_key = get_thread_trace_key(id);
-		    struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&thread_trace_info_heap, &t_key);
-		    if (trace_info) {
-			    if (trace_info->trace_id == 0){
-				    cw_bpf_debug("get trace_id from enter pid:[%d]|thread_trace_key_t:[%llu]", tid,trace_info->trace_id);
-				    e->trace_id=trace_info->trace_id;
-				}
-		    } else {
-				// 从子事件的头获取线程id
-			}
-	    }
-	    cw_bpf_debug("[Kernel End][HTTP]:pid:[%d]|CURRENT-GOID:[%llu]|trace_id:[%llu]---------\n", tid, get_current_goroutine(),e->trace_id);
+		response = is_http_response(payload, &e->status);
+//	    cw_bpf_debug("[Kernel End][HTTP]:pid:[%d]|CURRENT-GOID:[%llu]|trace_id:[%llu]---------\n", tid, get_current_goroutine(),e->trace_id);
 
-    } else if (e->protocol == PROTOCOL_POSTGRES) {
-        response = is_postgres_response(payload, ret, &e->status);
-        if (req->request_type == POSTGRES_FRAME_PARSE) {
-            e->method = METHOD_STATEMENT_PREPARE;
-        }
-    } else if (e->protocol == PROTOCOL_REDIS) {
-        cw_bpf_debug("[Response][Redis]:TGID:%d|type:%s|FD:%d\n",k.pid,"",k.fd);
-        __u64 trace_id = get_apm_trace_id(pid, tid);
-        cw_bpf_debug("[Redis] trace_id:%llu", trace_id);
-        e->trace_id = trace_id;
-        response = is_redis_response(payload, ret, &e->status);
-    } else if (e->protocol == PROTOCOL_MEMCACHED) {
-        response = is_memcached_response(payload, ret, &e->status);
-    } else if (e->protocol == PROTOCOL_MYSQL) {
-	    bpf_printk("[Response][Mysql]:thread_id:%d\n",tid);
-        __u64 trace_id = get_apm_trace_id(pid, tid);
+	} else if (e->protocol == PROTOCOL_POSTGRES) {
+		response = is_postgres_response(payload, ret, &e->status);
+		if (req->request_type == POSTGRES_FRAME_PARSE) {
+			e->method = METHOD_STATEMENT_PREPARE;
+		}
+	} else if (e->protocol == PROTOCOL_REDIS) {
+		cw_bpf_debug("[Response][Redis]:TGID:%d|type:%s|FD:%d\n", k.pid, "", k.fd);
+		__u64 trace_id = get_apm_trace_id(pid, tid);
+		cw_bpf_debug("[Redis] trace_id:%llu", trace_id);
+		e->trace_id = trace_id;
+		response = is_redis_response(payload, ret, &e->status);
+	} else if (e->protocol == PROTOCOL_MEMCACHED) {
+		response = is_memcached_response(payload, ret, &e->status);
+	} else if (e->protocol == PROTOCOL_MYSQL) {
+		cw_bpf_debug("[Response][Mysql]:thread_id:%d\n", tid);
+		__u64 trace_id = get_apm_trace_id(pid, tid);
 //        cw_bpf_debug("[Mysql] trace_id:%llu", trace_id);
-        e->trace_id = trace_id;
-        response = is_mysql_response(payload, ret, req->request_type, &e->statement_id, &e->status);
-        if (req->request_type == MYSQL_COM_STMT_PREPARE) {
-            e->method = METHOD_STATEMENT_PREPARE;
-        }
-    } else if (e->protocol == PROTOCOL_DM) {
+		e->trace_id = trace_id;
+		response = is_mysql_response(payload, ret, req->request_type, &e->statement_id, &e->status);
+		if (req->request_type == MYSQL_COM_STMT_PREPARE) {
+			e->method = METHOD_STATEMENT_PREPARE;
+		}
+	} else if (e->protocol == PROTOCOL_DM) {
 
-//        bpf_printk("[Response][DM] start -------->");
+//        cw_bpf_debug("[Response][DM] start -------->");
 
-        struct l7_request_dm_ctx *dm_ctx = bpf_map_lookup_elem(&active_l7_requests_dm_ctx, &k);
-        if (!dm_ctx) {
-            return 0;
-        }
-        //获取server端的字符集类型（用来编码数据库名称，暂时不用）
+		struct l7_request_dm_ctx *dm_ctx = bpf_map_lookup_elem(&active_l7_requests_dm_ctx, &k);
+		if (!dm_ctx) {
+			return 0;
+		}
+		//获取server端的字符集类型（用来编码数据库名称，暂时不用）
 //        if(req->request_type == DM_VERSION_CLI) {
 //            __u8 srv_encode_buf[4];
 //            bpf_read(payload+DM_SVR_ENCODING_OFFSET, srv_encode_buf);
 //
-//            bpf_printk(" srv_encode_buf:[0x%x]",srv_encode_buf[0]);
-//            bpf_printk(" srv_encode_buf:[0x%x]",srv_encode_buf[1]);
-//            bpf_printk(" srv_encode_buf:[0x%x]",srv_encode_buf[2]);
-//            bpf_printk(" srv_encode_buf:[0x%x]",srv_encode_buf[3]);
+//            cw_bpf_debug(" srv_encode_buf:[0x%x]",srv_encode_buf[0]);
+//            cw_bpf_debug(" srv_encode_buf:[0x%x]",srv_encode_buf[1]);
+//            cw_bpf_debug(" srv_encode_buf:[0x%x]",srv_encode_buf[2]);
+//            cw_bpf_debug(" srv_encode_buf:[0x%x]",srv_encode_buf[3]);
 //            dm_ctx->srv_encoding = (__s32)(srv_encode_buf[0]& 0xff) | ((__s32)srv_encode_buf[1]&0xff) << 8 | ((__s32)srv_encode_buf[2]&0xff) << 16 | ((__s32)srv_encode_buf[3]&0xff) << 24 ;
-//            bpf_printk(" srv_encode_buf:[%d]",dm_ctx->srv_encoding);
+//            cw_bpf_debug(" srv_encode_buf:[%d]",dm_ctx->srv_encoding);
 //            bpf_map_update_elem(&active_l7_requests_dm_ctx, &k, dm_ctx, BPF_ANY);
 //           return 0;
 //        }
-            //获取数据库名称（暂时不用）
+		//获取数据库名称（暂时不用）
 //        if(req->request_type == DM_DO_LOGIN ) {
 ////            //todo 拷贝payload 不进行处理，留给用户空间处理
 ////            //pack_type == DM_LOGIN_ACK
@@ -886,50 +896,50 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 //            __s32 tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//            bpf_printk("=====    DM_LOGIN_ACK ----1 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====    DM_LOGIN_ACK ----1 tmp_len:%d",tmp_len);
 //
 //            //2
 //            bpf_read(payload+tmp_offset,tmp_1);
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//              bpf_printk("=====    DM_LOGIN_ACK ----2 tmp_len:%d",tmp_len);
+//              cw_bpf_debug("=====    DM_LOGIN_ACK ----2 tmp_len:%d",tmp_len);
 //            //3
 //            bpf_read(payload+tmp_offset,tmp_1);
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//            bpf_printk("=====    DM_LOGIN_ACK ----3 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====    DM_LOGIN_ACK ----3 tmp_len:%d",tmp_len);
 //            //4
 //            bpf_read(payload+tmp_offset,tmp_1);
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//            bpf_printk("=====   DM_LOGIN_ACK ----4 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====   DM_LOGIN_ACK ----4 tmp_len:%d",tmp_len);
 //            //5
 //            bpf_read(payload+tmp_offset,tmp_1);
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//            bpf_printk("=====    DM_LOGIN_ACK ----5 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====    DM_LOGIN_ACK ----5 tmp_len:%d",tmp_len);
 //            //6
 //            bpf_read(payload+tmp_offset,tmp_1);
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//            bpf_printk("=====   DM_LOGIN_ACK ----6 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====   DM_LOGIN_ACK ----6 tmp_len:%d",tmp_len);
 //            //7
 //            bpf_read(payload+tmp_offset,tmp_1);
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//            bpf_printk("=====  DM_LOGIN_ACK ----7 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====  DM_LOGIN_ACK ----7 tmp_len:%d",tmp_len);
 //            //8
 //            bpf_read(payload+tmp_offset,tmp_1);
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//            bpf_printk("=====  DM_LOGIN_ACK ----8 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====  DM_LOGIN_ACK ----8 tmp_len:%d",tmp_len);
 //
 //            //9
 //            tmp_offset = tmp_offset + 4 ;
@@ -942,14 +952,14 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_offset = tmp_offset+tmp_len ;
-//            bpf_printk("=====    DM_LOGIN_ACK ----12 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====    DM_LOGIN_ACK ----12 tmp_len:%d",tmp_len);
 //            //13
 //            bpf_read(payload+tmp_offset,tmp_1);
 //            tmp_offset = tmp_offset + 4 ;
 //            tmp_len = (__s32)(tmp_1[0]& 0xff) | ((__s32)tmp_1[1]&0xff) << 8 | ((__s32)tmp_1[2]&0xff) << 16 | ((__s32)tmp_1[3]&0xff) << 24 ;
-//            bpf_printk("=====    DM_LOGIN_ACK ----13 tmp_len:%d",tmp_len);
+//            cw_bpf_debug("=====    DM_LOGIN_ACK ----13 tmp_len:%d",tmp_len);
 //
-//            bpf_printk("=====   DM_LOGIN_ACK  dbName tmp_offset:%d",tmp_offset);
+//            cw_bpf_debug("=====   DM_LOGIN_ACK  dbName tmp_offset:%d",tmp_offset);
 //            if(tmp_len>0){
 ////                  char dbNameBuff[7] ;
 //                  char dbNameBuff[128] ;
@@ -958,62 +968,62 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 //                  }
 //                  bpf_read(payload+tmp_offset,dbNameBuff);
 //                  dbNameBuff[tmp_len-1]='\0';
-////                      bpf_printk("=====  DM-resp  DM_LOGIN_ACK dbName[0]:%c",dbNameBuff[0]);
-////                      bpf_printk("=====  DM-resp  DM_LOGIN_ACK dbName[1]:%c",dbNameBuff[1]);
-////                      bpf_printk("=====  DM-resp  DM_LOGIN_ACK dbName[2]:%c",dbNameBuff[2]);
-////                      bpf_printk("=====  DM-resp  DM_LOGIN_ACK dbName[3]:%c",dbNameBuff[3]);
-////                      bpf_printk("=====  DM-resp  DM_LOGIN_ACK dbName[4]:%c",dbNameBuff[4]);
-////                      bpf_printk("=====  DM-resp  DM_LOGIN_ACK dbName[5]:%c",dbNameBuff[5]);
-//                  bpf_printk("===== DM_LOGIN_ACK dbName:%s",dbNameBuff);
+////                      cw_bpf_debug("=====  DM-resp  DM_LOGIN_ACK dbName[0]:%c",dbNameBuff[0]);
+////                      cw_bpf_debug("=====  DM-resp  DM_LOGIN_ACK dbName[1]:%c",dbNameBuff[1]);
+////                      cw_bpf_debug("=====  DM-resp  DM_LOGIN_ACK dbName[2]:%c",dbNameBuff[2]);
+////                      cw_bpf_debug("=====  DM-resp  DM_LOGIN_ACK dbName[3]:%c",dbNameBuff[3]);
+////                      cw_bpf_debug("=====  DM-resp  DM_LOGIN_ACK dbName[4]:%c",dbNameBuff[4]);
+////                      cw_bpf_debug("=====  DM-resp  DM_LOGIN_ACK dbName[5]:%c",dbNameBuff[5]);
+//                  cw_bpf_debug("===== DM_LOGIN_ACK dbName:%s",dbNameBuff);
 //
 //                __u32 dn_name_size = (__u32)tmp_len + 1 ;
 //                COPY_PAYLOAD(dm_ctx->db_name,dn_name_size, dbNameBuff);
-//                bpf_printk("===== DM_LOGIN_ACK strcpy(db_name,dbNameBuff):%s",dm_ctx->db_name);
+//                cw_bpf_debug("===== DM_LOGIN_ACK strcpy(db_name,dbNameBuff):%s",dm_ctx->db_name);
 //                bpf_map_update_elem(&active_l7_requests_dm_ctx, &k, dm_ctx, BPF_ANY);
 //            }
 //          return 0;
 //        }
 
-	    __u64 trace_id = get_apm_trace_id(pid, tid);
-	    e->trace_id = trace_id;
-//	    bpf_printk("[Response][DM] trace_id:%llu", trace_id);
+		__u64 trace_id = get_apm_trace_id(pid, tid);
+		e->trace_id = trace_id;
+//	    cw_bpf_debug("[Response][DM] trace_id:%llu", trace_id);
 
-	    response = is_dm_response(payload, ret, req->request_type, &dm_ctx->status);
-//	    bpf_printk("[Response][DM]  is_dm_response status ---------- %d",dm_ctx->status);
+		response = is_dm_response(payload, ret, req->request_type, &dm_ctx->status);
+//	    cw_bpf_debug("[Response][DM]  is_dm_response status ---------- %d",dm_ctx->status);
 
-	    if(response) {
-	         req->ns = dm_ctx->req_start_at_ns;
-	         e->status = dm_ctx->status;
-	         e->payload_size = dm_ctx->query_sql_payload_size;
-	         COPY_PAYLOAD(e->payload, dm_ctx->query_sql_payload_size, dm_ctx->query_sql_payload);
-             bpf_map_delete_elem(&active_l7_requests_dm_ctx, &k);
-//             bpf_printk("[Response][DM]  is response ,delete  active_l7_requests_dm_ctx -- req->request_type<0x%x> , e->payload_size:[%d]",req->request_type,e->payload_size);
-        }
+		if (response) {
+			req->ns = dm_ctx->req_start_at_ns;
+			e->status = dm_ctx->status;
+			e->payload_size = dm_ctx->query_sql_payload_size;
+			COPY_PAYLOAD(e->payload, dm_ctx->query_sql_payload_size, dm_ctx->query_sql_payload);
+			bpf_map_delete_elem(&active_l7_requests_dm_ctx, &k);
+//             cw_bpf_debug("[Response][DM]  is response ,delete  active_l7_requests_dm_ctx -- req->request_type<0x%x> , e->payload_size:[%d]",req->request_type,e->payload_size);
+		}
 //        else {
-//             bpf_printk("[Response][DM]  not response req->request_type <0x%x>",req->request_type);
+//             cw_bpf_debug("[Response][DM]  not response req->request_type <0x%x>",req->request_type);
 //        }
 
-//        bpf_printk("[Response][DM] end <---------\n");
-    } else if (e->protocol == PROTOCOL_MONGO) {
-        response = is_mongo_response(payload, ret, req->partial);
-        if (response == 2) { // partial
-            struct l7_request *r = bpf_map_lookup_elem(&l7_request_heap, &zero);
-            if (!r) {
-                return 0;
-            }
-            r->partial = 1;
-            r->protocol = e->protocol;
-            r->ns = req->ns;
-            r->payload_size = req->payload_size;
-            COPY_PAYLOAD(r->payload, req->payload_size, req->payload);
-            bpf_map_update_elem(&active_l7_requests, &k, r, BPF_ANY);
-            return 0;
-        }
-    } else if (e->protocol == PROTOCOL_KAFKA) {
-        response = is_kafka_response(payload, req->request_id);
-    } else if (e->protocol == PROTOCOL_DUBBO2) {
-        response = is_dubbo2_response(payload, &e->status);
-    }
+//        cw_bpf_debug("[Response][DM] end <---------\n");
+	} else if (e->protocol == PROTOCOL_MONGO) {
+		response = is_mongo_response(payload, ret, req->partial);
+		if (response == 2) { // partial
+			struct l7_request *r = bpf_map_lookup_elem(&l7_request_heap, &zero);
+			if (!r) {
+				return 0;
+			}
+			r->partial = 1;
+			r->protocol = e->protocol;
+			r->ns = req->ns;
+			r->payload_size = req->payload_size;
+			COPY_PAYLOAD(r->payload, req->payload_size, req->payload);
+			bpf_map_update_elem(&active_l7_requests, &k, r, BPF_ANY);
+			return 0;
+		}
+	} else if (e->protocol == PROTOCOL_KAFKA) {
+		response = is_kafka_response(payload, req->request_id);
+	} else if (e->protocol == PROTOCOL_DUBBO2) {
+		response = is_dubbo2_response(payload, &e->status);
+	}
 
     if (!response) {
         return 0;

ebpftracer/ebpf/uprobe_base_bpf.c

@@ -67,6 +67,19 @@ struct bpf_map_def SEC("maps") go_ancerstor_map = {
 	.max_entries = HASH_ENTRIES_MAX,
 };
 
+
+// The mapping of coroutines to ancestors, the map is updated when a new
+// coroutine is created
+// key : current gorouting (struct go_key)
+// value : ancerstor goid
+struct bpf_map_def SEC("maps") go_ancerstor_by_runq_map = {
+		.type = BPF_MAP_TYPE_LRU_HASH,
+		.key_size = sizeof(struct go_key),
+		.value_size = sizeof(__u64),
+		.max_entries = HASH_ENTRIES_MAX,
+};
+
+
 // Used to determine the timeout, as a termination condition for finding
 // ancestors.
 // key : current gorouting (struct go_key)
@@ -230,6 +243,20 @@ static __inline bool is_final_ancestor(__u32 tgid, __u64 goid, __u64 now,
 	return now < *ts + timeout;
 }
 
+static __inline bool is_final_ancestor2(__u32 tgid, __u64 goid, __u64 now,
+                                      __u64 timeout)
+{
+	struct go_key key = { .tgid = tgid, .goid = goid };
+
+	__u64 *ts = bpf_map_lookup_elem(&go_rw_ts_map, &key);
+	if (!ts) {
+		bpf_printk("no ts");
+		return false;
+	}
+	bpf_printk("%llu-:%llu=*ts + timeout %llu",now, *ts,*ts + timeout);
+	return now < *ts + timeout;
+}
+
 // Try to find an ancestor coroutine that can represent this request.
 // The ancestor coroutine needs to meet two conditions:
 //  1. There have been socket read or write operations in the recent period of time
@@ -536,6 +563,81 @@ int exit_runtime_newproc1(struct pt_regs *ctx)
 	return 0;
 }
 
+// [schedule][runqget from inner GetGoroutineID0	[schedule][runqget from inner--------------------]: goidcache=[14]|goidcacheend=[17]|goid=[23]
+// [runtime.newproc1 end]: newg goid=13
+// runqget goid -> goidcache-1
+// runtime/proc.go:5957
+SEC("uprobe/enter_runtime.runqget")
+int enter_runtime_runqget(struct pt_regs *ctx)
+{
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+	__u32 tgid = pid_tgid >> 32;
+
+	int offset_g_goid = 152;
+
+	int offset_p_goidcache = 384;
+
+	int offset_p_runnext = 2456;
+
+	struct ebpf_proc_info *info =
+			bpf_map_lookup_elem(&proc_info_map, &tgid);
+	if (!info) {
+		return 0;
+	}
+
+	if(info->version < GO_VERSION(1, 15, 0)){
+		return 0;
+	}
+	void * sp = (void *)PT_REGS_SP(ctx);
+
+	void *p_ptr;// _p_ *p
+	if (is_register_based_call(info)) {
+		p_ptr = (void *)PT_GO_REGS_PARM1(ctx);
+	} else {
+		if (info->version >= GO_VERSION(1, 18, 0)) {
+			bpf_probe_read(&p_ptr, sizeof(p_ptr), sp + 32);
+		} else {
+			bpf_probe_read(&p_ptr, sizeof(p_ptr), sp + 48);
+		}
+	}
+	__s64 goidcache = 0;
+	bpf_probe_read(&goidcache, sizeof(goidcache), p_ptr + offset_p_goidcache);
+//	__u64 goid = get_current_goroutine();
+
+	void *runnext_ptr;
+	bpf_probe_read(&runnext_ptr, sizeof(runnext_ptr), p_ptr + offset_p_runnext);
+
+	if (runnext_ptr!= 0){
+		__s64 next_goid = 0;
+		bpf_probe_read(&next_goid, sizeof(next_goid), runnext_ptr + offset_g_goid);
+//		bpf_printk("[Go] [runtime.runqget.enter] [next_goid:%llu -----> goidcache:%llu]", next_goid, goidcache);
+		struct go_key key = { .tgid = tgid, .goid = next_goid };
+		// caller id
+		__u64 caller_goid = goidcache - 1;
+		cw_bpf_debug("[Go] [runtime.runqget.enter] [save] [(caller_goid):%llu ----->:%llu(next_goid)]", caller_goid,
+		           next_goid);
+		if (caller_goid == next_goid)
+			return 0;
+		bpf_map_update_elem(&go_ancerstor_by_runq_map, &key, &caller_goid, BPF_ANY);
+	}
+
+
+//	__s64 goidcache = 0;
+//	bpf_probe_read(&goidcache, sizeof(goidcache), runnext_ptr + offset_p_goidcache);
+
+//	bpf_printk("[Go] [runtime.runqget.enter] goidcache:%llu [goid:%llu]",goidcache,);
+
+	return 0;
+
+//	__s64 goid = 0;
+
+
+
+
+//	bpf_map_update_elem(&go_ancerstor_map, &key, &caller_goid, BPF_ANY);
+
+}
+
 // /sys/kernel/debug/tracing/events/sched/sched_process_exit/format
 SEC("tracepoint/sched/sched_process_exit")
 int bpf_func_sched_process_exit(struct sched_comm_exit_ctx *ctx)

ebpftracer/ebpf/utrace/go/net/client.probe.bpf.c

@@ -541,7 +541,7 @@ static __always_inline long cw_inject_header_half(void* headers_ptr, char * head
 // func net/http/transport.roundTrip(req *Request) (*Response, error)
 SEC("uprobe/Transport_roundTrip")
 int uprobe_Transport_roundTrip(struct pt_regs *ctx) {
-	cw_bpf_debug("--------[Uprobe HTTP Client] start");
+	cw_bpf_debug("--------[Uprobe HTTP Client] start %llu",get_current_goroutine());
 
     u64 request_pos = 2;
     void *req_ptr = get_argument(ctx, request_pos);
@@ -565,7 +565,7 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) {
     void *httpReq_ptr = bpf_map_lookup_elem(&http_events, &key);
     if (httpReq_ptr != NULL)
     {
-        cw_bpf_debug("uprobe/Transport_RoundTrip already tracked with the current context");
+	    cw_bpf_debug("uprobe/Transport_RoundTrip already tracked with the current context");
         return 0;
     }
     // return 0;
@@ -686,14 +686,15 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) {
 		cw_bpf_debug("uprobe_Transport_roundTrip: Failed to inject header");
 		return -1;
 	}
+// Write event
+	bpf_map_update_elem(&http_events, &key, httpReq, 0);
+	start_tracking_span(context_ptr_val, &httpReq->sc);
 
 	bpf_tail_call(ctx, &NAME(progs_jmp_up_map), PROG_DATA_GO_UPDATE_HEADER_UP_IDX);
 
 //	bpf_tail_call(ctx, &jmp_table2, 0);
 
-    // Write event
-//    bpf_map_update_elem(&http_events, &key, httpReq, 0);
-//    start_tracking_span(context_ptr_val, &httpReq->sc);
+
 
     return 0;
 }
@@ -814,7 +815,7 @@ PROGUP(go_update_header)(struct pt_regs *ctx) {
 // func net/http/transport.roundTrip(req *Request) (*Response, error)
 SEC("uprobe/Transport_roundTrip")
 int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) {
-	cw_bpf_debug("[Uprobe HTTP Client] start Return----------");
+	cw_bpf_debug("[Uprobe HTTP Client] start Return----------%llu",get_current_goroutine());
 	clear_current_span_context();
     u64 end_time = bpf_ktime_get_ns();
 
@@ -846,7 +847,9 @@ int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) {
     http_req_span->end_time = end_time;
 
 //    bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, http_req_span, sizeof(*http_req_span));
-    stop_tracking_span(&http_req_span->sc, &http_req_span->psc);
+	cw_bpf_debug("goid:%llu path",get_current_goroutine());
+
+	stop_tracking_span(&http_req_span->sc, &http_req_span->psc);
 
     bpf_map_delete_elem(&http_events, &key);
     return 0;

ebpftracer/tls.go

@@ -27,6 +27,7 @@ const (
 	goTlsReadSymbol       = "crypto/tls.(*Conn).Read"
 	goExecute             = "runtime.execute"
 	goNewproc1            = "runtime.newproc1"
+	goRunqget             = "runtime.runqget"
 	goServeHTTP           = "net/http.serverHandler.ServeHTTP"
 	goTransport           = "net/http.(*Transport).roundTrip"
 )
@@ -190,6 +191,7 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32, appInfo *AppInfo, codeType uint1
 	}
 
 	offset, ok := tracer.GetOffset(tracer.NewID("std", "runtime", "g", "goid"), path)
+
 	fmt.Println(offset, ok, version)
 	if ok {
 		realVersion := strings.Replace(bi.GoVersion, "go", "", 1)
@@ -260,7 +262,7 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32, appInfo *AppInfo, codeType uint1
 		}
 		switch s.Name {
 		//case goTlsWriteSymbol, goTlsReadSymbol:
-		case goExecute, goNewproc1, goServeHTTP, goTransport:
+		case goExecute, goNewproc1, goRunqget, goServeHTTP, goTransport:
 		default:
 			continue
 		}
@@ -315,6 +317,33 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32, appInfo *AppInfo, codeType uint1
 				}
 				links = append(links, l)
 			}
+
+		case goRunqget:
+			l, err := exe.Uprobe(s.Name, t.uprobes["enter_runtime_runqget"], &link.UprobeOptions{Address: address})
+			if err != nil {
+				log("failed to attach goRunqget uprobe", err)
+				return nil, err
+			}
+			links = append(links, l)
+			//sStart := s.Value - textSection.Addr
+			//sEnd := sStart + s.Size
+			//if sEnd > textSectionLen {
+			//	continue
+			//}
+			//sBytes := textSectionData[sStart:sEnd]
+			//returnOffsets := getReturnOffsets(ef.Machine, sBytes)
+			//if len(returnOffsets) == 0 {
+			//	log("failed to attach enter_runtime_newproc1 uprobe", fmt.Errorf("no return offsets found"))
+			//	return nil
+			//}
+			//for _, offset := range returnOffsets {
+			//	l, err := exe.Uprobe(s.Name, t.uprobes["exit_runtime_newproc1"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
+			//	if err != nil {
+			//		log("failed to attach exit_runtime_newproc1 uprobe", err)
+			//		return nil
+			//	}
+			//	links = append(links, l)
+			//}
 		case goServeHTTP:
 			l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_HandlerFunc_ServeHTTP"], &link.UprobeOptions{Address: address})
 			if err != nil {

ebpftracer/tracer/filter.go

@@ -7,8 +7,8 @@ import (
 )
 
 func PidFilter(collectionSpec *ebpf.CollectionSpec) {
-	filterPid, ok := common.GetFilterPid()
-	if ok {
+	filterPid, isZero := common.GetFilterPid()
+	if !isZero {
 		type Editor struct {
 			instructions     *asm.Instructions
 			ReferenceOffsets map[string][]int