Feature #TASK_QT-18250 header2

ebpftracer/ebpf/include/apm_trace.h

@@ -46,6 +46,12 @@ struct apm_trace_key_t {
 	__u64 connectid;
 };
 
+struct apm_sk_msg_key_t {
+	__u32 tgid;
+	__u32 remote_ip4 ;
+	__u16 remote_port;
+};
+
 struct fd_trace_key_t {
 	__u32 tgid;
 	__u32 fd;
@@ -108,5 +114,6 @@ static __inline __u32 has_cw_header(const char *data);
 static __always_inline void copy_byte_arrays(unsigned char *src, unsigned char *dst, __u32 size);
 static __always_inline void generate_random_bytes(unsigned char *buff, __u32 size);
 static __always_inline void hex_string_to_bytes(char *str, __u32 size, unsigned char *out);
+static __always_inline void span_context_to_cw_string_stream(struct apm_span_context *ctx, char *buff, char type_from);
 
 #endif //EUSPACES_APM_TRACE_H

ebpftracer/ebpf/l7/apm_trace.c

@@ -16,6 +16,13 @@ struct {
 	__uint(max_entries, 32768);
 } apm_current_span_context_map SEC(".maps");
 
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct apm_sk_msg_key_t));
+	__uint(value_size, sizeof(struct apm_span_context));
+	__uint(max_entries, 32768);
+} apm_current_span_context_by_ipport_map SEC(".maps");
+
 struct {
 	__uint(type, BPF_MAP_TYPE_LRU_HASH);
 	__uint(key_size, sizeof(struct apm_trace_key_t));
@@ -474,6 +481,46 @@ struct apm_span_context *cw_get_current_tracking_span(struct apm_trace_info_t *t
 	return apm_sc;
 }
 
+
+static __inline __attribute__((__always_inline__))
+void cw_save_current_span_context_by_ipport(__u32 remote_ip4, __u16 remote_port, struct apm_span_context *sc) {
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+	struct apm_sk_msg_key_t sk_msg_key = {};
+	sk_msg_key.tgid = (__u32) (pid_tgid >> 32);
+	sk_msg_key.remote_ip4 = remote_ip4;
+	sk_msg_key.remote_port = remote_port;
+	bpf_map_update_elem(&apm_current_span_context_by_ipport_map, &sk_msg_key, sc, BPF_ANY);
+}
+
+static __inline __attribute__((__always_inline__))
+struct apm_span_context *cw_get_current_span_context_by_ipport(__u32 remote_ip4, __u16 remote_port) {
+	struct apm_span_context *apm_sc = {0};
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+	struct apm_sk_msg_key_t sk_msg_key = {};
+	sk_msg_key.tgid = (__u32) (pid_tgid >> 32);
+	sk_msg_key.remote_ip4 = remote_ip4;
+	sk_msg_key.remote_port = remote_port;
+//	bpf_printk("sk_msg_key.remote_ip4 [%llu]",sk_msg_key.remote_ip4);
+//	bpf_printk("sk_msg_key.remote_port [%llu]",sk_msg_key.remote_port);
+	struct apm_span_context *span_contexts = bpf_map_lookup_elem(&apm_current_span_context_by_ipport_map, &sk_msg_key);
+	if (span_contexts) {
+		apm_sc = span_contexts;
+//		for (int i = 0; i < APM_ASSUMED_APP_ID_SIZE; i++) {
+//			bpf_printk("cw_get_current_tracking_span-assumed_app_id[%d] = %02x", i, span_contexts->assumed_app_id[i]);
+//		}
+	}
+	return apm_sc;
+}
+
+static __inline __attribute__((__always_inline__))
+void cw_clean_current_span_context_by_ipport(__u32 remote_ip4, __u16 remote_port, __u32 tgid) {
+	struct apm_sk_msg_key_t sk_msg_key = {};
+	sk_msg_key.tgid = tgid;
+	sk_msg_key.remote_ip4 = remote_ip4;
+	sk_msg_key.remote_port = remote_port;
+	bpf_map_delete_elem(&goid_trace_info_heap, &sk_msg_key);
+}
+
 static __inline __attribute__((__always_inline__))
 __u32 cw_add_event_count(__u64 trace_id) {
 	__u32 *event_count = bpf_map_lookup_elem(&trace_event_count_heap, &trace_id);
@@ -610,7 +657,7 @@ static __inline struct apm_span_context * build_sc_by_ipport(struct ebpf_proc_in
 	generate_random_bytes(cw_sc->span_id, APM_SPAN_ID_SIZE);
 
 	// set host_id/appid
-	__u32 k0 = 0;
+	/*__u32 k0 = 0;
 	struct trace_conf_t *trace_conf = trace_conf_map__lookup(&k0);
 	if (trace_conf) {
 //		for (int i = 0; i < 8; ++i) {
@@ -618,7 +665,7 @@ static __inline struct apm_span_context * build_sc_by_ipport(struct ebpf_proc_in
 //		}
 		copy_byte_arrays(trace_conf->host_id, cw_sc->host_id, APM_HOST_ID_SIZE);
 //		copy_byte_arrays(trace_conf->app_id, cw_sc->app_id, APM_APP_ID_SIZE);
-	}
+	}*/
 
 	copy_byte_arrays(proc_info.instance_id, cw_sc->instance_id, APM_APP_ID_SIZE);
 	copy_byte_arrays(proc_info.app_id, cw_sc->app_id, APM_APP_ID_SIZE);

ebpftracer/ebpf/l7/l7.c

@@ -279,7 +279,6 @@ struct {
 
 
 
-static __always_inline void span_context_to_cw_string_stream(struct apm_span_context *ctx, char *buff, char type_from);
 static __always_inline void cw_string_to_span_context(char *str, struct apm_span_context *ctx);
 static __always_inline void generate_random_bytes(unsigned char *buff, __u32 size);
 static __inline __attribute__((__always_inline__)) void cw_save_parent_tracking_span(struct apm_span_context *sc);
@@ -963,39 +962,44 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 
     bpf_map_delete_elem(&active_l7_requests, &k);
 	if (e->protocol == PROTOCOL_HTTP) {
-		bpf_printk("Response -------: k.pid:%d, k.fd:%d", k.pid, k.fd);
+//		bpf_printk("Response -------: k.pid:%d, k.fd:%d", k.pid, k.fd);
 //		__u64 trace_id = req->trace_id;
 		e->trace_id = req->trace_id;
-//        cw_bpf_debug("l7.c addr is --------:%d,%s",conn->sport,conn->saddr);
+//      cw_bpf_debug("l7.c addr is --------:%d,%s",conn->sport,conn->saddr);
 
-		bpf_printk("l7.c src  addr is --------:%pI4:%d",conn->saddr,conn->sport);
-		bpf_printk("l7.c dist addr is --------:%pI4:%d",conn->daddr,conn->dport);
+//		bpf_printk("l7.c src  addr is --------:%pI4:%d",conn->saddr,conn->sport);
+//		bpf_printk("l7.c dist addr is --------:%pI4:%d",conn->daddr,conn->dport);
         e->component_sport = conn->sport;
         e->component_dport = conn->dport;
         __builtin_memcpy(&e->component_saddr, &conn->saddr, sizeof(e->component_saddr));
         __builtin_memcpy(&e->component_daddr, &conn->daddr, sizeof(e->component_daddr));
-		bpf_printk("l7.c src  addr is --------:%pI4:%d",&e->component_saddr,e->component_sport);
-
-
-		__be32 v4 = *(__be32 *)&e->component_saddr[12];  // 取最后4字节
-		__u32  h  = bpf_ntohl(v4);                       // 转主机序
-
-		bpf_printk("comp src v4: %u.%u.%u",
-		           (h >> 24) & 0xff,
-		           (h >> 16) & 0xff,
-		           (h >>  8) & 0xff
-//		           (h >>  0) & 0xff
-		           );
-		bpf_printk("%u\n",(h >>  0) & 0xff);
+//		bpf_printk("l7.c src  addr is --------:%pI4:%d",&e->component_saddr,e->component_sport);
+
+
+		__be32 v4 = *(__be32 *)&e->component_daddr[12];  // 取最后4字节
+		__u32  rip  = bpf_ntohl(v4);                     // 转主机序
+//		bpf_printk("comp src v4: %u.%u.%u",
+//		           (rip >> 24) & 0xff,
+//		           (rip >> 16) & 0xff,
+//		           (rip >>  8) & 0xff
+////		           (rip >>  0) & 0xff
+//		           );
+//		bpf_printk("%u\n",(rip >>  0) & 0xff);
+		unsigned char *src_assumed;
+		unsigned char *src_span;
+		struct apm_span_context *sk_msg_sc = cw_get_current_span_context_by_ipport(rip, conn->dport);
+		if (sk_msg_sc) {
+			src_assumed = sk_msg_sc->assumed_app_id;
+			src_span    = sk_msg_sc->span_id;
+			cw_clean_current_span_context_by_ipport(rip, conn->dport, pid);
+		} else {
+			src_assumed = req->assumed_app_id;
+			src_span    = req->span_id;
+		}
+		cw_copy_byte_arrays(src_assumed, e->assumed_app_id, APM_ASSUMED_APP_ID_SIZE);
+		cw_copy_byte_arrays(src_span,    e->span_id,         APM_SPAN_ID_SIZE);
 
-//	    struct  apm_span_context * sc = cw_get_current_tracking_span();
-//	    if (sc) {
-		cw_copy_byte_arrays(req->assumed_app_id, e->assumed_app_id, APM_ASSUMED_APP_ID_SIZE);
-		cw_copy_byte_arrays(req->span_id, e->span_id, APM_SPAN_ID_SIZE);
-//		    for (int i = 0; i < APM_ASSUMED_APP_ID_SIZE; i++) {
-//			    cw_bpf_debug("assumed_app_id-assumed_app_id[%d] = %02x", i, req->assumed_app_id[i]);
-//		    }
-//	    for (int i = 0; i < APM_SPAN_ID_SIZE; i++) {
+		//	    for (int i = 0; i < APM_SPAN_ID_SIZE; i++) {
 //			    cw_bpf_debug("cw_get_current_tracking_span-span_id[%d] = %02x", i, req->span_id[i]);
 //		    }
 //	    }
@@ -1522,10 +1526,10 @@ int sk_msg_handler(struct sk_msg_md *msg)
 
 	// 拉满整条消息查Header
 	if (bpf_msg_pull_data(msg, 0, max_buf, 0) < 0){
-		bpf_printk("HTTP request return");
+//		bpf_printk("HTTP request return");
 		return SK_PASS;
 	}
-	bpf_printk("HTTP request return2");
+//	bpf_printk("HTTP request return2");
 //	bpf_printk("HTTP request return2 ：%d",  (__u64)msg->data_end - (__u64)msg->data);
 
 //	return SK_PASS;
@@ -1541,7 +1545,7 @@ int sk_msg_handler(struct sk_msg_md *msg)
 	if (bpf_probe_read_kernel(map_data->payload, sizeof(map_data->payload), (void *) (__u64) msg->data) != 0) {
 		return SK_PASS;
 	}
-	bpf_printk("HTTP request: %s",map_data->payload);
+//	bpf_printk("HTTP request: %s",map_data->payload);
 
 	// header 查询
 	long header_start_native = 0x0a0d312e312f5054LL; // 小端序下的 "TP/1.1\r\n" (0x54 0x50 0x2f 0x31 0x2e 0x31 0x0d 0x0a)
@@ -1559,7 +1563,7 @@ int sk_msg_handler(struct sk_msg_md *msg)
 		return SK_PASS;
 	}
 
-	bpf_printk("HTTP header_offset_idx: %d\n", map_data->header_offset_idx);
+//	bpf_printk("HTTP header_offset_idx: %d\n", map_data->header_offset_idx);
 
 	if (map_data->header_offset_idx > msg->size){
 		return SK_PASS;
@@ -1598,20 +1602,20 @@ int sk_msg_handler(struct sk_msg_md *msg)
 //		bpf_printk("cw-assumed_app_id[%d] = %02x", i, cw_sc->assumed_app_id[i]);
 //	}
 
-	bpf_printk("[java client] header:[%s],sizeof %d ", map_data->header_stream, sizeof(map_data->header_stream));
+//	bpf_printk("[client] header:[%s],sizeof %d ", map_data->header_stream, sizeof(map_data->header_stream));
+	__u32 tid =  (__u32)bpf_get_current_pid_tgid();
+//	bpf_printk("tid-set %llu",tid);
 
-	__u32 raw = msg->remote_port;
-	bpf_printk("raw_port=0x%x low_net=0x%x high_net=0x%x\n",
-	           raw, (raw & 0xffff), ((raw >> 16) & 0xffff));
+//	__u32 raw = msg->remote_port;
+//	bpf_printk("raw_port=0x%x low_net=0x%x high_net=0x%x\n",raw, (raw & 0xffff), ((raw >> 16) & 0xffff));
 	__u16 port_host = ( __u16 )( __builtin_bswap32(msg->remote_port) & 0xffff );
 
-
-//	__u32 rip = __builtin_bswap32(msg->remote_ip4); // host order
+	__u32 rip = __builtin_bswap32(msg->remote_ip4); // host order
+	__u16 rport = __builtin_bswap32(msg->remote_port);
 //	__u8 b0 = (rip >> 24) & 0xff;
 //	__u8 b1 = (rip >> 16) & 0xff;
 //	__u8 b2 = (rip >> 8) & 0xff;
 //	__u8 b3 = rip & 0xff;
-//	__u16 rport = __builtin_bswap32(msg->remote_port);
 //	bpf_printk("remote %u.%u.%u\n", b0, b1, b2);
 //	bpf_printk("remote port %u:%u\n", b3, rport);
 //	bpf_printk("remote port %u\n", rport);
@@ -1631,7 +1635,7 @@ int sk_msg_handler(struct sk_msg_md *msg)
 		return SK_PASS;
 	}
 
-	bpf_printk("bpf_msg_push_data suc.");
+//	bpf_printk("bpf_msg_push_data suc.");
 
 	// push_data 后；拉到写满Header处的长度
 //	__u32 need = msg->size < MAX_L7_IOVEC_BUF_SIZE ? msg->size : MAX_L7_IOVEC_BUF_SIZE;
@@ -1641,7 +1645,7 @@ int sk_msg_handler(struct sk_msg_md *msg)
 		bpf_printk("pull 0..%u fail ret=%ld\n", header_offset + ins_len, ret);
 		return SK_PASS;
 	}
-	bpf_printk("bpf_msg_pull_data suc.");
+//	bpf_printk("bpf_msg_pull_data suc.");
 
 	// 3. 写入插入的内容
 	char *data     = (char *)(long)msg->data;
@@ -1692,13 +1696,14 @@ int sk_msg_handler(struct sk_msg_md *msg)
 
 	}
 
-	// 保存span context
-	cw_save_current_tracking_span(cw_sc);
-	char vb[200] = {};
-	if (bpf_probe_read_kernel(vb, 200, (void *)data) == 0) {
-		bpf_printk("Modified HTTP request: %s\n", vb);
-		bpf_printk("Modified HTTP request: %d\n", (long) msg->data_end - (long) msg->data);
-	}
+	// save span context
+	cw_save_current_span_context_by_ipport(rip, rport, cw_sc);
+
+//	char vb[200] = {};
+//	if (bpf_probe_read_kernel(vb, 200, (void *)data) == 0) {
+//		bpf_printk("Modified HTTP request: %s\n", vb);
+//		bpf_printk("Modified HTTP request: %d\n", (long) msg->data_end - (long) msg->data);
+//	}
 	return SK_PASS;
 
 //	bpf_printk("sk_msg");