Fixed #TASK_QT-9810 JAVA横向堆栈

containers/container.go

@@ -1,12 +1,13 @@
 package containers
 
 import (
-	"github.com/coroot/coroot-node-agent/utils"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/coroot/coroot-node-agent/utils"
+
 	"github.com/coroot/coroot-node-agent/cgroup"
 	"github.com/coroot/coroot-node-agent/common"
 	"github.com/coroot/coroot-node-agent/ebpftracer"
@@ -1103,6 +1104,17 @@ func (c *Container) attachTlsUprobes(tracer *ebpftracer.Tracer, pid uint32) {
 	}
 }
 
+func (c *Container) attachJVMUprobes(tracer *ebpftracer.Tracer, pid uint32) {
+	p := c.processes[pid]
+	if p == nil {
+		return
+	}
+	if !p.jvmUprobesChecked {
+		p.uprobes = append(p.uprobes, tracer.AttachJavaNioReadUprobes(pid, c.instanceID)...)
+		p.jvmUprobesChecked = true
+	}
+}
+
 func resolveFd(pid uint32, fd uint64) (mntId string, logPath string) {
 	info := proc.GetFdInfo(pid, fd)
 	if info == nil {

containers/process.go

@@ -25,6 +25,7 @@ type Process struct {
 	uprobes               []link.Link
 	goTlsUprobesChecked   bool
 	openSslUprobesChecked bool
+	jvmUprobesChecked     bool
 }
 
 func NewProcess(pid uint32, stats *taskstats.Stats) *Process {

containers/registry.go

@@ -242,7 +242,7 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 				if c := r.getOrCreateContainer(e.Pid); c != nil {
 					c.onListenOpen(e.Pid, e.SrcAddr, false)
 					c.buildInstanceID()
-					c.attachTlsUprobes(r.tracer, e.Pid)
+					// c.attachTlsUprobes(r.tracer, e.Pid)
 				} else {
 					klog.Infoln("TCP listen open from unknown container", e)
 				}
@@ -255,7 +255,8 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 				//fmt.Println("ebpftracer.EventTypeConnectionOpen==================", e.Pid)
 				if c := r.getOrCreateContainer(e.Pid); c != nil {
 					c.onConnectionOpen(e.Pid, e.Fd, e.SrcAddr, e.DstAddr, e.Timestamp, false)
-					c.attachTlsUprobes(r.tracer, e.Pid)
+					// c.attachTlsUprobes(r.tracer, e.Pid)
+					c.attachJVMUprobes(r.tracer, e.Pid)
 				} else {
 					klog.Infoln("TCP connection from unknown container", e)
 				}

ebpftracer/build_linux.sh

@@ -28,7 +28,7 @@ echo "debug: $debug"
 #clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_x86 -D__x86_64__ $debug $TAG -c ./ebpf/ebpf.c -o ebpf512x86.o && llvm-strip --strip-debug ebpf512x86.o
 #llvm-strip --strip-debug ./ebpf512x86.o
 #chown 501:dialout ./ebpf512x86.o
-/usr/bin/clang -I. -Ivmlinux -Iinclude -Iebpf/utrace/go/include  -D__BPF_TRACING__ -D GROUP_LEADER_OFFSET_OVERRIDE=0 -DSTART_BOOTTIME_OFFSET_OVERRIDE=0 -DSTART_BOOTTIME_VARNAME=real_start_time  -std=gnu99 -Wimplicit-function-declaration \
+/usr/bin/clang -I. -Ivmlinux -Iinclude -Iebpf/include -Iebpf/utrace/go/include -Iebpf/utrace/java/include  -D__BPF_TRACING__ -D GROUP_LEADER_OFFSET_OVERRIDE=0 -DSTART_BOOTTIME_OFFSET_OVERRIDE=0 -DSTART_BOOTTIME_VARNAME=real_start_time  -std=gnu99 -Wimplicit-function-declaration \
 	-ffreestanding -fno-builtin -Wall \
 	-Wno-deprecated-declarations \
 	-Wno-gnu-variable-sized-type-not-at-end \

ebpftracer/ebpf/ebpf.c

@@ -53,5 +53,6 @@
 //#include "l7/openssl.c"
 #include "utrace/go/net/server.probe.bpf.c"
 #include "utrace/go/net/client.probe.bpf.c"
+#include "utrace/java/net/server.probe.bpf.c"
 #include "utrace/go/net/stack.probe.bpf.c"
 char _license[] SEC("license") = "GPL";

ebpftracer/ebpf/utrace/go/include/span_context.h

@@ -241,11 +241,11 @@ static __always_inline void cw_string_to_span_context(char *str, struct apm_span
 	hex_string_to_bytes(str + type_from_start_pos, APM_TYPE_FROM_STRING_SIZE, ctx->type_from);
 	hex_string_to_bytes(str + sample_start_pos, APM_SAMPLE_STRING_SIZE, ctx->sample);
 	hex_string_to_bytes(str + host_id_start_pos, APM_HOST_ID_STRING_SIZE, ctx->host_id);
-	hex_string_to_bytes(str + app_id_start_pos, APM_APP_ID_STRING_SIZE, ctx->app_id);
+	// hex_string_to_bytes(str + app_id_start_pos, APM_APP_ID_STRING_SIZE, ctx->app_id);
 	hex_string_to_bytes(str + instance_id_start_pos, APM_INSTANCE_ID_STRING_SIZE, ctx->instance_id);
 	hex_string_to_bytes(str + trace_id_start_pos, APM_TRACE_ID_STRING_SIZE, ctx->trace_id);
 	hex_string_to_bytes(str + assumed_app_id_start_pos, APM_ASSUMED_APP_ID_STRING_SIZE, ctx->assumed_app_id);
-	hex_string_to_bytes(str + span_id_start_pos, APM_SPAN_ID_STRING_SIZE, ctx->span_id);
+	// hex_string_to_bytes(str + span_id_start_pos, APM_SPAN_ID_STRING_SIZE, ctx->span_id);
 }
 
 

ebpftracer/ebpf/utrace/java/net/server.probe.bpf.c

@@ -0,0 +1,137 @@
+
+// #include "common.h"
+
+// #include "bpf_tracing.h"
+// #include "bpf_helpers.h"
+// #include <asm/ptrace.h>
+// #include "bpf_base.h"
+// #define ROCK_MAX_BUFFER_SIZE 65536
+
+static long (*bpf_probe_read_user_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 114;
+#define ROCK_MAX_LEN 1022
+
+struct sock_t {
+	int size;
+	int header_offset_idx;
+	char payload[ROCK_MAX_LEN];
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__type(key, int);
+	__type(value, struct sock_t);
+	__uint(max_entries, 1);
+} socket_heap SEC(".maps");
+
+struct {
+    __uint(type, BPF_MAP_TYPE_ARRAY);
+    __type(key, int); // 键类型为int
+    __uint(value_size, 124); // 数组的值的大小
+    __uint(max_entries, 1);
+} large_array_map SEC(".maps");
+
+static __inline int is_http_header(const char *data)
+{
+	int header_off = 0 ;
+	for (int i = 0; i < 512; i++) 
+	{
+		if (data[i] == 'H' &&
+		data[i+1] == 'T' &&
+		data[i+2] == 'T' &&
+		data[i+3] == 'P' &&
+		data[i+4] == '/' &&
+		data[i+5] == '1' &&
+		data[i+6] == '.'&&
+		data[i+7] == '1'&& data[i+8] == '\r' && data[i + 9] == '\n')
+		{
+			header_off = i+10;
+			if(data[header_off] == 'c' &&
+				data[header_off+1] == 'w' &&
+				data[header_off+2] == 't' &&
+				data[header_off+3] == 'r' &&
+				data[header_off+4] == 'a' &&
+				data[header_off+5] == 'c' &&
+				data[header_off+6] == 'e')
+				{
+					return header_off + 9;
+				}
+			else
+			{
+				return 0;
+			}
+		}
+	}
+	return header_off;
+}
+
+SEC("uprobe/Java_sun_nio_ch_FileDispatcherImpl_read0")
+int uprobe_Java_sun_nio_ch_FileDispatcherImpl_read0(struct pt_regs *ctx) {
+	//计算 rbp-0x30
+    bpf_printk("enter the uprobe_Java_sun_nio_ch_FileDispatcherImpl_read0\n");
+	unsigned long jhttpdata_ptr;
+	jhttpdata_ptr = (ctx)->rbp;
+	jhttpdata_ptr = jhttpdata_ptr - 0x70;
+	// bpf_printk("enter rbp is %llx.\n", jhttpdata_ptr);
+	jhttpdata_ptr = jhttpdata_ptr - 0x30;
+	//p rbp - 0x30
+	// bpf_printk("enter rbp -0x30 to rcx is %llx.\n", jhttpdata_ptr);
+
+	// x/1gx (p rbp-0x30)
+	unsigned long jbytechar_ptr;
+	long ret = bpf_probe_read_user(&jbytechar_ptr, sizeof(unsigned long), (void *) jhttpdata_ptr);
+	// bpf_printk("x/1gx (p rbp-0x30) value is %llx.\n", jbytechar_ptr);
+    bpf_printk("enter the uprobe_Java_sun_nio_ch_FileDispatcherImpl_read0 ------22222222\n");
+	if (ret != 0) {
+		// bpf_printk("Failed to read first level ptr: %d\n", ret);
+		return 0;
+	}
+	if (!jbytechar_ptr) {
+		// bpf_printk("First level pointer is null.\n");
+		return 0;
+	}
+    bpf_printk("enter the uprobe_Java_sun_nio_ch_FileDispatcherImpl_read0 ----111111111\n");
+	
+	//x/s addr
+	int key = 0;
+	struct sock_t *map_data = bpf_map_lookup_elem(&socket_heap, &key);
+	if (!map_data) {
+		return 0;
+	}
+	long err = bpf_probe_read_user_str(map_data->payload, sizeof(map_data->payload), (void *) (jbytechar_ptr));
+	if (err < 0) {
+		// bpf_printk("bpf_probe_read_user failed with return code: %d\n", err);
+		return 0;
+	}
+
+	bpf_printk("read data is : %s\n", map_data->payload);
+	// if (!is_http_request2(httpdata, 1024))
+	// {
+	// 	bpf_printk("not a http request\n");
+	// 	return 0;
+	// }
+
+
+	int offset = is_http_header(map_data->payload);
+	bpf_printk("found the header, %d\n", offset);
+	char *data;
+	data = bpf_map_lookup_elem(&large_array_map, &key);
+	if (!data)
+		return 0;
+	struct apm_span_context cw_parent_span_context = {};
+    if(offset > 0 && offset < 512 - 123)
+	{
+        __builtin_memcpy(data, &(map_data->payload[offset]), 123);
+		bpf_printk("found the data, %s\n", data);
+
+		cw_string_to_span_context(data, &cw_parent_span_context);
+	}
+    else
+    {
+        generate_random_bytes(cw_parent_span_context.trace_id, TRACE_ID_SIZE);
+    }
+	// 保存 trace_id 到psc
+    cw_save_parent_tracking_span(&cw_parent_span_context);
+	return 0;
+}
+
+// char __license[] SEC("license") = "Dual MIT/GPL";

ebpftracer/jvm.go

@@ -0,0 +1,40 @@
+package ebpftracer
+
+import (
+	"fmt"
+
+	"github.com/coroot/coroot-node-agent/utils"
+
+	"github.com/cilium/ebpf/link"
+)
+
+const (
+	// goServeHTTP           = "net/http.serverHandler.ServeHTTP"
+	binPath           = "/home/cloudwise/rock/code/jdk8u/build/linux-x86_64-normal-server-slowdebug/jdk/lib/amd64/libnio.so"
+	symbolsocketRead0 = "Java_sun_nio_ch_FileDispatcherImpl_read0"
+)
+
+func (t *Tracer) AttachJavaNioReadUprobes(pid uint32, insID utils.ID) []link.Link {
+	if t.disableL7Tracing {
+		return nil
+	}
+	var links []link.Link
+	ex, err := link.OpenExecutable(binPath)
+	if err != nil {
+		return nil
+	}
+	opt := link.UprobeOptions{
+		Offset: 103,
+	}
+	upread02, err := ex.Uprobe(symbolsocketRead0, t.uprobes["uprobe_Java_sun_nio_ch_FileDispatcherImpl_read0"], &opt)
+	if err != nil {
+		return nil
+	}
+	links = append(links, upread02)
+
+	if len(links) == 0 {
+		return nil
+	}
+	fmt.Println("jvm uprobes attached")
+	return links
+}