Fixed #TASK_GK-2944 多进程初始化

ebpftracer/ebpf/l7/l7.c

@@ -328,7 +328,19 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
 //        }
 
 //        cw_bpf_debug("off->task__files_offset:%x", off->task__files_offset);
-        cw_bpf_debug("e->test_id:%d", ttt->test_id);
+        struct ebpf_proc_info *info =
+                bpf_map_lookup_elem(&proc_info_map, &pid);
+        if (!info) {
+            return 0;
+        }
+
+        if(info->version >= GO_VERSION(1, 18, 0)){
+            cw_bpf_debug("e->GO_VERSION > 1.18:%d",info->version);
+        }
+        int offset_g_goid = info->offsets[OFFSET_IDX_GOID_RUNTIME_G];
+        cw_bpf_debug("offset_g_goid:%d", offset_g_goid);
+
+        cw_bpf_debug("e->test_id1111:%d", ttt->test_id);
         cw_bpf_debug("HTTP_END");
         return 0;
     }

ebpftracer/ebpf/uprobe_base_bpf.c

@@ -378,33 +378,29 @@ SEC("uprobe/runtime.execute")
 int runtime_execute(struct pt_regs *ctx)
 {
 	__u64 pid_tgid = bpf_get_current_pid_tgid();
-//	__u32 tgid = pid_tgid >> 32;
-//    __u64 goroutine_id2 = GOROUTINE(ctx);
-//    debug("[Go] [runtime.execute]%llu",goroutine_id2);
-
-//	struct ebpf_proc_info *info = bpf_map_lookup_elem(&proc_info_map, &tgid);
-//	if (!info) {
-//		return 0;
-//	}
-//    debug("[Go] [runtime.execute] has info");
+	__u32 tgid = pid_tgid >> 32;
 
-    int offset_g_goid = 152;
-//	if (offset_g_goid < 0) {
-//		return 0;
-//	}
+	struct ebpf_proc_info *info = bpf_map_lookup_elem(&proc_info_map, &tgid);
+	if (!info) {
+		return 0;
+	}
+	int offset_g_goid = info->offsets[OFFSET_IDX_GOID_RUNTIME_G];
+	if (offset_g_goid < 0) {
+		return 0;
+	}
 
 	void *g_ptr;
-    g_ptr = (void *)PT_GO_REGS_PARM1(ctx);
 
-//	if (is_register_based_call(info)) {
-//		g_ptr = (void *)PT_GO_REGS_PARM1(ctx);
-//	} else {
-//		bpf_probe_read(&g_ptr, sizeof(g_ptr), (void *)(PT_REGS_SP(ctx) + 8));
-//	}
+	if (is_register_based_call(info)) {
+		g_ptr = (void *)PT_GO_REGS_PARM1(ctx);
+	} else {
+		bpf_probe_read(&g_ptr, sizeof(g_ptr), (void *)(PT_REGS_SP(ctx) + 8));
+	}
 
 	__s64 goid = 0;
 	bpf_probe_read(&goid, sizeof(goid), g_ptr + offset_g_goid);
-    debug("[Go] [runtime.execute] goid:%llu",goid);
+	debug("[Go] [runtime.execute] goid:%llu",goid);
+
 	bpf_map_update_elem(&goroutines_map, &pid_tgid, &goid, BPF_ANY);
 
 	return 0;
@@ -420,26 +416,56 @@ SEC("uprobe/enter_runtime.newproc1")
 int enter_runtime_newproc1(struct pt_regs *ctx)
 {
 	__u64 pid_tgid = bpf_get_current_pid_tgid();
+	__u32 tgid = pid_tgid >> 32;
 
-	int offset_g_goid = 152;
+	struct ebpf_proc_info *info =
+			bpf_map_lookup_elem(&proc_info_map, &tgid);
+	if (!info) {
+		return 0;
+	}
+
+	// go less than 1.15 cannot get parent-child coroutine relationship
+	// ~ go1.14: func newproc1(fn *funcval, argp unsafe.Pointer, narg int32, callergp *g, callerpc uintptr)
+	if (info->version < GO_VERSION(1, 15, 0)) {
+		return 0;
+	}
 
+	int offset_g_goid = info->offsets[OFFSET_IDX_GOID_RUNTIME_G];
+	if (offset_g_goid < 0) {
+		return 0;
+	}
 
 	void *g_ptr;
-    g_ptr = (void *)PT_GO_REGS_PARM2(ctx);
+	if (is_register_based_call(info)) {
+		// https://github.com/golang/go/commit/8e5304f7298a0eef48e4796017c51b4d9aeb52b5
+		if (info->version >= GO_VERSION(1, 18, 0)) {
+			g_ptr = (void *)PT_GO_REGS_PARM2(ctx);
+		} else {
+			g_ptr = (void *)PT_GO_REGS_PARM4(ctx);
+		}
+	} else {
+		if (info->version >= GO_VERSION(1, 18, 0)) {
+			bpf_probe_read(&g_ptr, sizeof(g_ptr),
+			               (void *)(PT_REGS_SP(ctx) + 16));
+		} else {
+			bpf_probe_read(&g_ptr, sizeof(g_ptr),
+			               (void *)(PT_REGS_SP(ctx) + 32));
+		}
+	}
 
 	__s64 goid = 0;
 	bpf_probe_read(&goid, sizeof(goid), g_ptr + offset_g_goid);
-
 	if (!goid) {
 		return 0;
 	}
-    debug("[Go] [runtime.newproc1] goid:%llu",goid);
+	debug("[Go] [runtime.newproc1] goid:%llu",goid);
+
 	struct go_newproc_caller caller = {
-		.goid = goid,
-		.sp = (void *)PT_REGS_SP(ctx),
+			.goid = goid,
+			.sp = (void *)PT_REGS_SP(ctx),
 	};
 	bpf_map_update_elem(&pid_tgid_callerid_map, &pid_tgid, &caller,
-			    BPF_ANY);
+	                    BPF_ANY);
 	return 0;
 }
 
@@ -453,30 +479,49 @@ int exit_runtime_newproc1(struct pt_regs *ctx)
 	__u64 pid_tgid = bpf_get_current_pid_tgid();
 	__u32 tgid = pid_tgid >> 32;
 
-	int offset_g_goid = 152;
+	struct ebpf_proc_info *info =
+			bpf_map_lookup_elem(&proc_info_map, &tgid);
+	if (!info) {
+		return 0;
+	}
+
+	if(info->version < GO_VERSION(1, 15, 0)){
+		return 0;
+	}
+
+	int offset_g_goid = info->offsets[OFFSET_IDX_GOID_RUNTIME_G];
+	if (offset_g_goid < 0) {
+		return 0;
+	}
 
 	struct go_newproc_caller *caller =
-		bpf_map_lookup_elem(&pid_tgid_callerid_map, &pid_tgid);
+			bpf_map_lookup_elem(&pid_tgid_callerid_map, &pid_tgid);
 	if (!caller) {
 		return 0;
 	}
 
 	void *g_ptr;
-    g_ptr = (void *)PT_GO_REGS_PARM1(ctx);
+	if (is_register_based_call(info)) {
+		g_ptr = (void *)PT_GO_REGS_PARM1(ctx);
+	} else {
+		if (info->version >= GO_VERSION(1, 18, 0)) {
+			bpf_probe_read(&g_ptr, sizeof(g_ptr), caller->sp + 32);
+		} else {
+			bpf_probe_read(&g_ptr, sizeof(g_ptr), caller->sp + 48);
+		}
+	}
 
-    // 获取当前协程id
 	__s64 goid = 0;
 	bpf_probe_read(&goid, sizeof(goid), g_ptr + offset_g_goid);
 	if (!goid) {
 		bpf_map_delete_elem(&pid_tgid_callerid_map, &pid_tgid);
 		return 0;
 	}
-    debug("[Go] [runtime.newproc1.exit] current->goid:%llu",goid);
-    // 生成当前协程key
-    struct go_key key = { .tgid = tgid, .goid = goid };
-    goid = caller->goid;
-    debug("[Go] [runtime.newproc1.exit] caller->goid:%llu",goid);
-
+	debug("[Go] [runtime.newproc1.exit] current->goid:%llu",goid);
+	// 生成当前协程key
+	struct go_key key = { .tgid = tgid, .goid = goid };
+	goid = caller->goid;
+	debug("[Go] [runtime.newproc1.exit] caller->goid:%llu",goid);	goid = caller->goid;
 	bpf_map_update_elem(&go_ancerstor_map, &key, &goid, BPF_ANY);
 
 	bpf_map_delete_elem(&pid_tgid_callerid_map, &pid_tgid);

ebpftracer/tls.go

@@ -191,53 +191,37 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 		parts := strings.Split(realVersion, ".")
 		var major, minor, revision int
 		if len(parts) >= 3 {
-			// 输出每个版本号部分
 			major, err = strconv.Atoi(parts[0])
 			if err != nil {
-				fmt.Println("Error converting major version:", err)
+				log("Error converting major version:", err)
 			}
 			minor, err = strconv.Atoi(parts[1])
 			if err != nil {
-				fmt.Println("Error converting minor version:", err)
+				log("Error converting minor version:", err)
 			}
 			revision, err = strconv.Atoi(parts[2])
 			if err != nil {
-				fmt.Println("Error converting revision version:", err)
+				log("Error converting revision version:", err)
 			}
 			goVersion := ((major & 0xFF) << 16) + ((minor & 0xFF) << 8) + min(revision, 255)
 
-			// 输出每个版本号部分
-			fmt.Println("Major:", major)
-			fmt.Println("Minor:", minor)
-			fmt.Println("Revision:", revision)
-			fmt.Println("goVersion", goVersion)
+			klog.Infoln("Major:", major)
+			klog.Infoln("Minor:", minor)
+			klog.Infoln("Revision:", revision)
+			klog.Infoln("goVersion", goVersion)
 			info := tracer.EbpfProcInfo{}
 			info.Version = uint32(goVersion)
 			info.Offsets[tracer.OFFSET_IDX_GOID_RUNTIME_G] = uint16(offset)
 			info.NetTCPConnItab = uint64(0)
 			info.CryptoTLSConnItab = uint64(0)
 			info.CredentialsSyscallConnItab = uint64(0)
-			if tracer.Update_proc_info_to_map(t.collection, pid, info) == tracer.ETR_OK {
-				fmt.Println("set ok")
+			_, err = tracer.UpdateProcInfoToMap(t.collection, pid, info)
+			if err != nil {
+				klog.Error("failed to update program info", err)
 			}
 		}
 	}
 
-	//struct ebpf_proc_info {
-	//	__u32 version;
-	//	__u16 offsets[OFFSET_IDX_MAX];
-	//
-	//	// In golang, itab represents type, and in interface, struct is represented
-	//	// by the address of itab. We use itab to judge the structure type, and
-	//	// find the fd representing the connection after multiple jumps. These
-	//	// types are not available in Go ELF files without a symbol table.
-	//	// Go 用 itab 表示类型, 在 interface 中通过 itab 确定具体的 struct, 并根据
-	//	// struct 找到表示连接的 fd.
-	//	__u64 net_TCPConn_itab;
-	//	__u64 crypto_tls_Conn_itab; // TLS_HTTP1,TLS_HTTP2
-	//	__u64 credentials_syscallConn_itab; // gRPC
-	//}
-
 	var links []link.Link
 	for _, s := range symbols {
 		if elf.ST_TYPE(s.Info) != elf.STT_FUNC || s.Size == 0 {
@@ -267,23 +251,18 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 			l, err := exe.Uprobe(s.Name, t.uprobes["runtime_execute"], &link.UprobeOptions{Address: address})
 			if err != nil {
 				log("failed to attach write_enter uprobe", err)
-				fmt.Println("runtime.execute no")
-				fmt.Println(err)
+				klog.Infoln("runtime.execute no")
 				return nil
 			} else {
-				fmt.Println("runtime.execute ok")
+				klog.Infoln("runtime.execute ok")
 			}
 			links = append(links, l)
 
 		case "runtime.newproc1":
 			l, err := exe.Uprobe(s.Name, t.uprobes["enter_runtime_newproc1"], &link.UprobeOptions{Address: address})
 			if err != nil {
-				log("failed to attach write_enter uprobe", err)
-				fmt.Println("runtime.newproc1 no")
-				fmt.Println(err)
+				log("failed to attach newproc1 uprobe", err)
 				return nil
-			} else {
-				fmt.Println("runtime.newproc1 ok")
 			}
 			links = append(links, l)
 			sStart := s.Value - textSection.Addr

ebpftracer/tracer.go

@@ -140,7 +140,6 @@ func NewTracer(kernelVersion string, disableL7Tracing bool) *Tracer {
 }
 
 func (t *Tracer) Run(events chan<- Event) error {
-	time.Sleep(2 * time.Second)
 	if err := t.ebpf(events); err != nil {
 		return err
 	}

ebpftracer/tracer/btf_vmlinux.go

@@ -2,6 +2,7 @@ package tracer
 
 import (
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/btf"
@@ -80,7 +81,7 @@ func bpf_table_pre_set_value(collectionSpec *ebpf.CollectionSpec, opts *ebpf.Col
 
 }
 
-func bpf_table_set_value(collection *ebpf.Collection, mapName string, key uint32, data any) int {
+func bpf_table_set_value(collection *ebpf.Collection, mapName string, key uint32, data any) (int, error) {
 	m, ok := collection.Maps[mapName]
 	//for s, m2 := range collection.Maps {
 	//	fmt.Println(s, m2.String())
@@ -90,13 +91,12 @@ func bpf_table_set_value(collection *ebpf.Collection, mapName string, key uint32
 		k := make([]byte, 4)                  // Assuming int k size is 4 bytes
 		binary.LittleEndian.PutUint32(k, key) // Assuming the key is an integer
 		if err := m.Update(k, data, ebpf.UpdateAny); err != nil {
-			fmt.Println(err)
-			return ETR_UPDATE_MAP_FAILD
+			return ETR_UPDATE_MAP_FAILD, err
 		}
 	} else {
-		return ETR_UPDATE_MAP_FAILD
+		return ETR_UPDATE_MAP_FAILD, errors.New("cannot find map " + mapName)
 	}
-	return ETR_OK
+	return ETR_OK, nil
 }
 
 func test(collectionSpec *ebpf.CollectionSpec, opts *ebpf.CollectionOptions) {

ebpftracer/tracer/offset.go

@@ -190,6 +190,6 @@ func GetOffset(id ID, path string) (uint64, bool) {
 	return uint64(f.Val.(int64)), true
 }
 
-func Update_proc_info_to_map(collection *ebpf.Collection, pid uint32, proc any) int {
+func UpdateProcInfoToMap(collection *ebpf.Collection, pid uint32, proc any) (int, error) {
 	return bpf_table_set_value(collection, MAP_PROC_INFO_MAP_NAME, pid, proc)
 }

ebpftracer/tracer/socket.go

@@ -69,21 +69,18 @@ func __insert_output_prog_to_map(collection *ebpf.Collection, mapName string, pr
 	fmt.Println(prog, ok)
 	if ok {
 		progFd := prog.FD()
-		//fmt.Println("progFd", progFd)
-		if bpf_table_set_value(collection, mapName, key, uint32(progFd)) != ETR_OK {
-			//fmt.Println("no")
-		} else {
-			//fmt.Println("ok")
+		code, err := bpf_table_set_value(collection, mapName, key, uint32(progFd))
+		if err != nil {
+			klog.Error(err, code)
 		}
 	}
 }
 
 func update_protocol_filter_array(collection *ebpf.Collection) {
 	for i := 0; i < PROTO_NUM; i++ {
-		if bpf_table_set_value(collection, MAP_PROTO_FILTER_NAME, uint32(i), EbpfConfigProtocolFilter[i]) != ETR_OK {
-			//fmt.Println("no")
-		} else {
-			//fmt.Println("ok")
+		code, err := bpf_table_set_value(collection, MAP_PROTO_FILTER_NAME, uint32(i), EbpfConfigProtocolFilter[i])
+		if err != nil || code != ETR_OK {
+			klog.Error(err, code)
 		}
 	}
 }
@@ -102,10 +99,9 @@ func insert_adapt_kern_uid_to_map(collection *ebpf.Collection) {
 	pid := os.Getpid()
 	tid := syscall.Gettid()
 	adaptKernUID := uint64(pid)<<32 | uint64(tid)
-	if bpf_table_set_value(collection, MAP_ADAPT_KERN_UID_NAME, 0, uint32(adaptKernUID)) != ETR_OK {
-		fmt.Println("no")
-	} else {
-		fmt.Println("ok")
+	code, err := bpf_table_set_value(collection, MAP_ADAPT_KERN_UID_NAME, 0, uint32(adaptKernUID))
+	if err != nil || code != ETR_OK {
+		klog.Error(err, code)
 	}
 }