Fixed #TASK_QT-9810 处理容器进程注入问题

containers/container.go

@@ -46,6 +46,7 @@ type ContainerMetadata struct {
 	logDecoder  logparser.Decoder
 	hostListens map[string][]netaddr.IPPort
 	networks    map[string]ContainerNetwork
+	rootfs      string
 }
 
 type Delays struct {
@@ -1129,7 +1130,7 @@ func (c *Container) GetCodeTypeFromCache(pid uint32) CodeType {
 		return CodeTypeUnknown
 	}
 	if p.codeType.IsWaitCheck() {
-		p.codeType = GetExeType(pid)
+		p.codeType = GetExeType(pid, c.getRootfs())
 	}
 	return p.codeType
 }
@@ -1171,14 +1172,14 @@ func (c *Container) attachJVMUprobes(tracer *ebpftracer.Tracer, pid uint32) erro
 	if !p.jvmUprobesChecked {
 		tracer.InitKProcInfo(pid, &c.AppInfo)
 
-		libNioProbes, err := tracer.AttachJavaNioReadUprobes(pid, codeType)
+		libNioProbes, err := tracer.AttachJavaNioReadUprobes(pid, codeType, c.getRootfs())
 		if err != nil {
 			klog.Error(err)
 			return err
 		}
 		p.uprobes = append(p.uprobes, libNioProbes...)
 
-		libNetProbes, err := tracer.AttachJavaNetWriteUprobes(pid)
+		libNetProbes, err := tracer.AttachJavaNetWriteUprobes(pid, c.getRootfs())
 		if err != nil {
 			klog.Error(err)
 			return err

containers/container_apm.go

@@ -547,3 +547,10 @@ func (c *Container) detachUprobes(pid uint32) {
 		}
 	}
 }
+
+func (c *Container) getRootfs() string {
+	if c.metadata != nil && c.metadata.rootfs != "" {
+		return c.metadata.rootfs
+	}
+	return ""
+}

containers/containerd.go

@@ -65,6 +65,7 @@ func ContainerdInspect(containerID string) (*ContainerMetadata, error) {
 		labels:  c.Labels,
 		image:   c.Image,
 		volumes: map[string]string{},
+		rootfs:  fmt.Sprintf("/run/containerd/io.containerd.runtime.v2.task/k8s.io/%s/rootfs", containerID),
 	}
 
 	var spec oci.Spec

containers/dockerd.go

@@ -56,6 +56,7 @@ func DockerdInspect(containerID string) (*ContainerMetadata, error) {
 		volumes:     map[string]string{},
 		hostListens: map[string][]netaddr.IPPort{},
 		networks:    map[string]ContainerNetwork{},
+		rootfs:      c.GraphDriver.Data["MergedDir"],
 	}
 	for _, m := range c.Mounts {
 		res.volumes[m.Destination] = common.ParseKubernetesVolumeSource(m.Source)

containers/registry.go

@@ -196,7 +196,7 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 			}
 			runtimeApps := make(map[uint32]AppStatusInfo)
 			for pid, c := range r.containersByPid {
-				if !common.IsOpenFilter() {
+				if c != nil && !common.IsOpenFilter() {
 					verifyAttachConditions := c.verifyAttachConditions(r, pid)
 					if verifyAttachConditions {
 						err = c.RegisterAppInfo(r, pid)
@@ -530,7 +530,8 @@ func (r *Registry) getOrCreateContainer(pid uint32) *Container {
 }
 
 func calcId(cg *cgroup.Cgroup, md *ContainerMetadata, pid uint32) (ContainerID, map[string]string) {
-	extensionTag := map[string]string{Namespace: "", Workload: "", PodName: "", ProcessName: ""}
+	procName := proc.GetProcName(pid)
+	extensionTag := map[string]string{Namespace: "", Workload: "", PodName: "", ProcessName: procName}
 	if cg.ContainerType == cgroup.ContainerTypeSystemdService {
 		if strings.HasPrefix(cg.ContainerId, "/system.slice/crio-conmon-") {
 			return "", extensionTag
@@ -538,8 +539,7 @@ func calcId(cg *cgroup.Cgroup, md *ContainerMetadata, pid uint32) (ContainerID,
 		return ContainerID(cg.ContainerId), extensionTag
 	}
 	if cg.ContainerType == cgroup.ContainerTypeStandaloneProcess {
-		procName := proc.GetProcName(pid)
-		extensionTag[ProcessName] = procName
+		//extensionTag[ProcessName] = procName
 		return ContainerID(fmt.Sprintf("/%s/%s/%d", "standalone", proc.GetProcName(pid), pid)), extensionTag
 	}
 	switch cg.ContainerType {
@@ -563,7 +563,7 @@ func calcId(cg *cgroup.Cgroup, md *ContainerMetadata, pid uint32) (ContainerID,
 		extensionTag[Namespace] = namespace
 		extensionTag[Workload] = ""
 		extensionTag[PodName] = pod
-		extensionTag[ProcessName] = name
+		//extensionTag[ProcessName] = name
 		return ContainerID(fmt.Sprintf("/k8s/%s/%s/%s", namespace, pod, name)), extensionTag
 	}
 	if taskNameParts := strings.SplitN(md.labels["com.docker.swarm.task.name"], ".", 3); len(taskNameParts) == 3 {

containers/util.go

@@ -5,19 +5,17 @@ import (
 	"fmt"
 	. "github.com/coroot/coroot-node-agent/utils/modelse"
 	klog "github.com/sirupsen/logrus"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"regexp"
 	"runtime"
 	"strings"
-
 )
 
 var libjvmRegex = regexp.MustCompile(`.*/libjvm\.so`)
 
-func GetExeType(pid uint32) CodeType {
-	mapsFilePath := fmt.Sprintf("/proc/%d/maps", pid)
+func GetExeType(pid uint32, rootfs string) CodeType {
+	mapsFilePath := fmt.Sprintf("%sproc/%d/maps", "/", pid)
 
 	data, err := os.ReadFile(mapsFilePath)
 	if err != nil {
@@ -29,18 +27,18 @@ func GetExeType(pid uint32) CodeType {
 
 	if libjvmRegex.MatchString(content) {
 		//fmt.Println("is java process")
-		if isJavaAotProcess(pid) {
+		if isJavaAotProcess(pid, rootfs) {
 			//fmt.Println("is javaAot process")
 			return CodeTypeJavaAot
 		}
 		return CodeTypeJava
-	} else if isJavaAotProcess(pid) {
+	} else if isJavaAotProcess(pid, rootfs) {
 		//fmt.Println("is javaAot process")
 		return CodeTypeJavaAot
-	} else if isGoProcess(pid) {
+	} else if isGoProcess(pid, rootfs) {
 		//fmt.Println("is go process")
 		return CodeTypeGo
-	} else if isNetCoreProcess(pid) {
+	} else if isNetCoreProcess(pid, rootfs) {
 		//	fmt.Println("is netcore process")
 		return CodeTypeNetCoreAot
 	}
@@ -48,18 +46,18 @@ func GetExeType(pid uint32) CodeType {
 }
 
 // isJavaAotProcess checks if the process with the given PID is a GraalVM native image
-func isJavaAotProcess(pid uint32) bool {
+func isJavaAotProcess(pid uint32, rootfs string) bool {
 	// Get the executable path for the given PID
-	exePath, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
+	exePath, err := os.Readlink(fmt.Sprintf("%sproc/%d/exe", "/", pid))
 	if err != nil {
 		fmt.Printf("Error reading executable path for PID %d: %v\n", pid, err)
 		return false
 	}
 
 	// Read the content of the executable file
-	content, err := ioutil.ReadFile(exePath)
+	content, err := os.ReadFile(fmt.Sprintf("%s%s", rootfs, exePath))
 	if err != nil {
-		fmt.Printf("Error reading executable file for PID %d: %v\n", pid, err)
+		klog.WithError(err).Errorf("Error reading executable file for PID %d\n", pid)
 		return false
 	}
 
@@ -71,15 +69,15 @@ func isJavaAotProcess(pid uint32) bool {
 	return false
 }
 
-func isNetCoreProcess(pid uint32) bool {
+func isNetCoreProcess(pid uint32, rootfs string) bool {
 	path, err := getProcessPath(pid)
 	if err != nil {
-		fmt.Printf("无法获取进程路径：%s\n", err)
+		//fmt.Printf("无法获取进程路径：%s\n", err)
 		return false
 	}
-	ef, err := elf.Open(path)
+	ef, err := elf.Open(rootfs + path)
 	if err != nil {
-		fmt.Println("failed to open as elf binary", err)
+		klog.WithError(err).Error("failed to open as elf binary")
 		return false
 	}
 	defer ef.Close()
@@ -92,13 +90,13 @@ func isNetCoreProcess(pid uint32) bool {
 	return false
 }
 
-func isGoProcess(pid uint32) bool {
+func isGoProcess(pid uint32, rootfs string) bool {
 	path, err := getProcessPath(pid)
 	if err != nil {
-		fmt.Printf("无法获取进程路径：%s\n", err)
+		klog.WithError(err).Error("failed to open as elf binary")
 		return false
 	}
-	ef, err := elf.Open(path)
+	ef, err := elf.Open(rootfs + path)
 	if err != nil {
 		fmt.Println("failed to open as elf binary", err)
 		return false

ebpftracer/ebpf/include/bpf_base.h

@@ -106,6 +106,7 @@ static long (*bpf_probe_read_user_str)(void *dst, __u32 size, const void *unsafe
 #define PT_REGS_PARM4(x) ((x)->rcx)
 #define PT_REGS_PARM5(x) ((x)->r8)
 #define PT_REGS_PARM6(x) ((x)->r9)
+#define PT_REGS_12(x) ((x)->r12)
 #define PT_REGS_RET(x) ((x)->rsp)
 #define PT_REGS_FP(x) ((x)->rbp)
 #define PT_REGS_RC(x) ((x)->rax)

ebpftracer/ebpf/utrace/java/include/java_common.h

@@ -8,7 +8,7 @@
 
 #if defined(__x86_64__)
 #define PT_LEN_REGS(x) (PT_REGS_RBP(x) - 0x10058);
-#define PT_HTTP_RESP_REGS(x) PT_REGS_PARM2(x)
+#define PT_HTTP_RESP_REGS(x) PT_REGS_12(x)
 #elif defined(__aarch64__)
 #define PT_LEN_REGS(x) PT_REGS_SP(x);
 #define PT_HTTP_RESP_REGS(ctx) ({                \

ebpftracer/jvm.go

@@ -26,16 +26,16 @@ const (
 	symbolsocketRead0 = "Java_sun_nio_ch_FileDispatcherImpl_read0"
 )
 
-func (t *Tracer) AttachJavaNioReadUprobes(pid uint32, codeType CodeType) ([]link.Link, error) {
+func (t *Tracer) AttachJavaNioReadUprobes(pid uint32, codeType CodeType, rootfs string) ([]link.Link, error) {
 	if t.DisableL7Tracing() {
 		return nil, nil
 	}
 	var links []link.Link
-	var bpath = ""
+	var bpath string
 
 	// JavaAOT 逻辑
 	if codeType.IsJavaAotCode() {
-		bpath = proc.Path(uint32(pid), "exe")
+		bpath = proc.Path(pid, "exe")
 	} else {
 		version := UsePIDToGetJDKVersion(pid)
 		klog.Infof("[attach] java version is %s", version)
@@ -44,15 +44,17 @@ func (t *Tracer) AttachJavaNioReadUprobes(pid uint32, codeType CodeType) ([]link
 			return nil, errors.New("can not find nio.so")
 		}
 	}
+	bpath = rootfs + bpath
 	klog.Infof("[attach] find the nio.so path is  %s", bpath)
 	ex, err := link.OpenExecutable(bpath)
 	if err != nil {
+		klog.Errorf("[attach] open executable: %v", err)
 		return nil, err
 	}
 	ef, err := elf.Open(bpath)
 	if err != nil {
+		klog.Errorf("[attach] open elf: %v", err)
 		return nil, err
-		//PID:    int(pid),
 	}
 	defer ef.Close()
 
@@ -120,6 +122,7 @@ func (t *Tracer) AttachJavaNioReadUprobes(pid uint32, codeType CodeType) ([]link
 				if err != nil {
 					return nil, fmt.Errorf("failed to attach uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0 uprobe")
 				}
+				klog.Infof("[attach] java symbol offset is  %s", offset)
 				links = append(links, l)
 			}
 		}
@@ -132,7 +135,7 @@ func (t *Tracer) AttachJavaNioReadUprobes(pid uint32, codeType CodeType) ([]link
 	return links, nil
 }
 
-func (t *Tracer) AttachJavaNetWriteUprobes(pid uint32) ([]link.Link, error) {
+func (t *Tracer) AttachJavaNetWriteUprobes(pid uint32, rootfs string) ([]link.Link, error) {
 	if t.DisableL7Tracing() {
 		return nil, nil
 	}
@@ -141,6 +144,17 @@ func (t *Tracer) AttachJavaNetWriteUprobes(pid uint32) ([]link.Link, error) {
 		return nil, nil
 	}
 	cwJvmLibPath := utils.GetDefaultLibsPath("jvm", "cwlibnet.so")
+	tmpSo := "/tmp/cwlibnet.so"
+	procLoadPath := cwJvmLibPath
+	if rootfs != "" {
+		// copy
+		size, err := utils.CopyFile(cwJvmLibPath, rootfs+tmpSo)
+		if err != nil || size == 0 {
+			return nil, fmt.Errorf("[jvm] Failed to copy cwlibnet.so %v", err)
+		}
+		cwJvmLibPath = rootfs + tmpSo
+		procLoadPath = tmpSo
+	}
 	//inject
 	originFunc := "Java_java_net_SocketOutputStream_socketWrite0"
 
@@ -174,13 +188,16 @@ func (t *Tracer) AttachJavaNetWriteUprobes(pid uint32) ([]link.Link, error) {
 			FuncSymbol: inject.InstInfo{
 				SymName: uProbeData.Func,
 			},
+			ProcLoadPath: procLoadPath,
 		},
 		RecodeInfo: inject.LibNetInfo{FuncSymbol: inject.InstInfo{SymName: "CW_RECODE_" + originFunc}},
 		Uprobe:     uProbeData,
+		Rootfs:     rootfs,
 	}
 
 	err := inject.JvmInject(jvmInjector)
 	if err != nil {
+		klog.WithError(err).Errorf("[jvm] inject.JvmInject error.")
 		return nil, err
 	}
 

ebpftracer/tracer/inject/include/hotpatch.h

@@ -108,7 +108,7 @@ int hotpatch_detach(hotpatch_t *);
 int hotpatch_set_execution_pointer(hotpatch_t *, uintptr_t location);
 
 
-int cw_inject_library(int pid, int verbose, char *dll);
+int cw_inject_library(int pid, int verbose, char *dll, char *rootfs);
 
 #ifdef __cplusplus
 } /* end of extern C */

ebpftracer/tracer/inject/inject_linux_amd64.go

@@ -45,10 +45,11 @@ type InnerSymbolInfo struct {
 }
 
 type LibNetInfo struct {
-	LibName     string
-	LibPath     string
-	FuncSymbol  InstInfo
-	InnerSymbol InnerSymbolInfo
+	LibName      string
+	LibPath      string
+	FuncSymbol   InstInfo
+	InnerSymbol  InnerSymbolInfo
+	ProcLoadPath string
 }
 
 type UprobeData struct {
@@ -75,6 +76,7 @@ type JvmInjector struct {
 		NetSendFuncCheck bool
 	}
 	Uprobe UprobeData
+	Rootfs string
 }
 
 func (j *JvmInjector) findReleaseAddressInfoFromMem() error {
@@ -468,8 +470,8 @@ func (j *JvmInjector) findLibBaseFromProcMaps(libName string) (uint64, string, e
 			if len(fields) > 5 {
 				path := fields[5]
 				if strings.HasSuffix(path, ".so") {
-					klog.Infof("[inject] found library %s", path)
-					return start, path, nil
+					klog.Infof("[inject] found library in map %s", path)
+					return start, j.Rootfs + path, nil
 				}
 			}
 		}
@@ -502,7 +504,7 @@ func (j *JvmInjector) findLibBaseByPathFromProcMaps(libPath string) (uint64, str
 		}
 	}
 
-	return 1, "", fmt.Errorf("library %s not found", libPath)
+	return 1, "", fmt.Errorf("library %s not found in process.", libPath)
 }
 
 func (j *JvmInjector) getFunctionOffset(libPath, functionName string) (elf.Symbol, error) {
@@ -578,16 +580,16 @@ func (j *JvmInjector) findDebugFuncContextFromLibPath() error {
 	//libName := j.DebugLibNetInfo.LibPath
 
 	// 获取release库的基地址
-	baseAddress, libPath, err := j.findLibBaseByPathFromProcMaps(j.DebugLibNetInfo.LibPath)
-	fmt.Println("debug libPath", libPath)
+	baseAddress, libPath, err := j.findLibBaseByPathFromProcMaps(j.DebugLibNetInfo.ProcLoadPath)
+	klog.Infof("[inject] debug base address of %s : %x", libPath, baseAddress)
 	functionName := j.DebugLibNetInfo.FuncSymbol.SymName
-	j.DebugLibNetInfo.LibPath = libPath
+	//j.DebugLibNetInfo.LibPath = libPath
 	if err != nil {
 		return err
 	}
 
 	// 获取函数的偏移量
-	functionSym, err := j.getFunctionOffset(libPath, functionName)
+	functionSym, err := j.getFunctionOffset(j.DebugLibNetInfo.LibPath, functionName)
 	// 计算函数的实际内存地址
 	j.DebugLibNetInfo.FuncSymbol.SymAddr = baseAddress + functionSym.Value
 	j.DebugLibNetInfo.FuncSymbol.SymSize = functionSym.Size
@@ -624,9 +626,10 @@ func printCodeData(data LibNetInfo) {
 }
 
 func (j *JvmInjector) jvmInjectLib() int {
-	dll := C.CString(j.DebugLibNetInfo.LibPath) // 替换为实际的DLL路径
-	defer C.free(unsafe.Pointer(dll))           // 确保在使用完字符串后释放内存
-	result := C.cw_inject_library(C.int(j.Pid), C.int(1), dll)
+	dll := C.CString(j.DebugLibNetInfo.ProcLoadPath)
+	rootfs := C.CString(j.Rootfs)
+	defer C.free(unsafe.Pointer(dll))
+	result := C.cw_inject_library(C.int(j.Pid), C.int(1), dll, rootfs)
 	fmt.Printf("Result: %d\n", result)
 	return int(result)
 }
@@ -796,11 +799,14 @@ func JvmInject(jvmInjector *JvmInjector) error {
 	if err != nil {
 		// load so
 		if _type == 1 {
-			fmt.Println(err, "Load it.")
-			if jvmInjector.jvmInjectLib() == 0 {
+			klog.Infoln("[inject] start load so.")
+			resCode := jvmInjector.jvmInjectLib()
+			if resCode == 0 {
+				klog.Infof("[inject] load so successful. proc load path is [%s], file path in node is [%s]", jvmInjector.DebugLibNetInfo.ProcLoadPath, jvmInjector.DebugLibNetInfo.LibPath)
 				jvmInjector.PreCheck.LoadingCheck = true
 			} else {
-				return err
+				klog.Errorf("[inject] Failed load so. so path is [%s]", jvmInjector.DebugLibNetInfo.LibPath)
+				return fmt.Errorf("[inject] Failed load so. code is %d so path is [%s]", resCode, jvmInjector.DebugLibNetInfo.LibPath)
 			}
 		}
 	} else {

utils/util.go

@@ -410,7 +410,7 @@ func UpdateConfig(oldPath, newPath string) error {
 	return nil
 }
 
-func CopyFile(dstName, srcName string) (written int64, err error) {
+func CopyFile(srcName, dstName string) (written int64, err error) {
 	src, err := os.Open(srcName)
 	if err != nil {
 		return