ebpf/ebpftracer/jvm.go

package ebpftracer

import (
	"errors"
	"github.com/coroot/coroot-node-agent/ebpftracer/tracer/inject"
	"github.com/coroot/coroot-node-agent/utils"
	. "github.com/coroot/coroot-node-agent/utils/modelse"
	klog "github.com/sirupsen/logrus"
	"io/ioutil"
	"runtime"
	"strings"

	"debug/elf"
	"fmt"
	"path/filepath"

	"github.com/cilium/ebpf/link"
	"github.com/coroot/coroot-node-agent/proc"
	"golang.org/x/arch/arm64/arm64asm"
	"golang.org/x/arch/x86/x86asm"
)

const (
	// goServeHTTP           = "net/http.serverHandler.ServeHTTP"
	// binPath           = "/root/code/jdk8u/build/linux-x86_64-normal-server-release/jdk/lib/amd64/libnio.so"
	symbolsocketRead0 = "Java_sun_nio_ch_FileDispatcherImpl_read0"
)

func (t *Tracer) AttachJavaNioReadUprobes(pid uint32, codeType CodeType, rootfs string) ([]link.Link, error) {
	if t.DisableL7Tracing() {
		return nil, nil
	}
	var links []link.Link
	var bpath string

	// JavaAOT 逻辑
	if codeType.IsJavaAotCode() {
		bpath = proc.Path(pid, "exe")
	} else {
		version := UsePIDToGetJDKVersion(pid)
		klog.Infof("[attach] java version is %s", version)
		bpath = getSoPath(pid, "libnio.so")
		if bpath == "" {
			return nil, errors.New("can not find nio.so")
		}
	}
	bpath = rootfs + bpath
	klog.Infof("[attach] find the nio.so path is  %s", bpath)
	ex, err := link.OpenExecutable(bpath)
	if err != nil {
		klog.Errorf("[attach] open executable: %v", err)
		return nil, err
	}
	ef, err := elf.Open(bpath)
	if err != nil {
		klog.Errorf("[attach] open elf: %v", err)
		return nil, err
	}
	defer ef.Close()

	symbols, err := ef.DynamicSymbols()
	if err != nil {
		if errors.Is(err, elf.ErrNoSymbols) {
			return nil, err
		}
		return nil, err
	}

	textSection := ef.Section(".text")
	if textSection == nil {
		return nil, errors.New("can not find .text section")
	}
	textSectionData, err := textSection.Data()
	if err != nil {
		return nil, err
	}
	textSectionLen := uint64(len(textSectionData) - 1)

	// opt := link.UprobeOptions{
	// 	Offset: 61,
	// }
	// upread02, err := ex.Uprobe(symbolsocketRead0, t.uprobes["uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0"], &opt)
	// if err != nil {
	// 	return nil
	// }
	// links = append(links, upread01)
	for _, s := range symbols {
		if elf.ST_TYPE(s.Info) != elf.STT_FUNC || s.Size == 0 {
			continue
		}
		switch s.Name {
		case symbolsocketRead0:
		default:
			continue
		}
		address := s.Value
		for _, p := range ef.Progs {
			if p.Type != elf.PT_LOAD || (p.Flags&elf.PF_X) == 0 {
				continue
			}

			if p.Vaddr <= s.Value && s.Value < (p.Vaddr+p.Memsz) {
				address = s.Value - p.Vaddr + p.Off
				break
			}
		}
		klog.Infof("[attach] java symbol name is  %s", s.Name)
		switch s.Name {
		case symbolsocketRead0:
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getCallNextMoveOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				return nil, fmt.Errorf("failed to attach uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0 uprobe")
			}
			for _, offset := range returnOffsets {
				l, err := ex.Uprobe(s.Name, t.uprobes["uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0"], &link.UprobeOptions{Address: address, Offset: uint64(offset), PID: int(pid)})
				if err != nil {
					return nil, fmt.Errorf("failed to attach uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0 uprobe")
				}
				klog.Infof("[attach] java symbol offset is  %s", offset)
				links = append(links, l)
			}
		}
	}

	if len(links) == 0 {
		return nil, fmt.Errorf("no links found for %s", bpath)
	}
	klog.WithField("pid", pid).Infof("[attach] libnio attached!")
	return links, nil
}

func (t *Tracer) AttachJavaNetWriteUprobes(pid uint32, rootfs string) ([]link.Link, error) {
	if t.DisableL7Tracing() {
		return nil, nil
	}
	// 关闭横向串联
	if t.DisableE2ETracing() {
		return nil, nil
	}
	cwJvmLibPath := utils.GetDefaultLibsPath("jvm", "cwlibnet.so")
	tmpSo := "/tmp/cwlibnet.so"
	procLoadPath := cwJvmLibPath
	if rootfs != "" {
		// copy
		size, err := utils.CopyFile(cwJvmLibPath, rootfs+tmpSo)
		if err != nil || size == 0 {
			return nil, fmt.Errorf("[jvm] Failed to copy cwlibnet.so %v", err)
		}
		cwJvmLibPath = rootfs + tmpSo
		procLoadPath = tmpSo
	}
	//inject
	originFunc := "Java_java_net_SocketOutputStream_socketWrite0"

	uProbeData := inject.UprobeData{
		Offset:  53,
		Func:    originFunc,
		ELFPath: cwJvmLibPath,
	}

	if runtime.GOARCH == "arm64" {
		uProbeData = inject.UprobeData{
			Offset:  8,
			Func:    "CW_" + originFunc,
			ELFPath: cwJvmLibPath,
		}
	}

	jvmInjector := &inject.JvmInjector{
		Pid: int(pid),
		ReleaseLibNetInfo: inject.LibNetInfo{
			LibName: "libnet.so",
			FuncSymbol: inject.InstInfo{
				SymName: originFunc,
			},
		},
		DebugLibNetInfo: inject.LibNetInfo{
			// TODO 根据版本设置
			LibName: filepath.Base(uProbeData.ELFPath),
			// TODO 根据版本设置
			LibPath: uProbeData.ELFPath,
			FuncSymbol: inject.InstInfo{
				SymName: uProbeData.Func,
			},
			ProcLoadPath: procLoadPath,
		},
		RecodeInfo: inject.LibNetInfo{FuncSymbol: inject.InstInfo{SymName: "CW_RECODE_" + originFunc}},
		Uprobe:     uProbeData,
		Rootfs:     rootfs,
	}

	err := inject.JvmInject(jvmInjector)
	if err != nil {
		klog.WithError(err).Errorf("[jvm] inject.JvmInject error.")
		return nil, err
	}

	var links []link.Link
	ex, err := link.OpenExecutable(jvmInjector.Uprobe.ELFPath)
	if err != nil {
		return nil, err
	}
	opt := link.UprobeOptions{
		Offset: uint64(jvmInjector.Uprobe.Offset),
		PID:    int(pid),
	}
	upread02, err := ex.Uprobe(jvmInjector.Uprobe.Func, t.uprobes["uprobe_Java_java_net_SocketOutputStream_socketWrite0"], &opt)
	if err != nil {
		return nil, err
	}

	links = append(links, upread02)

	if len(links) == 0 {
		return nil, errors.New("can not find uprobe_Java_net_SocketOutputStream_socketWrite0")
	}
	klog.WithField("pid", pid).Infoln("[jvm] libnet attached")
	return links, nil
}

func getCallNextMoveOffsets(machine elf.Machine, instructions []byte) []int {
	var res []int
	firstCall := 0
	switch machine {
	case elf.EM_X86_64:
		for i := 0; i < len(instructions); {
			ins, err := x86asm.Decode(instructions[i:], 64)
			if err == nil && ins.Op == x86asm.CALL {
				if firstCall == 0 {
					firstCall = 1
				} else {
					i += ins.Len
					res = append(res, i)
					return res
				}
			}
			i += ins.Len
		}
	case elf.EM_AARCH64:
		for i := 0; i < len(instructions); {
			ins, err := arm64asm.Decode(instructions[i:])
			if err == nil && ins.Op == arm64asm.BL {
				if firstCall == 0 {
					firstCall = 1
				} else {
					i += 4
					res = append(res, i)
					return res
				}
			}
			i += 4
		}
	}
	return res
}

func getSoPath(pid uint32, soname string) string {
	// 获取进程的maps文件路径
	mapsPath := fmt.Sprintf("/proc/%d/maps", pid)

	// 读取maps文件内容
	mapsData, err := ioutil.ReadFile(mapsPath)
	if err != nil {
		fmt.Println("lookup so error.")
		return ""
	}

	lines := strings.Split(string(mapsData), "\n")
	for _, line := range lines {
		fields := strings.Fields(line)
		if len(fields) >= 6 {
			perms := fields[1]
			path := fields[len(fields)-1]

			if strings.Contains(perms, "x") && filepath.Ext(path) == ".so" {
				if strings.Contains(path, soname) {
					return path
				}
			}
		}
	}
	return ""
}