ebpf/ebpftracer/jvm.go

package ebpftracer

import (
	"debug/elf"
	"errors"
	"fmt"
	"github.com/cilium/ebpf/link"
	"github.com/coroot/coroot-node-agent/ebpftracer/tracer/inject"
	"github.com/coroot/coroot-node-agent/proc"
	"github.com/coroot/coroot-node-agent/utils"
	. "github.com/coroot/coroot-node-agent/utils/modelse"
	klog "github.com/sirupsen/logrus"
	"golang.org/x/arch/arm64/arm64asm"
	"golang.org/x/arch/x86/x86asm"
	"runtime"
)

const (
	// goServeHTTP           = "net/http.serverHandler.ServeHTTP"
	// binPath           = "/root/code/jdk8u/build/linux-x86_64-normal-server-release/jdk/lib/amd64/libnio.so"
	symbolsocketRead0  = "Java_sun_nio_ch_FileDispatcherImpl_read0"
	symbolsocketWrite0 = "Java_java_net_SocketOutputStream_socketWrite0"
)

func (t *Tracer) AttachJavaNioReadUprobes(pid uint32, codeType CodeType, rootfs string) ([]link.Link, error) {
	if t.DisableL7Tracing() {
		return nil, nil
	}
	var links []link.Link
	var bpath string

	// JavaAOT 逻辑
	if codeType.IsJavaAotCode() {
		bpath = proc.Path(pid, "exe")
	} else {
		//version := UsePIDToGetJDKVersion(pid)
		//klog.Infof("[attach] java version is %s", version)
		var err error
		bpath, err = utils.GetSoPath(pid, "libnio.so", rootfs)
		if err != nil {
			return nil, err
		}
		if bpath == "" {
			return nil, errors.New("can not find nio.so")
		}
	}
	//bpath = rootfs + bpath
	klog.Infof("[attach] find the nio.so path is  %s", bpath)
	ex, err := link.OpenExecutable(bpath)
	if err != nil {
		klog.Errorf("[attach] open executable: %v", err)
		return nil, err
	}
	ef, err := elf.Open(bpath)
	if err != nil {
		klog.Errorf("[attach] open elf: %v", err)
		return nil, err
	}
	defer ef.Close()

	symbols, err := ef.DynamicSymbols()
	if err != nil {
		if errors.Is(err, elf.ErrNoSymbols) {
			return nil, err
		}
		return nil, err
	}

	textSection := ef.Section(".text")
	if textSection == nil {
		return nil, errors.New("can not find .text section")
	}
	textSectionData, err := textSection.Data()
	if err != nil {
		return nil, err
	}
	textSectionLen := uint64(len(textSectionData) - 1)

	// opt := link.UprobeOptions{
	// 	Offset: 61,
	// }
	// upread02, err := ex.Uprobe(symbolsocketRead0, t.uprobes["uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0"], &opt)
	// if err != nil {
	// 	return nil
	// }
	// links = append(links, upread01)
	for _, s := range symbols {
		if elf.ST_TYPE(s.Info) != elf.STT_FUNC || s.Size == 0 {
			continue
		}
		switch s.Name {
		case symbolsocketRead0:
		default:
			continue
		}
		address := s.Value
		for _, p := range ef.Progs {
			if p.Type != elf.PT_LOAD || (p.Flags&elf.PF_X) == 0 {
				continue
			}

			if p.Vaddr <= s.Value && s.Value < (p.Vaddr+p.Memsz) {
				address = s.Value - p.Vaddr + p.Off
				break
			}
		}
		klog.Infof("[attach] java symbol name is  %s", s.Name)
		switch s.Name {
		case symbolsocketRead0:
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getCallNextMoveOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				return nil, fmt.Errorf("failed to attach uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0 uprobe")
			}
			for _, offset := range returnOffsets {
				l, err := ex.Uprobe(s.Name, t.uprobes["uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0"], &link.UprobeOptions{Address: address, Offset: uint64(offset), PID: int(pid)})
				if err != nil {
					return nil, fmt.Errorf("failed to attach uprobe_ret_Java_sun_nio_ch_FileDispatcherImpl_read0 uprobe")
				}
				klog.Infof("[attach] java symbol offset is  %d", offset)
				links = append(links, l)
			}
		}
	}

	if len(links) == 0 {
		return nil, fmt.Errorf("no links found for %s", bpath)
	}
	klog.WithField("pid", pid).Infof("[attach] libnio attached!")
	return links, nil
}

func (t *Tracer) AttachJavaNetWriteUprobes(pid uint32, rootfs string) ([]link.Link, error) {
	if t.DisableL7Tracing() {
		return nil, nil
	}
	// 关闭横向串联
	if t.DisableE2ETracing() {
		return nil, nil
	}
	var err error
	libnetso := "libnet.so"
	elfFileName := "cwlibnet.so"

	//inject
	originFunc := symbolsocketWrite0

	uProbeData := inject.UprobeData{
		Offset: 53,
		Func:   originFunc,
		//ELFPath: physicalPath,
	}

	if runtime.GOARCH == "arm64" {
		uProbeData = inject.UprobeData{
			Offset: 8,
			Func:   "CW_" + originFunc,
			//ELFPath: physicalPath,
		}
	}
	klog.Infof("[jvm] ProbeData %s", utils.ToString(uProbeData))

	jvmInjector := &inject.JvmInjector{
		Pid: int(pid),
		ReleaseLibNetInfo: inject.LibNetInfo{
			LibName: libnetso,
			//LibPath:     mapsPath,
			//FileDeleted: baseDeleted,
			FuncSymbol: inject.InstInfo{
				SymName: originFunc,
			},
		},
		DebugLibNetInfo: inject.LibNetInfo{
			// TODO 根据版本设置
			LibName: elfFileName,
			//LibPath:     uProbeData.ELFPath,
			//FileDeleted: deleted,
			FuncSymbol: inject.InstInfo{
				SymName: uProbeData.Func,
			},
			//ProcLoadPath: procMapsLoadPath,
		},
		RecodeInfo: inject.LibNetInfo{FuncSymbol: inject.InstInfo{SymName: "CW_RECODE_" + originFunc}},
		Uprobe:     uProbeData,
		Rootfs:     rootfs,
	}
	err = inject.JvmInject(jvmInjector)
	klog.Infof("[jvm] ReleaseLibNetInfo :[%s]", utils.ToString(jvmInjector.ReleaseLibNetInfo))
	klog.Infof("[jvm] DebugLibNetInfo :[%s]", utils.ToString(jvmInjector.DebugLibNetInfo))
	if err != nil {
		klog.WithError(err).Errorf("[jvm] inject.JvmInject error.")
		return nil, err
	}

	var links []link.Link
	ex, err := link.OpenExecutable(jvmInjector.Uprobe.ELFPath)
	if err != nil {
		return nil, err
	}
	opt := link.UprobeOptions{
		Offset: uint64(jvmInjector.Uprobe.Offset),
		PID:    int(pid),
	}
	upread02, err := ex.Uprobe(jvmInjector.Uprobe.Func, t.uprobes["uprobe_Java_java_net_SocketOutputStream_socketWrite0"], &opt)
	if err != nil {
		return nil, err
	}

	links = append(links, upread02)

	if len(links) == 0 {
		return nil, errors.New("can not find uprobe_Java_net_SocketOutputStream_socketWrite0")
	}
	klog.WithField("pid", pid).Infoln("[jvm] libnet has been successfully injected.")
	return links, nil
}

func getCallNextMoveOffsets(machine elf.Machine, instructions []byte) []int {
	var res []int
	firstCall := 0
	switch machine {
	case elf.EM_X86_64:
		for i := 0; i < len(instructions); {
			ins, err := x86asm.Decode(instructions[i:], 64)
			if err == nil && ins.Op == x86asm.CALL {
				if firstCall == 0 {
					firstCall = 1
				} else {
					i += ins.Len
					res = append(res, i)
					return res
				}
			}
			i += ins.Len
		}
	case elf.EM_AARCH64:
		for i := 0; i < len(instructions); {
			ins, err := arm64asm.Decode(instructions[i:])
			if err == nil && ins.Op == arm64asm.BL {
				if firstCall == 0 {
					firstCall = 1
				} else {
					i += 4
					res = append(res, i)
					return res
				}
			}
			i += 4
		}
	}
	return res
}