ebpf/ebpftracer/tls.go

package ebpftracer

import (
	"bufio"
	"bytes"
	"debug/buildinfo"
	"debug/elf"
	"errors"
	"fmt"
	"github.com/coroot/coroot-node-agent/ebpftracer/tracer"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/cilium/ebpf/link"
	"github.com/coroot/coroot-node-agent/proc"
	"golang.org/x/arch/arm64/arm64asm"
	"golang.org/x/arch/x86/x86asm"
	"golang.org/x/mod/semver"
	"k8s.io/klog/v2"
)

const (
	minSupportedGoVersion = "v1.17.0"
	goTlsWriteSymbol      = "crypto/tls.(*Conn).Write"
	goTlsReadSymbol       = "crypto/tls.(*Conn).Read"
	goExecute             = "runtime.execute"
	goNewproc1            = "runtime.newproc1"
	goServeHTTP           = "net/http.serverHandler.ServeHTTP"
	goTransport           = "net/http.(*Transport).roundTrip"
)

var (
	opensslVersionRe = regexp.MustCompile(`OpenSSL\s(\d\.\d+\.\d+)`)
)

func (t *Tracer) AttachOpenSslUprobes(pid uint32) []link.Link {
	if t.disableL7Tracing {
		return nil
	}
	libPath, version := getSslLibPathAndVersion(pid)
	if libPath == "" || version == "" {
		return nil
	}

	log := func(msg string, err error) {
		if err != nil {
			for _, s := range []string{"no such file or directory", "no such process", "permission denied"} {
				if strings.HasSuffix(err.Error(), s) {
					return
				}
			}
			klog.ErrorfDepth(1, "pid=%d libssl_version=%s: %s: %s", pid, version, msg, err)
			return
		}
		klog.InfofDepth(1, "pid=%d libssl_version=%s: %s", pid, version, msg)
	}

	exe, err := link.OpenExecutable(libPath)
	if err != nil {
		log("failed to open executable", err)
		return nil
	}
	var links []link.Link
	writeEnter := "openssl_SSL_write_enter"
	readEnter := "openssl_SSL_read_enter"
	readExEnter := "openssl_SSL_read_ex_enter"
	readExit := "openssl_SSL_read_exit"
	switch {
	case semver.Compare(version, "v3.0.0") >= 0:
		writeEnter = "openssl_SSL_write_enter_v3_0"
		readEnter = "openssl_SSL_read_enter_v3_0"
		readExEnter = "openssl_SSL_read_ex_enter_v3_0"
	case semver.Compare(version, "v1.1.1") >= 0:
		writeEnter = "openssl_SSL_write_enter_v1_1_1"
		readEnter = "openssl_SSL_read_enter_v1_1_1"
		readExEnter = "openssl_SSL_read_ex_enter_v1_1_1"
	}

	type prog struct {
		symbol    string
		uprobe    string
		uretprobe string
	}
	progs := []prog{
		{symbol: "SSL_write", uprobe: writeEnter},
		{symbol: "SSL_write_ex", uprobe: writeEnter},
		{symbol: "SSL_read", uprobe: readEnter},
		{symbol: "SSL_read_ex", uprobe: readExEnter},
		{symbol: "SSL_read", uretprobe: readExit},
		{symbol: "SSL_read_ex", uretprobe: readExit},
	}
	for _, p := range progs {
		if p.uprobe != "" {
			l, err := exe.Uprobe(p.symbol, t.uprobes[p.uprobe], nil)
			if err != nil {
				log("failed to attach uprobe", err)
				return nil
			}
			links = append(links, l)
		}
		if p.uretprobe != "" {
			l, err := exe.Uretprobe(p.symbol, t.uprobes[p.uretprobe], nil)
			if err != nil {
				log("failed to attach uretprobe", err)
				return nil
			}
			links = append(links, l)
		}
	}

	log("libssl uprobes attached", nil)
	return links
}

// TODO: 核心方法
func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link { // TODO: 核心方法

	fmt.Println("AttachGoTlsUprobes------------pid: ", pid)
	// TODO: 区分语言
	// return t.AttachTlsUprobes4GO(pid)
	return t.AttachTlsUprobes4Java(pid)
}

func (t *Tracer) AttachTlsUprobes4Java(pid uint32) []link.Link {
	fmt.Println("AttachTlsUprobes4Java-------pid: ", pid)
	//if pid != 13096 {
	//	return nil
	//}
	path := proc.Path(pid, "exe")
	fmt.Println("path: ", path)
	exe, err := link.OpenExecutable(path)
	if err != nil {
		fmt.Println("OpenExecutable error: ", err)
		return nil
	}
	var links []link.Link
	// 733f8cc7aaa0 fd0 Lsun/net/www/protocol/http/HttpURLConnection;::setRequestProperty
	// 参考: l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_Transport_roundTrip"], &link.UprobeOptions{Address: address})
	//spec := &ebpf.ProgramSpec{
	//	Name: "uprobe_HttpURLConnection_setRequestProperty",
	//	Type: ebpf.Kprobe,
	//	//// AttachTo: "uprobe_HttpURLConnection_setRequestProperty",
	//	//Instructions: asm.Instructions{
	//	//	asm.Instruction{OpCode: Opcode. },
	//	//},
	//	// Type:    ebpf.TracePoint,
	//	License: "GPL",
	//}
	//println("&ebpf.ProgramSpec: ", spec)
	//prog, err := ebpf.NewProgram(spec) // func NewProgram(spec *ProgramSpec) (*Program, error) {
	//if err != nil {
	//	fmt.Println("NewProgram: ", err)
	//	return nil
	//}

	prog := t.uprobes["uprobe_HttpURLConnection_setRequestProperty"]
	fmt.Println("t.uprobes[\"uprobe_HttpURLConnection_setRequestProperty\"]=", prog)
	// 733f8cc7aaa0 fd0 Lsun/net/www/protocol/http/HttpURLConnection;::setRequestProperty
	// 报错: creating perf_uprobe PMU: token /proc/13096/exe:0x733f8cc7aaa0: opening perf event: invalid argument
	// &link.UprobeOptions{Address: 0x733f8cc7aaa0, Offset: 0xfd0, PID: 13096}
	// 733f8cc7aaa0(16进制) -> 126716782029472(10进制),很进制无关系, 报一样的错误....
	// fd0(十六进制) -> 4048（十进制）
	// 126716782033520 -> 733f8cc7ba70(十六进制）
	// 7ab420b42ee0 978 Lorg/springframework/boot/loader/jar/JarURLConnection;::getInputStream
	//7ab421408920 fb0 Lsun/net/www/protocol/http/HttpURLConnection;::setRequestProperty
	// 4016 134914070583504  7ab4214098d0
	link, err := exe.Uprobe("Lsun/net/www/protocol/http/HttpURLConnection;::setRequestProperty", prog, &link.UprobeOptions{Address: 0x7ab4214098d0})
	if err != nil {
		fmt.Println("exe.Uprobe error: ", err) // 报错: TODO: exe.Uprobe error:  prog cannot be nil: invalid input
		return nil
	}
	links = append(links, link)
	return links
}

func (t *Tracer) AttachTlsUprobes4GO(pid uint32) []link.Link {
	fmt.Println("AttachTlsUprobes4GO------------pid: ", pid)
	if t.disableL7Tracing {
		return nil
	}

	path := proc.Path(pid, "exe")

	var err error
	var name, version string
	log := func(msg string, err error) {
		if err != nil {
			for _, s := range []string{"not a Go executable", "no such file or directory", "no such process", "permission denied"} {
				if strings.HasSuffix(err.Error(), s) {
					return
				}
			}
			klog.ErrorfDepth(1, "pid=%d golang_app=%s golang_version=%s: %s: %s", pid, name, version, msg, err)
			return
		}
		klog.InfofDepth(1, "pid=%d golang_app=%s golang_version=%s: %s", pid, name, version, msg)
	}

	bi, err := buildinfo.ReadFile(path)
	if err != nil {
		log("failed to read build info", err)
		return nil
	}

	name, err = os.Readlink(path)
	if err != nil {
		log("failed to read name", err)
		return nil
	}
	version = strings.Replace(bi.GoVersion, "go", "v", 1)
	if semver.Compare(version, minSupportedGoVersion) < 0 {
		log(fmt.Sprintf("go_versions below %s are not supported", minSupportedGoVersion), nil)
		return nil
	}

	ef, err := elf.Open(path) // TODO: 核心逻辑
	if err != nil {
		log("failed to open as elf binary", err)
		return nil
	}
	defer ef.Close()

	// TODO: 判断是java进程还是go进程...
	// var lan := "x"

	symbols, err := ef.Symbols()
	// fmt.Println("--------------AttachGoTlsUprobes---------------symbols------", symbols)
	if err != nil {
		if errors.Is(err, elf.ErrNoSymbols) {
			log("no symbol section", nil)
			return nil
		}
		log("failed to read symbols", err)
		return nil
	}

	textSection := ef.Section(".text")
	if textSection == nil {
		log("no text section", nil)
		return nil
	}
	textSectionData, err := textSection.Data()
	if err != nil {
		log("failed to read text section", err)
		return nil
	}
	textSectionLen := uint64(len(textSectionData) - 1)

	exe, err := link.OpenExecutable(path)
	if err != nil {
		log("failed to open executable", err)
		return nil
	}

	offset, ok := tracer.GetOffset(tracer.NewID("std", "runtime", "g", "goid"), path)
	fmt.Println(offset, ok, version)
	if ok {
		realVersion := strings.Replace(bi.GoVersion, "go", "", 1)
		parts := strings.Split(realVersion, ".")
		var major, minor, revision int
		if len(parts) >= 3 {
			major, err = strconv.Atoi(parts[0])
			if err != nil {
				log("Error converting major version:", err)
			}
			minor, err = strconv.Atoi(parts[1])
			if err != nil {
				log("Error converting minor version:", err)
			}
			revision, err = strconv.Atoi(parts[2])
			if err != nil {
				log("Error converting revision version:", err)
			}
			goVersion := ((major & 0xFF) << 16) + ((minor & 0xFF) << 8) + min(revision, 255)

			klog.Infoln("Major:", major)
			klog.Infoln("Minor:", minor)
			klog.Infoln("Revision:", revision)
			klog.Infoln("goVersion", goVersion)
			info := tracer.EbpfProcInfo{}
			info.Version = uint32(goVersion)
			info.Offsets[tracer.OFFSET_IDX_GOID_RUNTIME_G] = uint16(offset)
			info.NetTCPConnItab = uint64(0)
			info.CryptoTLSConnItab = uint64(0)
			info.CredentialsSyscallConnItab = uint64(0)
			_, err = tracer.UpdateProcInfoToMap(t.collection, pid, info)
			if err != nil {
				klog.Error("failed to update program info", err)
			}
		}
	}

	var links []link.Link
	for _, s := range symbols { // TODO: 遍历的方式找Func的Offset?
		if elf.ST_TYPE(s.Info) != elf.STT_FUNC || s.Size == 0 {
			continue
		}
		switch s.Name {
		//case goTlsWriteSymbol, goTlsReadSymbol:
		case goExecute, goNewproc1, goServeHTTP, goTransport:
		default:
			continue
		}
		address := s.Value
		for _, p := range ef.Progs {
			if p.Type != elf.PT_LOAD || (p.Flags&elf.PF_X) == 0 {
				continue
			}

			if p.Vaddr <= s.Value && s.Value < (p.Vaddr+p.Memsz) {
				address = s.Value - p.Vaddr + p.Off
				break
			}
		}
		fmt.Println("s.Name-----:", s.Name) // TODO: s.Name-----: runtime.execute | net/http.serverHandler.ServeHTTP
		fmt.Println("s.Value-----:", s.Value)
		switch s.Name {
		case goExecute:
			l, err := exe.Uprobe(s.Name, t.uprobes["runtime_execute"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach write_enter uprobe", err)
				klog.Infoln("runtime.execute no")
				return nil
			} else {
				klog.Infoln("runtime.execute ok")
			}
			links = append(links, l)

		case goNewproc1:
			l, err := exe.Uprobe(s.Name, t.uprobes["enter_runtime_newproc1"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach newproc1 uprobe", err)
				return nil
			}
			links = append(links, l)
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				log("failed to attach enter_runtime_newproc1 uprobe", fmt.Errorf("no return offsets found"))
				return nil
			}
			for _, offset := range returnOffsets {
				l, err := exe.Uprobe(s.Name, t.uprobes["exit_runtime_newproc1"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
				if err != nil {
					log("failed to attach exit_runtime_newproc1 uprobe", err)
					return nil
				}
				links = append(links, l)
			}
		case goServeHTTP:
			l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_HandlerFunc_ServeHTTP"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach write_enter uprobe", err)
				fmt.Println("net/http.serverHandler.ServeHTTP no")
				fmt.Println(err)
				continue
			} else {
				fmt.Println("net/http.serverHandler.ServeHTTP ok")
			}
			links = append(links, l)
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				log("failed to attach uprobe_HandlerFunc_ServeHTTP uprobe", fmt.Errorf("no return offsets found"))
				return nil
			}
			for _, offset := range returnOffsets {
				l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_HandlerFunc_ServeHTTP_Returns"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
				if err != nil {
					log("failed to attach exit_runtime_newproc1 uprobe", err)
					return nil
				}
				links = append(links, l)
			}

		case goTransport:
			l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_Transport_roundTrip"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach write_enter uprobe", err)
				fmt.Println("net/http.uprobe_Transport_roundTrip no")
				fmt.Println(err)
				continue
			} else {
				fmt.Println("net/http.uprobe_Transport_roundTrip ok")
			}
			links = append(links, l)
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				log("failed to attach uprobe_Transport_roundTrip uprobe", fmt.Errorf("no return offsets found"))
				return nil
			}
			for _, offset := range returnOffsets {
				l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_Transport_roundTrip_Returns"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
				if err != nil {
					log("failed to attach exit_runtime_newproc1 uprobe", err)
					return nil
				}
				links = append(links, l)
			}

		case goTlsWriteSymbol:
			l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_write_enter"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach write_enter uprobe", err)
				return nil
			}
			links = append(links, l)
		case goTlsReadSymbol:
			l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_read_enter"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach read_enter uprobe", err)
				return nil
			}
			links = append(links, l)
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				log("failed to attach read_exit uprobe", fmt.Errorf("no return offsets found"))
				return nil
			}
			for _, offset := range returnOffsets {
				l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_read_exit"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
				if err != nil {
					log("failed to attach read_exit uprobe", err)
					return nil
				}
				links = append(links, l)
			}
		}
	}
	if len(links) == 0 {
		return nil
	}
	log("crypto/tls uprobes attached", nil)
	return links
}

func getSslLibPathAndVersion(pid uint32) (string, string) {
	f, err := os.Open(proc.Path(pid, "maps"))
	if err != nil {
		return "", ""
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	scanner.Split(bufio.ScanLines)
	var libsslPath, libcryptoPath string
	for scanner.Scan() {
		parts := strings.Fields(scanner.Text())
		if len(parts) <= 5 {
			continue
		}
		libPath := parts[5]
		switch {
		case libsslPath == "" && strings.Contains(libPath, "libssl.so"):
			fullPath := proc.Path(pid, "root", libPath)
			if _, err = os.Stat(fullPath); err == nil {
				libsslPath = fullPath
			}
		case libcryptoPath == "" && strings.Contains(libPath, "libcrypto.so"):
			fullPath := proc.Path(pid, "root", libPath)
			if _, err = os.Stat(fullPath); err == nil {
				libcryptoPath = fullPath
			}
		default:
			continue
		}
		if libsslPath != "" && libcryptoPath != "" {
			break
		}
	}
	if libsslPath == "" || libcryptoPath == "" {
		return "", ""
	}

	ef, err := elf.Open(libcryptoPath)
	if err != nil {
		return "", ""
	}
	defer ef.Close()
	rodataSection := ef.Section(".rodata")
	if rodataSection == nil {
		return "", ""
	}
	rodataSectionData, err := rodataSection.Data()
	if err != nil {
		return "", ""
	}
	var version string
	for _, b := range bytes.Split(rodataSectionData, []byte("\x00")) {
		if len(b) == 0 {
			continue
		}
		s := string(b)
		if !strings.HasPrefix(s, "OpenSSL") {
			continue
		}
		if m := opensslVersionRe.FindStringSubmatch(s); len(m) > 1 {
			version = m[1]
		}
	}
	return libsslPath, "v" + version
}

func getReturnOffsets(machine elf.Machine, instructions []byte) []int {
	var res []int
	switch machine {
	case elf.EM_X86_64:
		for i := 0; i < len(instructions); {
			ins, err := x86asm.Decode(instructions[i:], 64)
			if err == nil && ins.Op == x86asm.RET {
				res = append(res, i)
			}
			i += ins.Len
		}
	case elf.EM_AARCH64:
		for i := 0; i < len(instructions); {
			ins, err := arm64asm.Decode(instructions[i:])
			if err == nil && ins.Op == arm64asm.RET {
				res = append(res, i)
			}
			i += 4
		}
	}
	return res
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}