ebpf/ebpftracer/tls.go

package ebpftracer

import (
	"bufio"
	"bytes"
	"debug/buildinfo"
	"debug/elf"
	"errors"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/cilium/ebpf/link"
	"github.com/coroot/coroot-node-agent/ebpftracer/tracer"
	"github.com/coroot/coroot-node-agent/proc"
	. "github.com/coroot/coroot-node-agent/utils/modelse"
	klog "github.com/sirupsen/logrus"
	"golang.org/x/arch/arm64/arm64asm"
	"golang.org/x/arch/x86/x86asm"
	"golang.org/x/mod/semver"
)

const (
	minSupportedGoVersion = "v1.15.0"
	goTlsWriteSymbol      = "crypto/tls.(*Conn).Write"
	goTlsReadSymbol       = "crypto/tls.(*Conn).Read"
	goExecute             = "runtime.execute"
	goNewproc1            = "runtime.newproc1"
	goRunqget             = "runtime.runqget"
	goServeHTTP           = "net/http.serverHandler.ServeHTTP"
	goTransport           = "net/http.(*Transport).roundTrip"
)

var (
	opensslVersionRe = regexp.MustCompile(`OpenSSL\s(\d\.\d+\.\d+)`)
)

func (t *Tracer) AttachOpenSslUprobes(pid uint32) ([]link.Link, error) {
	if t.DisableL7Tracing() {
		return nil, nil
	}
	libPath, version := getSslLibPathAndVersion(pid)
	if libPath == "" || version == "" {
		return nil, nil
	}

	log := func(msg string, err error) {
		if err != nil {
			for _, s := range []string{"no such file or directory", "no such process", "permission denied"} {
				if strings.HasSuffix(err.Error(), s) {
					return
				}
			}
			klog.Errorf("pid=%d libssl_version=%s: %s: %s", pid, version, msg, err)
			return
		}
		klog.Infof("pid=%d libssl_version=%s: %s", pid, version, msg)
	}

	exe, err := link.OpenExecutable(libPath)
	if err != nil {
		log("failed to open executable", err)
		return nil, err
	}
	var links []link.Link
	writeEnter := "openssl_SSL_write_enter"
	readEnter := "openssl_SSL_read_enter"
	readExEnter := "openssl_SSL_read_ex_enter"
	readExit := "openssl_SSL_read_exit"
	switch {
	case semver.Compare(version, "v3.0.0") >= 0:
		writeEnter = "openssl_SSL_write_enter_v3_0"
		readEnter = "openssl_SSL_read_enter_v3_0"
		readExEnter = "openssl_SSL_read_ex_enter_v3_0"
	case semver.Compare(version, "v1.1.1") >= 0:
		writeEnter = "openssl_SSL_write_enter_v1_1_1"
		readEnter = "openssl_SSL_read_enter_v1_1_1"
		readExEnter = "openssl_SSL_read_ex_enter_v1_1_1"
	}

	type prog struct {
		symbol    string
		uprobe    string
		uretprobe string
	}
	progs := []prog{
		{symbol: "SSL_write", uprobe: writeEnter},
		{symbol: "SSL_read", uprobe: readEnter},
		{symbol: "SSL_read", uretprobe: readExit},
	}
	if semver.Compare(version, "v1.1.1") >= 0 {
		progs = append(progs, []prog{
			{symbol: "SSL_write_ex", uprobe: writeEnter},
			{symbol: "SSL_read_ex", uprobe: readExEnter},
			{symbol: "SSL_read_ex", uretprobe: readExit},
		}...)
	}
	for _, p := range progs {
		if p.uprobe != "" {
			l, err := exe.Uprobe(p.symbol, t.uprobes[p.uprobe], nil)
			klog.Infoln("fucktls crypto/tls uprobes attached", p.symbol)
			if err != nil {
				//log("failed to attach uprobe", err)
				klog.Infoln("fucktls crypto/tls uprobes attached error", p.symbol)
				return nil, err
			}
			links = append(links, l)
		}
		if p.uretprobe != "" {
			klog.Infoln("fucktls crypto/tls uprobes attached ret", p.symbol)
			l, err := exe.Uretprobe(p.symbol, t.uprobes[p.uretprobe], nil)
			if err != nil {
				klog.Infoln("fucktls crypto/tls uprobes attached ret error", p.symbol)
				//log("failed to attach uretprobe", err)
				return nil, err
			}
			links = append(links, l)
		}
	}

	//log("libssl uprobes attached", nil)
	return links, nil
}

func (t *Tracer) AttachGoTlsUprobes(pid uint32, appInfo *AppInfo, codeType uint16) ([]link.Link, error) {
	if t.DisableL7Tracing() {
		return nil, nil
	}

	path := proc.Path(pid, "exe")
	instanceID := appInfo.InstanceIdHash.HashtVal
	appID := appInfo.AppIdHash.HashtVal
	var err error
	var name, version string
	log := func(msg string, err error) {
		if err != nil {
			for _, s := range []string{"not a Go executable", "no such file or directory", "no such process", "permission denied"} {
				if strings.HasSuffix(err.Error(), s) {
					return
				}
			}
			klog.Errorf("pid=%d golang_app=%s golang_version=%s: %s: %s", pid, name, version, msg, err)
			return
		}
		klog.Infof("pid=%d golang_app=%s golang_version=%s: %s", pid, name, version, msg)
	}

	bi, err := buildinfo.ReadFile(path)
	if err != nil {
		log("failed to read build info", err)
		return nil, err
	}
	// isGolangApp = true

	name, err = os.Readlink(path)
	if err != nil {
		log("failed to read name", err)
		return nil, err
	}
	version = strings.Replace(bi.GoVersion, "go", "v", 1)
	if semver.Compare(version, minSupportedGoVersion) < 0 {
		log(fmt.Sprintf("go_versions below %s are not supported", minSupportedGoVersion), nil)
		return nil, err
	}

	ef, err := elf.Open(path)
	if err != nil {
		log("failed to open as elf binary", err)
		return nil, err
	}
	defer ef.Close()

	symbols, err := ef.Symbols()
	if err != nil {
		if errors.Is(err, elf.ErrNoSymbols) {
			log("no symbol section", nil)
			return nil, err
		}
		log("failed to read symbols", err)
		return nil, err
	}

	textSection := ef.Section(".text")
	if textSection == nil {
		log("no text section", nil)
		return nil, err
	}
	textSectionData, err := textSection.Data()
	if err != nil {
		log("failed to read text section", err)
		return nil, err
	}
	textSectionLen := uint64(len(textSectionData) - 1)

	exe, err := link.OpenExecutable(path)
	if err != nil {
		log("failed to open executable", err)
		return nil, err
	}

	offset, ok := tracer.GetOffset(tracer.NewID("std", "runtime", "g", "goid"), path)
	bucketsOff, ok2 := tracer.GetOffset(tracer.NewID("std", "runtime", "hmap", "buckets"), path)

	if ok && ok2 {
		realVersion := strings.Replace(bi.GoVersion, "go", "", 1)
		parts := strings.Split(realVersion, ".")
		var major, minor, revision int
		if len(parts) >= 2 {
			major, err = strconv.Atoi(parts[0])
			if err != nil {
				log("Error converting major version:", err)
				return nil, err
			}
			minor, err = strconv.Atoi(parts[1])
			if err != nil {
				log("Error converting minor version:", err)
				return nil, err
			}
			if len(parts) >= 3 {
				revision, err = strconv.Atoi(parts[2])
				if err != nil {
					log("Error converting revision version:", err)
				}
			}

			goVersion := ((major & 0xFF) << 16) + ((minor & 0xFF) << 8) + min(revision, 255)

			info := EbpfProcInfo{}
			info.Version = uint32(goVersion)
			info.Offsets[OFFSET_IDX_GOID_RUNTIME_G] = uint16(offset)
			info.NetTCPConnItab = uint64(0)
			info.CryptoTLSConnItab = uint64(0)
			info.CredentialsSyscallConnItab = uint64(0)
			info.InstanceId = instanceID
			info.AppId = appID
			info.CodeType = codeType
			// go
			info.BucketsPtrPos = bucketsOff
			fields := map[*uint64]tracer.ID{
				&info.MethodPtrPos:   tracer.NewID("std", "net/http", "Request", "Method"),
				&info.UrlPtrPos:      tracer.NewID("std", "net/http", "Request", "URL"),
				&info.PathPtrPos:     tracer.NewID("std", "net/url", "URL", "Path"),
				&info.StatusCodePos:  tracer.NewID("std", "net/http", "response", "status"),
				&info.RequestHostPos: tracer.NewID("std", "net/http", "Request", "Host"),
				&info.ProtoPos:       tracer.NewID("std", "net/http", "Request", "Proto"),
				&info.CtxPtrPos:      tracer.NewID("std", "net/http", "Request", "ctx"),
				&info.HeadersPtrPos:  tracer.NewID("std", "net/http", "Request", "Header"),
			}

			for field, id := range fields {
				off, ok := tracer.GetOffset(id, path)
				if !ok {
					klog.Warnf("failed to get offset for ID: %v", id)
				}
				*field = off
			}

			// 获取内存地址
			if appInfo.GoProcCache.StartAddr == 0 && appInfo.GoProcCache.EndAddr == 0 {
				allocDetails, allocErr := tracer.Allocate(int(pid))
				if allocErr != nil {
					return nil, allocErr
				}
				if allocDetails != nil {
					//info.StartAddr = allocDetails.StartAddr
					//info.EndAddr = allocDetails.EndAddr
					appInfo.GoProcCache.StartAddr = allocDetails.StartAddr
					appInfo.GoProcCache.EndAddr = allocDetails.EndAddr
				}
			}
			info.StartAddr = appInfo.GoProcCache.StartAddr
			info.EndAddr = appInfo.GoProcCache.EndAddr
			klog.Debugln("Major:", major)
			klog.Debugln("Minor:", minor)
			klog.Debugln("Revision:", revision)
			klog.Debugln("goVersion", goVersion)
			klog.WithField("pid", pid).Debugln("info.StartAddr", info.StartAddr)
			klog.WithField("pid", pid).Debugln("info.EndAddr", info.EndAddr)
			_, err = tracer.UpdateProcInfoToMap(t.collection, pid, info)
			if err != nil {
				klog.Error("failed to update program info", err)
				return nil, err
			}
			appInfo.EBPFProcInfo = &info
		}
	}

	var links []link.Link
	for _, s := range symbols {
		if elf.ST_TYPE(s.Info) != elf.STT_FUNC || s.Size == 0 {
			continue
		}
		switch s.Name {
		case goTlsWriteSymbol, goTlsReadSymbol:
		case goExecute, goNewproc1, goRunqget, goServeHTTP, goTransport:
		default:
			continue
		}
		address := s.Value
		for _, p := range ef.Progs {
			if p.Type != elf.PT_LOAD || (p.Flags&elf.PF_X) == 0 {
				continue
			}

			if p.Vaddr <= s.Value && s.Value < (p.Vaddr+p.Memsz) {
				address = s.Value - p.Vaddr + p.Off
				break
			}
		}
		//fmt.Println("s.Name-----:", s.Name)

		switch s.Name {
		case goExecute:
			l, err := exe.Uprobe(s.Name, t.uprobes["runtime_execute"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach write_enter uprobe", err)
				klog.Infoln("runtime.execute no")
				return nil, err
			} else {
				klog.Infoln("runtime.execute ok")
			}
			links = append(links, l)

		case goNewproc1:
			l, err := exe.Uprobe(s.Name, t.uprobes["enter_runtime_newproc1"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach newproc1 uprobe", err)
				return nil, err
			}
			links = append(links, l)
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				log("failed to attach enter_runtime_newproc1 uprobe", fmt.Errorf("no return offsets found"))
				return nil, err
			}
			for _, offset := range returnOffsets {
				l, err := exe.Uprobe(s.Name, t.uprobes["exit_runtime_newproc1"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
				if err != nil {
					log("failed to attach exit_runtime_newproc1 uprobe", err)
					return nil, err
				}
				links = append(links, l)
			}

		case goRunqget:
			l, err := exe.Uprobe(s.Name, t.uprobes["enter_runtime_runqget"], &link.UprobeOptions{Address: address})
			if err != nil {
				log("failed to attach goRunqget uprobe", err)
				return nil, err
			}
			links = append(links, l)
			//sStart := s.Value - textSection.Addr
			//sEnd := sStart + s.Size
			//if sEnd > textSectionLen {
			//	continue
			//}
			//sBytes := textSectionData[sStart:sEnd]
			//returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			//if len(returnOffsets) == 0 {
			//	log("failed to attach enter_runtime_newproc1 uprobe", fmt.Errorf("no return offsets found"))
			//	return nil
			//}
			//for _, offset := range returnOffsets {
			//	l, err := exe.Uprobe(s.Name, t.uprobes["exit_runtime_newproc1"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
			//	if err != nil {
			//		log("failed to attach exit_runtime_newproc1 uprobe", err)
			//		return nil
			//	}
			//	links = append(links, l)
			//}
		case goServeHTTP:
			l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_HandlerFunc_ServeHTTP"], &link.UprobeOptions{Address: address})
			if err != nil {
				klog.WithError(err).Errorln("failed to attach uprobe_HandlerFunc_ServeHTTP uprobe")
				continue
			}
			klog.Infoln("net/http.serverHandler.ServeHTTP ok")
			links = append(links, l)
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				err = fmt.Errorf("failed to attach uprobe_HandlerFunc_ServeHTTP no return offsets found")
				klog.Errorln(err)
				return nil, err
			}
			for _, offset := range returnOffsets {
				l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_HandlerFunc_ServeHTTP_Returns"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
				if err != nil {
					klog.WithError(err).Errorln(fmt.Errorf("failed to attach exit_runtime_newproc1 uprobe"))
					return nil, err
				}
				links = append(links, l)
			}

		case goTransport:
			if t.DisableE2ETracing() {
				continue
			}
			l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_Transport_roundTrip"], &link.UprobeOptions{Address: address})
			if err != nil {
				klog.WithError(err).Errorln(fmt.Errorf("failed to attach write_enter uprobe"))
				continue
			} else {
			}
			klog.Infoln("net/http.uprobe_Transport_roundTrip ok")

			links = append(links, l)
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				err = fmt.Errorf("failed to attach uprobe_Transport_roundTrip uprobe no return offsets found")
				klog.Errorln(err)
				return nil, err
			}
			for _, offset := range returnOffsets {
				l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_Transport_roundTrip_Returns"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
				if err != nil {
					klog.WithError(err).Errorln("failed to attach exit_runtime_newproc1 uprobe")
					return nil, err
				}
				links = append(links, l)
			}

		case goTlsWriteSymbol:
			klog.Infoln("fucktls goTlsWriteSymbol crypto/tls uprobes attached")
			l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_write_enter"], &link.UprobeOptions{Address: address})
			if err != nil {
				klog.WithError(err).Errorln("failed to attach write_enter uprobe")
				return nil, err
			}
			links = append(links, l)

		case goTlsReadSymbol:
			klog.Infoln("fucktls goTlsReadSymbol crypto/tls uprobes attached")
			l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_read_enter"], &link.UprobeOptions{Address: address})
			if err != nil {
				klog.WithError(err).Errorln("failed to attach read_enter uprobe")
				return nil, err
			}
			links = append(links, l)
			sStart := s.Value - textSection.Addr
			sEnd := sStart + s.Size
			if sEnd > textSectionLen {
				continue
			}
			sBytes := textSectionData[sStart:sEnd]
			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
			if len(returnOffsets) == 0 {
				err = fmt.Errorf("failed to attach read_exit uprobe no return offsets found")
				klog.Errorln(err)
				return nil, err
			}
			for _, offset := range returnOffsets {
				l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_read_exit"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
				if err != nil {
					klog.WithError(err).Errorln("failed to attach read_exit uprobe")
					return nil, err
				}
				links = append(links, l)
			}
		}
	}
	if len(links) == 0 {
		return nil, err
	}
	klog.Infoln("crypto/tls uprobes attached")
	return links, nil
}

func getSslLibPathAndVersion(pid uint32) (string, string) {
	f, err := os.Open(proc.Path(pid, "maps"))
	if err != nil {
		return "", ""
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	scanner.Split(bufio.ScanLines)
	var libsslPath, libcryptoPath string
	for scanner.Scan() {
		parts := strings.Fields(scanner.Text())
		if len(parts) <= 5 {
			continue
		}
		libPath := parts[5]
		switch {
		case libsslPath == "" && strings.Contains(libPath, "libssl.so"):
			fullPath := proc.Path(pid, "root", libPath)
			if _, err = os.Stat(fullPath); err == nil {
				libsslPath = fullPath
			}
		case libcryptoPath == "" && strings.Contains(libPath, "libcrypto.so"):
			fullPath := proc.Path(pid, "root", libPath)
			if _, err = os.Stat(fullPath); err == nil {
				libcryptoPath = fullPath
			}
		default:
			continue
		}
		if libsslPath != "" && libcryptoPath != "" {
			break
		}
	}
	if libsslPath == "" || libcryptoPath == "" {
		return "", ""
	}

	ef, err := elf.Open(libcryptoPath)
	if err != nil {
		return "", ""
	}
	defer ef.Close()
	rodataSection := ef.Section(".rodata")
	if rodataSection == nil {
		return "", ""
	}
	rodataSectionData, err := rodataSection.Data()
	if err != nil {
		return "", ""
	}
	var version string
	for _, b := range bytes.Split(rodataSectionData, []byte("\x00")) {
		if len(b) == 0 {
			continue
		}
		s := string(b)
		if !strings.HasPrefix(s, "OpenSSL") {
			continue
		}
		if m := opensslVersionRe.FindStringSubmatch(s); len(m) > 1 {
			version = m[1]
		}
	}
	return libsslPath, "v" + version
}

func getReturnOffsets(machine elf.Machine, instructions []byte) []int {
	var res []int
	switch machine {
	case elf.EM_X86_64:
		for i := 0; i < len(instructions); {
			ins, err := x86asm.Decode(instructions[i:], 64)
			if err == nil && ins.Op == x86asm.RET {
				res = append(res, i)
			}
			i += ins.Len
		}
	case elf.EM_AARCH64:
		for i := 0; i < len(instructions); {
			ins, err := arm64asm.Decode(instructions[i:])
			if err == nil && ins.Op == arm64asm.RET {
				res = append(res, i)
			}
			i += 4
		}
	}
	return res
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}