Fixed #TASK_GK-2944 支持 JAVA Aot

containers/container_apm.go

@@ -1,10 +1,13 @@
 package containers
 
 import (
+	"bufio"
 	"debug/elf"
 	"fmt"
-	"math/rand"
+	"os"
 	"sort"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/coroot/coroot-node-agent/ebpftracer"
@@ -215,92 +218,16 @@ func (c *Container) StackProcess(event ebpftracer.StackEvent, tracer *ebpftracer
 			return
 		}
 
-		fmt.Println("StackProcess 函数入口开始处理 fun:", event.TraceId, uprobe.Funcname, event.TimeNsEnd-event.TimeNsStart)
+		fmt.Printf("StackProcess 函数入口开始处理 fun:TraceId:%lld, Funcname:%s, time: %lld\n", event.TraceId, uprobe.Funcname, event.TimeNsEnd-event.TimeNsStart)
 		apmTrace, ok := c.getTrace(event.TraceId)
 		if ok {
 			fmt.Println("append FuncTraceQuery fun:", event.TraceId, uprobe.Funcname, event.Pid)
 			duration := event.TimeNsEnd - event.TimeNsStart
-			apmTrace.FuncTraceQuery(uprobe.Funcname, time.Duration(duration), int(event.Level), int(event.Fpid), int(event.Nid))
+			apmTrace.FuncTraceQuery(uprobe.Funcname, time.Duration(duration), event.TimeNsStart, event.TimeNsEnd)
 		}
 	}
 }
 
-func (c *Container) StackProcessBak(event ebpftracer.StackEvent, tracer *ebpftracer.Tracer) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	// get the associated uprobe
-	uprobe, err := c.GetUprobe(event, tracer)
-	if err != nil {
-		//fmt.Println("GetUprobeGetUprobe errer: %v", err)
-		// log.Errorf("failed to get uprobe for event %+v: %+v", event, err)
-		return
-	}
-
-	if event.TraceId <= 0 {
-		//fmt.Println("StackProcess TraceId id 0")
-		// log.Errorf("failed to get uprobe for event %+v: %+v", event, err)
-		return
-	}
-
-	length := len(c.goEventStacks[event.TraceId])
-	if length <= 0 {
-		c.goEventStacks = map[uint64]map[uint64][]ebpftracer.StackFunEvent{}
-		c.goEventStacks[event.TraceId] = map[uint64][]ebpftracer.StackFunEvent{}
-		c.goEventStacks[event.TraceId][event.Goid] = []ebpftracer.StackFunEvent{}
-	}
-
-	switch event.Location {
-	case 0: // entry
-		level := 100
-		pid := 100000 + event.Goid
-		length := len(c.goEventStacks[event.TraceId][event.Goid])
-		//fmt.Println("StackProcess 函数入口开始处理 fun:", event.TraceId, uprobe.Funcname, length)
-		if length > 0 {
-			funEvent := c.goEventStacks[event.TraceId][event.Goid][length-1]
-			//fmt.Println("funEvent goEventStacks fun:", event.TraceId, funEvent.Uprobe.Funcname, funEvent.Nid, funEvent.Level)
-			lastEvent := funEvent.StackEvent
-			if lastEvent.Location == event.Location && lastEvent.Ip == event.Ip && lastEvent.Bp != event.CallerBp {
-				// duplicated entry event due to stack expansion/shrinkage
-				// log.Debugf("duplicated entry event: %+v", event)
-				//fmt.Println("GetUprobeGetUprobe duplicated entry event: %+v", event)
-				c.goEventStacks[event.TraceId][event.Goid][length-1].StackEvent = event
-				return
-			}
-			level = int(funEvent.Level)
-			pid = uint64(funEvent.Nid)
-		}
-		rand.Seed(time.Now().UnixNano())
-		// append new event
-		//fmt.Println("append goEventStacks fun:", event.TraceId, uprobe.Funcname, pid, level+1)
-		c.goEventStacks[event.TraceId][event.Goid] = append(c.goEventStacks[event.TraceId][event.Goid], ebpftracer.StackFunEvent{
-			StackEvent: event,
-			Uprobe:     &uprobe,
-			Level:      level + 1,
-			Pid:        int(pid),
-			Nid:        rand.Intn(100000000),
-		})
-		length = len(c.goEventStacks[event.TraceId][event.Goid])
-		//fmt.Println("append goEventStacks end:", event.TraceId, uprobe.Funcname, pid, level+1, length)
-	case 1: // ret
-		//// fmt.Println("StackProcess 函数出口开始处理 fun:", event.TraceId, uprobe.Funcname)
-		length := len(c.goEventStacks[event.TraceId][event.Goid])
-		//fmt.Println("StackProcess 函数出口开始处理 fun:", event.TraceId, uprobe.Funcname, length)
-		if length > 0 {
-			funEvent := c.goEventStacks[event.TraceId][event.Goid][length-1]
-			entFun := funEvent.StackEvent
-			apmTrace, ok := c.getTrace(event.TraceId)
-			//fmt.Println("StackProcess 函数出口处理 fun:", event.TraceId, funEvent.Uprobe.Funcname, length)
-			if ok {
-				//fmt.Println("append FuncTraceQuery fun:", event.TraceId, uprobe.Funcname, funEvent.Pid, funEvent.Level, funEvent.Nid)
-				duration := event.TimeNsEnd - entFun.TimeNsStart
-				c.goEventStacks[event.TraceId][event.Goid] = c.goEventStacks[event.TraceId][event.Goid][:length-1]
-				apmTrace.FuncTraceQuery(funEvent.Uprobe.Funcname, time.Duration(duration), funEvent.Level, funEvent.Pid, funEvent.Nid)
-			}
-		}
-	}
-
-}
-
 // ResolveAddress returns the symbol(s) and offset of the given address.
 func (c *Container) ResolveAddress(addr uint64, symbols []elf.Symbol) (syms []elf.Symbol, offset uint, err error) {
 	if addr == 0 {
@@ -329,15 +256,72 @@ func (c *Container) ResolveAddress(addr uint64, symbols []elf.Symbol) (syms []el
 	return syms, uint(addr - sym.Value), nil
 }
 
+type MemoryMap struct {
+	Start, End uint64
+}
+
+// ReadFirstLineOfMapsFile reads the first line of /proc/<pid>/maps file and return the memory map as a MemoryMap struct
+func ReadFirstLineOfMapsFile(pid string) (*MemoryMap, error) {
+	file, err := os.Open(fmt.Sprintf("/proc/%s/maps", pid))
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	if scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		addresses := strings.Split(fields[0], "-")
+		if len(addresses) != 2 {
+			return nil, errors.New("unexpected format in /proc/<pid>/maps")
+		}
+
+		start, err := strconv.ParseUint(addresses[0], 16, 64)
+		if err != nil {
+			return nil, err
+		}
+
+		end, err := strconv.ParseUint(addresses[1], 16, 64)
+		if err != nil {
+			return nil, err
+		}
+
+		return &MemoryMap{
+			Start: start,
+			End:   end,
+		}, nil
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return nil, errors.New("empty /proc/<pid>/maps")
+}
+
 func (c *Container) GetUprobe(event ebpftracer.StackEvent, tracer *ebpftracer.Tracer) (uprobe tracer.Uprobe, err error) {
 	//fmt.Println("GetUprobe entory:")
+
+	memoryMap, _ := ReadFirstLineOfMapsFile(strconv.Itoa(int(event.Pid)))
+	Address := event.Ip - memoryMap.Start
+	fmt.Printf("memoryMap.Start: %x, event.Ip: %x, Address: %x\n", memoryMap.Start, event.Ip, Address)
+
+	for _, fun := range tracer.UprobesMap {
+		// funAddress := fun.Address + fun.AbsOffset
+		// fmt.Printf("GetUprobeGetUprobeGetUprobe:fun.Address %x, fun.AbsOffset: %x\n", fun.Address, fun.AbsOffset)
+		if fun.Address == Address {
+			// fmt.Printf("---GetUprobeGetUprobeGetUprobe: %x, event.Ip: %x\n", memoryMap.Start, event.Ip)
+			return fun, nil
+		}
+	}
+
 	syms, _, err := c.ResolveAddress(event.Ip, tracer.Symbols)
 	if err != nil {
 		return
 	}
 	for _, sym := range syms {
 		//fmt.Println("GetUprobeGetUprobeGetUprobe: %s+%d", sym.Name, offset)
-		uprobe, ok := tracer.UprobesMap[fmt.Sprintf("%s", sym.Name)]
+		uprobe, ok := tracer.UprobesMap[fmt.Sprintf("%s-%s", sym.Name, sym.Value)]
 		if ok {
 			return uprobe, nil
 		}

containers/registry.go

@@ -300,12 +300,12 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 				}
 
 				if c := r.containersByPid[uint32(e.StackEvent.Pid)]; c != nil {
-					fmt.Println("e.EventTypeFunEnt: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
-					fmt.Println("e.EventTypeFunEnt: FPid:%d, Nid:%d, Level:%d", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
+					fmt.Printf("e.EventTypeFunEnt: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x\n", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
+					fmt.Printf("e.EventTypeFunEnt: FPid:%x, Nid:%x, Level:%d\n", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
 					c.StackProcess(*e.StackEvent, r.tracer)
 				} else {
-					fmt.Println("e.EventTypeFunEnt ErrorError: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
-					fmt.Println("e.EventTypeFunEnt ErrorError: TraceId:%d, FPid:%d, Nid:%d, Level:%d", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
+					fmt.Printf("e.EventTypeFunEnt ErrorError: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
+					fmt.Printf("e.EventTypeFunEnt ErrorError: TraceId:%x, FPid:%x, Nid:%x, Level:%d\n", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
 				}
 			}
 		}

ebpftracer/ebpf/utrace/go/net/stack.probe.bpf.c

@@ -100,7 +100,7 @@ int ent(struct pt_regs *ctx)
 	e->caller_bp = PT_REGS_FP(ctx); // 父函数的基地址
 
 	void *ra;
-	ra = (void *)PT_REGS_FP(ctx); // 父函数的 sp，此时 sp 指向父函数的最后一个地址
+	ra = (void *)PT_REGS_SP(ctx); // 父函数的 sp，此时 sp 指向父函数的最后一个地址
 	bpf_probe_read_user(&e->caller_ip, sizeof(e->caller_ip), ra);
 
 	cw_bpf_debug("[Go] [uprobe/ent]: goid: %llu", e->goid);
@@ -111,6 +111,9 @@ int ent(struct pt_regs *ctx)
 	struct trace_stack_entry_key_t trace_key = {};
 	trace_key.caller_bp = e->caller_bp % 0x800 + (e->goid << 12);
 	trace_key.bp = e->bp % 0x800 + (e->goid << 12);
+
+	// trace_key.caller_bp = e->caller_bp;
+	// trace_key.bp = e->bp;
 	// cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: bp: %x", trace_key.bp);
 	cw_bpf_debug("[Go] [uprobe/ent]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x", trace_key.caller_bp, trace_key.bp);
 
@@ -167,17 +170,20 @@ int ret(struct pt_regs *ctx)
 	e->location = RETPOINT;
 	e->ip = PT_REGS_IP(ctx);
 	e->time_ns_end = bpf_ktime_get_ns();
-	e->bp = PT_REGS_FP(ctx);
+	e->bp = PT_REGS_SP(ctx) - 8;
 
-	__u64 caller_bp = PT_REGS_SP(ctx) - 8;
-	cw_bpf_debug("[Go] [uprobe/ret]: e->ip:%x,bp:%x", e->ip, e->bp);
-	cw_bpf_debug("[Go] [uprobe/ret]: goid: %llu, goid:0x:%x", e->goid, e->goid);
+	__u64 caller_bp = PT_REGS_FP(ctx);
+	cw_bpf_debug("[Go] [uprobe/ret]: e->ip:%x,bp:%x,caller_bp:%x", e->ip, e->bp, caller_bp);
+	cw_bpf_debug("[Go] [uprobe/ret]: sp:%x, goid: %llu, goid:0x:%x", PT_REGS_SP(ctx), e->goid, e->goid);
 
 	cw_bpf_debug("[Go] [uprobe/ret]: event: location:%x,ip:%x,time_ns_end:%lld\n", e->location, e->ip, e->time_ns_end);
 
 	struct trace_stack_entry_key_t trace_key = {};
-	trace_key.bp = caller_bp % 0x800 + (e->goid << 12);
-	trace_key.caller_bp = e->bp % 0x800 + (e->goid << 12);
+	trace_key.caller_bp = caller_bp % 0x800 + (e->goid << 12);
+	trace_key.bp = e->bp % 0x800 + (e->goid << 12);
+
+	// trace_key.caller_bp = caller_bp;
+	// trace_key.bp = e->bp;
 
 	cw_bpf_debug("[Go] [uprobe/ret]: trace_keytrace_keytrace_key: caller_bp: %x, bp: %x", trace_key.caller_bp, trace_key.bp);
 	// cw_bpf_debug("[Go] [uprobe/ret]: trace_keytrace_keytrace_key: bp: %x", trace_key.bp);
@@ -204,7 +210,3 @@ int ret(struct pt_regs *ctx)
 	cw_bpf_debug("[Go] [uprobe/ret] end");
 	return 1;
 }
-
-
-
-

ebpftracer/stack.go

@@ -2,6 +2,7 @@ package ebpftracer
 
 import (
 	"context"
+	"debug/dwarf"
 	"debug/elf"
 	debugelf "debug/elf"
 	"fmt"
@@ -19,44 +20,209 @@ import (
 	"golang.org/x/sync/semaphore"
 )
 
+type uprobesDef struct {
+	Name       string
+	Offset     uint64
+	EntAddress uint64
+	RetAddress uint64
+}
+
 func (t *Tracer) stack() error {
-	// uprobes := []tracer.Uprobe{}
 	if t.disableL7Tracing {
 		return nil
 	}
 
+	binType := "dotnet"
+	MatchString := ".*HandleFunc|.*main.*|testfun.*|.*serverHandler.*|.*ServeHTTP.*"
+	dbgpath := ""
+
 	ENV_PID := os.Getenv("FILTER_PID")
 	WHITE_LIST := os.Getenv("WHITE_LIST")
+	BIN_TYPE := os.Getenv("BIN_TYPE")
+	DBG_PATH := os.Getenv("DBG_PATH")
+
 	if ENV_PID == "" {
 		return nil
 	}
 
-	MatchString := ".*HandleFunc|.*main.*|testfun.*|.*serverHandler.*|.*ServeHTTP.*"
-
 	if WHITE_LIST != "" {
 		MatchString = WHITE_LIST
 	}
 
-	fmt.Println("UprobesMatchString::: init", MatchString)
+	if DBG_PATH != "" {
+		dbgpath = DBG_PATH
+	}
+
+	if BIN_TYPE != "" {
+		binType = BIN_TYPE
+	}
+
+	fmt.Println("UprobesMatchString:::init", MatchString)
 
 	pid, _ := strconv.ParseInt(ENV_PID, 10, 32)
 	path := proc.Path(uint32(pid), "exe")
 
-	t.Uprobes, _ = t.getUprobes(path, MatchString)
+	if dbgpath != "" {
+		t.Uprobes, _ = t.getJavaAOTUprobes(binType, path, dbgpath, MatchString)
+	} else {
+		t.Uprobes, _ = t.getUprobes(path, MatchString)
+	}
+
 	t.UprobesMap = map[string]tracer.Uprobe{}
-	fmt.Println("UprobesMap::: init")
+	fmt.Println("UprobesMap:::init")
 	for _, up := range t.Uprobes {
 		fmt.Println("UprobesMap:::", up.Funcname)
-		t.UprobesMap[fmt.Sprintf("%s", up.Funcname)] = up
+		t.UprobesMap[fmt.Sprintf("%s-%s", up.Funcname, up.Address)] = up
 	}
 	links := t.attachUprobes(path, t.Uprobes)
 	t.links = append(t.links, links...)
 
 	// defer t.detachUprobes(links)
-
 	return nil
 }
 
+func (t *Tracer) getJavaAOTUprobes(binType, path string, dbgpath string, MatchString string) ([]tracer.Uprobe, error) {
+	uprobes := []tracer.Uprobe{}
+
+	elfFile, err := elf.Open(path)
+
+	funSection := ".text"
+
+	if binType == "dotnet" {
+		funSection = "__managedcode"
+	}
+
+	textSection := elfFile.Section(funSection)
+	if textSection == nil {
+		fmt.Println("no text section", nil)
+		return nil, nil
+	}
+	textSectionData, err := textSection.Data()
+	if err != nil {
+		fmt.Println("failed to read text section", err)
+		return nil, nil
+	}
+	textSectionLen := uint64(len(textSectionData) - 1)
+
+	dwarfFile, err := elf.Open(dbgpath)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	dwarfData, err := dwarfFile.DWARF()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	entryReader := dwarfData.Reader()
+
+	// var targetAddress uint64
+
+	listEntry := make(map[dwarf.Offset]uprobesDef)
+	SpecListEntry := []dwarf.Entry{}
+
+	for {
+		entry, err := entryReader.Next()
+		if err == io.EOF {
+			// We've reached the end of DWARF entries
+			break
+		}
+		if err != nil {
+			log.Fatalf("Error reading entry: %v", err)
+		}
+		if entry == nil {
+			log.Println("Warning: a nil entry was returned with no error")
+			break
+		}
+		if entry.Tag == dwarf.TagSubprogram {
+			// fmt.Printf("entry address: %x, %d\n", entry.Offset, entry.Children)
+			funName, _ := entry.Val(dwarf.AttrName).(string)
+			found, _ := regexp.MatchString(MatchString, funName)
+			if found {
+				entAddress, _ := entry.Val(dwarf.AttrLowpc).(uint64)
+				retAddress, _ := entry.Val(dwarf.AttrHighpc).(uint64)
+				// fmt.Printf("Function %s address: %x, %x\n", funName, address, entry.Offset)
+				uprobes := uprobesDef{}
+				uprobes.EntAddress = entAddress
+				uprobes.RetAddress = retAddress
+				uprobes.Offset = uint64(entry.Offset)
+				uprobes.Name = funName
+
+				listEntry[entry.Offset] = uprobes
+			}
+
+			specAddr, _ := entry.Val(dwarf.AttrSpecification).(dwarf.Offset)
+			lowpc := entry.Val(dwarf.AttrLowpc)
+			if lowpc != nil && specAddr > 0 && lowpc.(uint64) > 0 {
+				// fmt.Printf("AttrSpecification address: %x, %x\n", specAddr, entry.Offset)
+				SpecListEntry = append(SpecListEntry, *entry)
+			}
+		}
+	}
+
+	for _, v := range SpecListEntry {
+		specAddr, _ := v.Val(dwarf.AttrSpecification).(dwarf.Offset)
+		// fmt.Printf("SpecListEntrySpecListEntrySpecListEntry Attach Function: %x\n", specAddr)
+		_, ok := listEntry[specAddr]
+		if ok {
+			vv := listEntry[specAddr]
+			entAddr := v.Val(dwarf.AttrLowpc)
+			if entAddr != nil {
+				vv.EntAddress = entAddr.(uint64)
+			}
+			retAddr := v.Val(dwarf.AttrHighpc)
+			if retAddr != nil {
+				switch retAddr.(type) {
+				case uint64:
+					vv.RetAddress = uint64(retAddr.(uint64))
+				case int64:
+					vv.RetAddress = uint64(retAddr.(int64))
+				default:
+					fmt.Println("Unknown type")
+				}
+			}
+			listEntry[specAddr] = vv
+		}
+	}
+
+	for _, v := range listEntry {
+		fmt.Printf("Need Attach Function %s address: %x, %x\n", v.Name, v.EntAddress, v.RetAddress)
+
+		// 函数入口加入待 attach 列表
+		uprobes = append(uprobes, tracer.Uprobe{
+			Funcname:  v.Name,         // 函数名
+			Location:  tracer.AtEntry, // 入口
+			Address:   v.EntAddress,   // 函数地址
+			AbsOffset: 0,              // 函数相对 ELF 偏移
+			RelOffset: 0,              // 函数真实偏移
+		})
+
+		sStart := v.EntAddress - textSection.Addr
+		sSize := v.RetAddress
+		if v.RetAddress > v.EntAddress {
+			sSize = v.RetAddress - v.EntAddress
+		}
+		sEnd := sStart + sSize
+		if sEnd > textSectionLen {
+			continue
+		}
+		sBytes := textSectionData[sStart:sEnd]
+		returnOffsets := getReturnOffsets(elfFile.Machine, sBytes)
+
+		for _, offset := range returnOffsets {
+			uprobes = append(uprobes, tracer.Uprobe{
+				Funcname:  v.Name,
+				Location:  tracer.AtRet,
+				Address:   v.EntAddress,
+				AbsOffset: uint64(offset),
+				RelOffset: 0,
+			})
+		}
+	}
+	return uprobes, nil
+}
+
 func (t *Tracer) getUprobes(path string, MatchString string) ([]tracer.Uprobe, error) {
 	uprobes := []tracer.Uprobe{}
 
@@ -109,7 +275,7 @@ func (t *Tracer) getUprobes(path string, MatchString string) ([]tracer.Uprobe, e
 
 		if found {
 			// 匹配到了加入 attachFuncs 列表
-			fmt.Println("Fuck This: ", symbol.Name)
+			fmt.Printf("Fuck This: %s, %x\n", symbol.Name, symbol.Value)
 			// attachFuncs = append(attachFuncs, symbol.Name)
 			// 根据函数名拿到当前函数的符号结构体
 			sym := symnames[symbol.Name]
@@ -169,14 +335,10 @@ func (t *Tracer) attachUprobes(path string, uprobes []tracer.Uprobe) []link.Link
 	if err != nil {
 		return nil
 	}
-	fmt.Println("AttachAttachAttach", path)
-
-	for i, up := range t.uprobes {
-		fmt.Println("attachingERROR---", i, up)
-	}
+	fmt.Println("Attach Start", path)
 
 	for i, up := range uprobes {
-		fmt.Printf("attaching %d -> %d -> %s -> 0x%x -> 0x%x\n", i, len(uprobes), up.Funcname, up.AbsOffset, up.Address)
+		fmt.Printf("attaching %d -> %d -> %s -> 0x%x -> 0x%x -> 0x%x\n", i, len(uprobes), up.Funcname, up.AbsOffset, up.Address, up.AbsOffset+up.Address)
 		var prog *ebpf.Program
 		switch up.Location {
 		case tracer.AtEntry:
@@ -184,13 +346,12 @@ func (t *Tracer) attachUprobes(path string, uprobes []tracer.Uprobe) []link.Link
 		case tracer.AtRet:
 			prog = t.uprobes["ret"]
 		}
-		fmt.Println("progprogprogprogprogprog---", prog)
-		up, err := ex.Uprobe(up.Funcname, prog, &link.UprobeOptions{Address: up.Address, Offset: up.AbsOffset})
+		uplink, err := ex.Uprobe(up.Funcname, prog, &link.UprobeOptions{Address: up.Address, Offset: up.AbsOffset})
 		if err != nil {
-			fmt.Println("attachingERROR", err)
+			fmt.Printf("attachingERROR:%v, %v, %v\n", err, up, uplink)
 			// return nil
 		} else {
-			links = append(links, up)
+			links = append(links, uplink)
 		}
 	}
 	return links

pkg/go.opentelemetry.io/otel/exporters/otlp/otlptrace/apm_exporter.go

@@ -4,6 +4,7 @@ import (
 	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -94,6 +95,7 @@ type MapInfoT struct {
 	ServiceName    string   `json:"service_name"`
 	ServiceType    string   `json:"service_type"`
 	StartTime      uint64   `json:"start_time"`
+	EndTime        uint64   `json:"end_time"`
 	WallTime       uint64   `json:"wall_time"`
 	Schema         string   `json:"schema,omitempty"`
 	AssumedAppId   int64    `json:"assumed_app_id,omitempty"`
@@ -160,44 +162,70 @@ func tracetransformData(sdl []tracesdk.ReadOnlySpan) []RootDataT {
 	fmt.Println(len(sdl))
 	return sendData
 }
-func buildLevelRecursion(parentMap map[int]map[int]*MapInfoT, pid int, level int) {
-	for i, v := range parentMap[pid] {
-		parentMap[pid][i].Level = level
-		// fmt.Printf("buildLevelRecursion: %x, %x\n", v.Nid, v.Nid%0x1000000)
-		buildLevelRecursion(parentMap, v.Nid%0x1000000, level+1)
-	}
+
+type TimeMap struct {
+	Time uint64
+	Type int
+	Map  *MapInfoT
 }
 
 func buildLevel(sdl *TraceMapT) {
-	parentMap := make(map[int]map[int]*MapInfoT)
 	nidMap := make(map[int]*MapInfoT)
 
+	mapSlice := []TimeMap{}
+
 	for i, v := range sdl.RootData.Maps {
-		nidMap[v.Nid%0x1000000] = &sdl.RootData.Maps[i]
-	}
-	// 没有 pid 的放到 application 上
-	for i, v := range sdl.RootData.Maps {
-		_, ok := nidMap[v.Pid]
-		if !ok && v.Pid != 0 {
-			sdl.RootData.Maps[i].Pid = 1
+		if v.ServiceType == "APPLICATION" {
+			continue
 		}
+		nidMap[v.Nid] = &sdl.RootData.Maps[i]
+		timeStartMap := TimeMap{
+			Time: v.StartTime,
+			Type: 0,
+			Map:  &sdl.RootData.Maps[i],
+		}
+		mapSlice = append(mapSlice, timeStartMap)
+		timeEndMap := TimeMap{
+			Time: v.EndTime,
+			Type: 1,
+			Map:  &sdl.RootData.Maps[i],
+		}
+		mapSlice = append(mapSlice, timeEndMap)
 	}
-	// fmt.Println("nidMapnidMapnidMapnidMap--------------")
-	// fmt.Println(nidMap)
-	// fmt.Println(sdl.RootData.Maps)
-	// 构建 pid 映射map
-	for i, v := range sdl.RootData.Maps {
-		_, ok := parentMap[v.Pid]
-		if !ok {
-			parentMap[v.Pid] = make(map[int]*MapInfoT)
+	sort.Slice(mapSlice, func(i, j int) bool {
+		return mapSlice[i].Time < mapSlice[j].Time
+	})
+
+	funStack := []TimeMap{}
+
+	currentNid := 1
+	Nid := 2
+	level := 2
+
+	for k, v := range mapSlice {
+		fmt.Println("SliceSliceindex", k, "value", v.Time, v.Type, v.Map.MethodName, v.Map.Nid)
+		if v.Type == 0 {
+			// 函数入口
+			funStack = append(funStack, v)
+			v.Map.Pid = currentNid
+			v.Map.Level = level
+			v.Map.Nid = Nid
+			currentNid = Nid
+			level += 1
+			Nid += 1
+		} else if v.Type == 1 {
+			// 函数出口
+			len := len(funStack)
+			funStack = funStack[:len-1]
+			if (len - 2) < 0 {
+				currentNid = 1
+			} else {
+				currentNid = funStack[len-2].Map.Nid
+			}
+
+			level -= 1
 		}
-		parentMap[v.Pid][v.Nid] = &sdl.RootData.Maps[i]
 	}
-	// fmt.Println(parentMap)
-	// fmt.Println("nidMapnidMapnidMapnidMap--------------||||")
-	// 构建层级
-	level := 1
-	buildLevelRecursion(parentMap, 0, level)
 }
 
 func initRootData(traceId string) RootDataT {
@@ -263,7 +291,8 @@ func initMapNode(spanSd *tracepb.Span) (MapInfoT, string) {
 	mNode.MethodName = spanSd.Name
 	mNode.PureTime = (spanSd.EndTimeUnixNano - spanSd.StartTimeUnixNano) / 1e3
 	mNode.WallTime = mNode.PureTime
-	mNode.StartTime = spanSd.StartTimeUnixNano / 1e6
+	mNode.StartTime = spanSd.StartTimeUnixNano
+	mNode.EndTime = spanSd.EndTimeUnixNano
 
 	for _, attr := range spanSd.GetAttributes() {
 		switch attr.Key {
@@ -294,7 +323,7 @@ func buildAndAssemblyMap(sd apmTraceSpan, traceRoot *TraceMapT) MapInfoT {
 		buildRedisMap(&mNode, sd)
 	}
 	if mapType != "" {
-		// mNode.Nid = traceRoot.Index
+		mNode.Nid = traceRoot.Index
 		traceRoot.RootData.Maps = append(traceRoot.RootData.Maps, mNode)
 	}
 	return mNode

tracing/apm_tracing.go

@@ -5,13 +5,14 @@ import (
 	"fmt"
 	"time"
 
+	"strconv"
+
 	"github.com/coroot/coroot-node-agent/ebpftracer/l7"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	semconv "go.opentelemetry.io/otel/semconv/v1.18.0"
 	"go.opentelemetry.io/otel/trace"
 	"inet.af/netaddr"
-	"strconv"
 )
 
 /**
@@ -146,13 +147,24 @@ func (t *Trace) HttpTraceRequest(method, path, ip string, port uint16, r *l7.Req
 	)
 }
 
-func (t *Trace) FuncTraceQuery(funcname string, duration time.Duration, level int, pid int, nid int) {
+func (t *Trace) FuncTraceQuery(funcname string, duration time.Duration, start uint64, end uint64) {
 	if t == nil || funcname == "" {
 		return
 	}
-	t.createTraceSpan(funcname, duration, false,
-		attribute.Int("level", level),
-		attribute.Int("pid", pid),
-		attribute.Int("nid", nid),
-	)
+	t.createTraceSpanNoTime(funcname, duration, false, start, end)
+}
+
+func (t *Trace) createTraceSpanNoTime(name string, duration time.Duration, error bool, start uint64, end uint64, attrs ...attribute.KeyValue) {
+	// end := time.Now()
+	// start := end.Add(-duration)
+	startTime := time.Unix(0, int64(start))
+	endTime := time.Unix(0, int64(end))
+	//fmt.Println("createTraceSpan:", t.ctx)
+	_, span := tracer(t.containerId).Start(t.ctx, name, trace.WithTimestamp(startTime), trace.WithSpanKind(trace.SpanKindClient))
+	span.SetAttributes(t.commonAttrs...)
+	span.SetAttributes(attrs...)
+	if error {
+		span.SetStatus(codes.Error, "")
+	}
+	span.End(trace.WithTimestamp(endTime))
 }