ebpf/containers/registry.go

package containers

import (
	"bytes"
	"fmt"
	. "github.com/coroot/coroot-node-agent/utils"
	"github.com/coroot/coroot-node-agent/utils/enums"
	"github.com/coroot/coroot-node-agent/utils/modelse"
	"github.com/coroot/coroot-node-agent/utils/try"
	. "github.com/coroot/coroot-node-agent/utils/worker"
	log "github.com/sirupsen/logrus"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/coroot/coroot-node-agent/cgroup"
	"github.com/coroot/coroot-node-agent/common"
	"github.com/coroot/coroot-node-agent/ebpftracer"
	"github.com/coroot/coroot-node-agent/ebpftracer/tracer"
	"github.com/coroot/coroot-node-agent/flags"
	"github.com/coroot/coroot-node-agent/proc"
	"github.com/prometheus/client_golang/prometheus"
	klog "github.com/sirupsen/logrus"
	"github.com/vishvananda/netns"
	"inet.af/netaddr"
)

var (
	selfNetNs         = netns.None()
	hostNetNsId       = netns.None().UniqueId()
	agentPid          = uint32(os.Getpid())
	containerIdRegexp = regexp.MustCompile(`[a-z0-9]{64}`)
)

type ProcessInfo struct {
	Pid         uint32
	ContainerId ContainerID
	StartedAt   time.Time
}

type Registry struct {
	reg prometheus.Registerer

	tracer *ebpftracer.Tracer
	events chan ebpftracer.Event

	hostConntrack *Conntrack

	containersById       map[ContainerID]*Container
	containersByCgroupId map[string]*Container
	containersByPid      map[uint32]*Container
	ip2fqdn              map[netaddr.IP]string
	ip2fqdnLock          sync.Mutex

	processInfoCh chan<- ProcessInfo

	whiteListRules       WhiteListMap
	whiteLastUpdatedTime int
	connServer           ServerWorker
}

var (
	uprobes    []tracer.Uprobe
	uprobesMap map[string]tracer.Uprobe
)

func NewRegistry(reg prometheus.Registerer, kernelVersion string, processInfoCh chan<- ProcessInfo) (*Registry, error) {
	ns, err := proc.GetSelfNetNs()
	if err != nil {
		return nil, err
	}
	selfNetNs = ns
	hostNetNs, err := proc.GetHostNetNs()
	if err != nil {
		return nil, err
	}
	defer hostNetNs.Close()
	hostNetNsId = hostNetNs.UniqueId()

	err = proc.ExecuteInNetNs(hostNetNs, selfNetNs, func() error {
		if err := TaskstatsInit(); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	if err := cgroup.Init(); err != nil {
		return nil, err
	}
	if err := DockerdInit(); err != nil {
		klog.Warningln(err)
	}
	if err := ContainerdInit(); err != nil {
		klog.Warningln(err)
	}
	if err := CrioInit(); err != nil {
		klog.Warningln(err)
	}
	if err := JournaldInit(); err != nil {
		klog.Warningln(err)
	}
	ct, err := NewConntrack(hostNetNs)
	if err != nil {
		return nil, err
	}

	r := &Registry{
		reg:    reg,
		events: make(chan ebpftracer.Event, 10000),

		hostConntrack: ct,

		containersById:       map[ContainerID]*Container{},
		containersByCgroupId: map[string]*Container{},
		containersByPid:      map[uint32]*Container{},
		ip2fqdn:              map[netaddr.IP]string{},

		processInfoCh: processInfoCh,

		tracer:         ebpftracer.NewTracer(kernelVersion, *flags.DisableL7Tracing, *flags.DisableE2ETracing, *flags.DisableStackTracing),
		whiteListRules: make(WhiteListMap),
	}
	// 初始化软负载集群节点
	proxyClient, clientErr := NewProxyClient(os.Getenv("CONFIG_ENDPOINT"), false)
	if clientErr == nil {
		// 负载健康检测
		try.Go(proxyClient.CheckEndpoints, CatchFn)
		log.Infof("New Proxy Client success.config_server is [%s]", "")
	} else {
		log.WithError(clientErr).Errorf("New Proxy Client error,Please check export CONFIG_ENDPOINT=")
		return nil, clientErr
	}

	r.connServer, err = NewServerHTTPWorker()
	if err != nil {
		log.Errorf("init connServer error:%s.", err)
		return nil, err
	}

	if err = reg.Register(r); err != nil {
		return nil, err
	}

	//_, err = r.getWhiteList()
	//if err != nil {
	//	return nil, err
	//}

	go r.handleEvents(r.events)
	if err = r.tracer.Run(r.events); err != nil {
		close(r.events)
		return nil, err
	}

	return r, nil
}

func (r *Registry) Describe(ch chan<- *prometheus.Desc) {
	ch <- metrics.Ip2Fqdn
}

func (r *Registry) Collect(ch chan<- prometheus.Metric) {
	r.ip2fqdnLock.Lock()
	defer r.ip2fqdnLock.Unlock()
	for ip, fqdn := range r.ip2fqdn {
		ch <- gauge(metrics.Ip2Fqdn, 1, ip.String(), fqdn)
	}
}

func (r *Registry) Close() {
	r.tracer.Close()
	close(r.events)
}

func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
	gcTicker := time.NewTicker(gcInterval)
	defer gcTicker.Stop()
	for {
		select {
		case now := <-gcTicker.C:
			_, err := r.getWhiteList()
			if err != nil {
				log.WithError(err).Errorf("connWhiteList error")
			}
			runtimeApps := make(map[uint32]modelse.AppStatusInfo)
			for pid, c := range r.containersByPid {
				if !common.IsOpenFilter() {
					verifyAttachConditions := c.verifyAttachConditions(r, pid)
					if verifyAttachConditions {
						err = c.RegisterAppInfo(r, pid)
						if err == nil {
							klog.WithField("pid", pid).Infoln("[registry] Attach uprobes.")
							err = c.attachUprobes(r.tracer, pid)
							if err != nil {
								klog.WithField("pid", pid).WithError(err).Errorf("[registry] Failed attach uprobes error!")
							} else {
								klog.WithField("pid", pid).Infoln("[registry] Attach uprobes success!")
							}
							klog.WithField("pid", pid).Infoln("[registry] Attach app stack.")
							err = c.stackTrace(r.tracer, pid)
							if err != nil {
								klog.WithField("pid", pid).WithError(err).Errorf("[registry][end] Failed attach stack trace!")
							} else {
								klog.WithField("pid", pid).Infoln("[registry] Attach Stack success!")
							}
						}
					}

					if !verifyAttachConditions && c.checkL7AttachReady() {
						// detach
						c.detachUprobes(pid)
					}
				}
				if c.AppInfo.AppName != "" {
					detail := modelse.AppStatusInfo{
						Pid:        pid,
						ProcName:   c.containerName,
						AppName:    c.AppInfo.AppName,
						Language:   c.AppInfo.CodeType.String(),
						AppID:      c.AppInfo.AppIdHash.IntVal,
						AgentID:    c.AppInfo.AgentId,
						InstanceID: c.AppInfo.InstanceIdHash.IntVal,
						Sn:         c.AppInfo.Sn,
						Sport:      c.AppInfo.Sport,
						RegisterAt: time.Unix(c.AppInfo.RegisterAt, 0).Format("060102 15:04:05"),
					}
					if c.AppInfo.UpdateAt != 0 {
						detail.UpdateAt = time.Unix(c.AppInfo.UpdateAt, 0).Format("060102 15:04:05")
					}

					runtimeApps[pid] = detail
				}

				cg, err := proc.ReadCgroup(pid)
				if err != nil {
					delete(r.containersByPid, pid)
					if c != nil {
						c.onProcessExit(pid, false)
					}
					continue
				}
				if c != nil && cg.Id != c.cgroup.Id {
					delete(r.containersByPid, pid)
					c.onProcessExit(pid, false)
				}
			}
			saveAppInfo(runtimeApps)

			activeIPs := map[netaddr.IP]struct{}{}
			for id, c := range r.containersById {
				if !c.Dead(now) {
					continue
				}
				for dst := range c.connectLastAttempt {
					activeIPs[dst.IP()] = struct{}{}
				}
				klog.Infoln("deleting dead container:", id)
				for cg, cc := range r.containersByCgroupId {
					if cc == c {
						delete(r.containersByCgroupId, cg)
					}
				}
				for pid, cc := range r.containersByPid {
					if cc == c {
						delete(r.containersByPid, pid)
					}
				}
				if ok := prometheus.WrapRegistererWith(setLabels(string(id),
					c.K8sContainer.ns,
					c.K8sContainer.podName,
					c.K8sContainer.containerName,
					c.K8sContainer.pid), r.reg).Unregister(c); !ok {
					klog.Warningln("failed to unregister container:", id)
				}
				delete(r.containersById, id)
				c.Close()
			}
			r.ip2fqdnLock.Lock()
			for ip := range r.ip2fqdn {
				if _, ok := activeIPs[ip]; !ok {
					delete(r.ip2fqdn, ip)
				}
			}
			r.ip2fqdnLock.Unlock()
		case e, more := <-ch:
			if e.Pid == uint32(os.Getpid()) {
				continue
			}
			if !more {
				return
			}
			switch e.Type {
			case ebpftracer.EventTypeProcessStart:
				c, seen := r.containersByPid[e.Pid]
				switch { // possible pids wraparound + missed `process-exit` event
				case c == nil && seen: // ignored
					delete(r.containersByPid, e.Pid)
				case c != nil: // revalidating by cgroup
					cg, err := proc.ReadCgroup(e.Pid)
					if err != nil || cg.Id != c.cgroup.Id {
						delete(r.containersByPid, e.Pid)
						c.onProcessExit(e.Pid, false)
					}
				}
				if c := r.getOrCreateContainer(e.Pid); c != nil {
					p := c.onProcessStart(e.Pid)
					if r.processInfoCh != nil && p != nil {
						r.processInfoCh <- ProcessInfo{Pid: p.Pid, ContainerId: c.id, StartedAt: p.StartedAt}
					}
				}
			case ebpftracer.EventTypeProcessExit:
				if c := r.containersByPid[e.Pid]; c != nil {
					c.onProcessExit(e.Pid, e.Reason == ebpftracer.EventReasonOOMKill)
				}
				delete(r.containersByPid, e.Pid)

			case ebpftracer.EventTypeFileOpen:
				if c := r.getOrCreateContainer(e.Pid); c != nil {
					c.onFileOpen(e.Pid, e.Fd)
				}

			case ebpftracer.EventTypeListenOpen:
				//fmt.Println("ebpftracer.EventTypeListenOpen==================", e.Pid)
				if c := r.getOrCreateContainer(e.Pid); c != nil {
					c.onListenOpen(e.Pid, e.SrcAddr, false)
					// cmdline InstanceID agentID
					if c.buildIDs(e.Pid) {
						c.eventReady()
					}
					if common.IsOpenFilter() && common.IsFilterPid(e.Pid) {
						c.WhiteSettingInfo.AppName = enums.TestApp
						c.RegisterAppInfo(r, e.Pid)
						c.attachUprobes(r.tracer, e.Pid)
						err := c.stackTrace(r.tracer, e.Pid)
						if err != nil {
							klog.Errorf("Stack trace error", err)
						}
					}
				} else {
					klog.Infoln("TCP listen open from unknown container", e)
				}
			case ebpftracer.EventTypeConnectionOpen:
				//fmt.Println("ebpftracer.EventTypeConnectionOpen==================", e.Pid)
				if c := r.getOrCreateContainer(e.Pid); c != nil {
					c.onConnectionOpen(e.Pid, e.Fd, e.SrcAddr, e.DstAddr, e.Timestamp, false)
					c.eventReady()
					if common.IsOpenFilter() && common.IsFilterPid(e.Pid) {
						c.WhiteSettingInfo.AppName = enums.TestApp
						c.RegisterAppInfo(r, e.Pid)
						c.attachUprobes(r.tracer, e.Pid)
						err := c.stackTrace(r.tracer, e.Pid)
						if err != nil {
							klog.Errorf("Stack trace error", err)
						}
					}
				} else {
					klog.Infoln("TCP connection from unknown container", e)
				}
			case ebpftracer.EventTypeListenClose:
				if c := r.containersByPid[e.Pid]; c != nil {
					c.onListenClose(e.Pid, e.SrcAddr)
				}
			case ebpftracer.EventTypeConnectionError:
				if c := r.getOrCreateContainer(e.Pid); c != nil {
					c.onConnectionOpen(e.Pid, e.Fd, e.SrcAddr, e.DstAddr, 0, true)
				} else {
					klog.Infoln("TCP connection error from unknown container", e)
				}
			case ebpftracer.EventTypeConnectionClose:
				srcDst := AddrPair{src: e.SrcAddr, dst: e.DstAddr}
				for _, c := range r.containersById {
					if c.onConnectionClose(srcDst) {
						break
					}
				}
			case ebpftracer.EventTypeTCPRetransmit:
				srcDst := AddrPair{src: e.SrcAddr, dst: e.DstAddr}
				for _, c := range r.containersById {
					if c.onRetransmit(srcDst) {
						break
					}
				}
			case ebpftracer.EventTypeL7Request:
				//fmt.Println("e.L7Request Payload:", string(e.L7Request.Payload))
				if e.L7Request == nil {
					continue
				}
				if c := r.containersByPid[e.Pid]; c != nil {
					//fmt.Println("EventTypeL7Request", e.Pid, c.checkL7AttachReady())
					//a, _ := json.Marshal(e.L7Request)
					//fmt.Println("EventTypeL7Request", e.Pid, string(a))
					ip2fqdn := c.onL7RequestApm(e.Pid, e.Fd, e.Timestamp, e.L7Request)
					r.ip2fqdnLock.Lock()
					for ip, fqdn := range ip2fqdn {
						r.ip2fqdn[ip] = fqdn
					}
					r.ip2fqdnLock.Unlock()
				}
			case ebpftracer.EventTypeFunEnt:
				if e.StackEvent == nil {
					continue
				}

				if c := r.containersByPid[uint32(e.StackEvent.Pid)]; c != nil {
					fmt.Printf("e.EventTypeFunEnt: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x\n", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
					c.StackProcess2(*e.StackEvent, r.tracer)
				} else {
					// fmt.Printf("e.EventTypeFunEnt ErrorError: TraceId:%d, Pid:%d, Location:%d, Goid:%d, TimeNs:%d, Ip:%X, CallerIp:%x, Bp:%x, CallerBp:%x", e.StackEvent.TraceId, e.StackEvent.Pid, e.StackEvent.Location, e.StackEvent.Goid, e.StackEvent.TimeNsStart, e.StackEvent.Ip, e.StackEvent.CallerIp, e.StackEvent.Bp, e.StackEvent.CallerBp)
					// fmt.Printf("e.EventTypeFunEnt ErrorError: TraceId:%x, FPid:%x, Nid:%x, Level:%d\n", e.StackEvent.Fpid, e.StackEvent.Nid, e.StackEvent.Level)
				}
			}
		}
	}
}

func (r *Registry) getOrCreateContainer(pid uint32) *Container {
	if c, seen := r.containersByPid[pid]; c != nil {
		return c
	} else if seen { // ignored
		return nil
	}
	cg, err := proc.ReadCgroup(pid)
	if err != nil {
		if !common.IsNotExist(err) {
			klog.Warningln("failed to read proc cgroup:", err)
		}
		return nil
	}
	cgId := fmt.Sprintf("%s/%d", cg.Id, pid)
	if c := r.containersByCgroupId[cgId]; c != nil {
		r.containersByPid[pid] = c
		return c
	}
	if cg.ContainerType == cgroup.ContainerTypeSandbox {
		cmdline := proc.GetCmdline(pid)
		parts := bytes.Split(cmdline, []byte{0})
		if len(parts) > 0 {
			cmd := parts[0]
			lastArg := parts[len(parts)-1]
			if (bytes.HasSuffix(cmd, []byte("runsc-sandbox")) || bytes.HasSuffix(cmd, []byte("runsc"))) && containerIdRegexp.Match(lastArg) {
				cg.ContainerId = string(lastArg)
			}
		}
	}
	md, err := getContainerMetadata(cg)
	if err != nil {
		klog.Warningf("failed to get container metadata for pid %d -> %s: %s", pid, cg.Id, err)
		return nil
	}
	// add ns/workload/podname
	id, extensionTag := calcId(cg, md, pid)

	//klog.Infof("calculated container id %d -> %s -> %s", pid, cg.Id, id)
	if id == "" {
		if cg.Id == "/init.scope" && pid != 1 {
			klog.Infoln("ignoring without persisting", "cg", cg.Id, "pid", pid)
		} else {
			klog.Infoln("ignoring", "cg", cg.Id, "pid", pid)
			r.containersByPid[pid] = nil
		}
		return nil
	}
	if c := r.containersById[id]; c != nil {
		//klog.Warningln("id conflict:", id)
		if cg.CreatedAt().After(c.cgroup.CreatedAt()) {
			c.cgroup = cg
			c.metadata = md
			c.runLogParser("")
			if c.nsConntrack != nil {
				_ = c.nsConntrack.Close()
				c.nsConntrack = nil
			}
		}
		setK8sTag(c, extensionTag, pid)
		r.containersByPid[pid] = c
		r.containersByCgroupId[cgId] = c
		return c
	}

	c, err := NewContainer(id, cg, md, r.hostConntrack, pid)
	if err != nil {
		klog.Warningf("failed to create container pid=%d cg=%s id=%s: %s", pid, cg.Id, id, err)
		return nil
	}

	//klog.Infoln("detected a new container", "pid", pid, "cg", cg.Id, "id", id)
	// add ns/workload/podname/pid/ctype
	//sType := fmt.Sprintf("%d", cg.ContainerType)

	setK8sTag(c, extensionTag, pid)
	if err := prometheus.WrapRegistererWith(setLabels(string(id),
		extensionTag[Namespace],
		extensionTag[PodName],
		extensionTag[ProcessName],
		fmt.Sprintf("%d", pid)), r.reg).Register(c); err != nil {
		klog.Warningln("failed to register container:", err)
		return nil
	}
	r.containersByPid[pid] = c
	r.containersByCgroupId[cgId] = c
	r.containersById[id] = c
	return c
}

func calcId(cg *cgroup.Cgroup, md *ContainerMetadata, pid uint32) (ContainerID, map[string]string) {
	extensionTag := map[string]string{Namespace: "", Workload: "", PodName: "", ProcessName: ""}
	if cg.ContainerType == cgroup.ContainerTypeSystemdService {
		if strings.HasPrefix(cg.ContainerId, "/system.slice/crio-conmon-") {
			return "", extensionTag
		}
		return ContainerID(cg.ContainerId), extensionTag
	}
	if cg.ContainerType == cgroup.ContainerTypeStandaloneProcess {
		procName := proc.GetProcName(pid)
		extensionTag[ProcessName] = procName
		return ContainerID(fmt.Sprintf("/%s/%s/%d", "standalone", proc.GetProcName(pid), pid)), extensionTag
	}
	switch cg.ContainerType {
	case cgroup.ContainerTypeDocker, cgroup.ContainerTypeContainerd, cgroup.ContainerTypeSandbox, cgroup.ContainerTypeCrio:
	default:
		return "", extensionTag
	}
	if cg.ContainerId == "" {
		return "", extensionTag
	}
	if md.labels["io.kubernetes.pod.name"] != "" {
		pod := md.labels["io.kubernetes.pod.name"]
		namespace := md.labels["io.kubernetes.pod.namespace"]
		name := md.labels["io.kubernetes.container.name"]
		if cg.ContainerType == cgroup.ContainerTypeSandbox {
			name = "sandbox"
		}
		if name == "" || name == "POD" { // skip pause containers
			return "", extensionTag
		}
		extensionTag[Namespace] = namespace
		extensionTag[Workload] = ""
		extensionTag[PodName] = pod
		extensionTag[ProcessName] = name
		return ContainerID(fmt.Sprintf("/k8s/%s/%s/%s", namespace, pod, name)), extensionTag
	}
	if taskNameParts := strings.SplitN(md.labels["com.docker.swarm.task.name"], ".", 3); len(taskNameParts) == 3 {
		namespace := md.labels["com.docker.stack.namespace"]
		service := md.labels["com.docker.swarm.service.name"]
		if namespace != "" {
			service = strings.TrimPrefix(service, namespace+"_")
		}
		if namespace == "" {
			namespace = "_"
		}
		return ContainerID(fmt.Sprintf("/swarm/%s/%s/%s", namespace, service, taskNameParts[1])), extensionTag
	}
	if md.name == "" { // should be "pure" dockerd container here
		klog.Warningln("empty dockerd container name for:", cg.ContainerId)
		return "", extensionTag
	}
	return ContainerID("/docker/" + md.name), extensionTag
}

func getContainerMetadata(cg *cgroup.Cgroup) (*ContainerMetadata, error) {
	switch cg.ContainerType {
	case cgroup.ContainerTypeDocker, cgroup.ContainerTypeContainerd, cgroup.ContainerTypeSandbox, cgroup.ContainerTypeCrio:
	default:
		return &ContainerMetadata{}, nil
	}
	if cg.ContainerId == "" {
		return &ContainerMetadata{}, nil
	}
	if cg.ContainerType == cgroup.ContainerTypeCrio {
		return CrioInspect(cg.ContainerId)
	}
	var dockerdErr error
	if dockerdClient != nil {
		md, err := DockerdInspect(cg.ContainerId)
		if err == nil {
			return md, nil
		}
		dockerdErr = err
	}
	var containerdErr error
	if containerdClient != nil {
		md, err := ContainerdInspect(cg.ContainerId)
		if err == nil {
			return md, nil
		}
		containerdErr = err
	}
	return nil, fmt.Errorf("failed to interact with dockerd (%s) or with containerd (%s)", dockerdErr, containerdErr)
}