eBPF-based TLS connections tracing for applications using OpenSSL

containers/container.go

@@ -91,8 +91,9 @@ type Process struct {
 	StartedAt time.Time
 	NetNsId   string
 
-	uprobes             []link.Link
-	goTlsUprobesChecked bool
+	uprobes               []link.Link
+	goTlsUprobesChecked   bool
+	openSslUprobesChecked bool
 }
 
 func (p *Process) isHostNs() bool {
@@ -999,6 +1000,10 @@ func (c *Container) attachTlsUprobes(tracer *ebpftracer.Tracer, pid uint32) {
 	if p == nil {
 		return
 	}
+	if !p.openSslUprobesChecked {
+		p.uprobes = append(p.uprobes, tracer.AttachOpenSslUprobes(pid)...)
+		p.openSslUprobesChecked = true
+	}
 	if !p.goTlsUprobesChecked {
 		p.uprobes = append(p.uprobes, tracer.AttachGoTlsUprobes(pid)...)
 		p.goTlsUprobesChecked = true

ebpftracer/Dockerfile

@@ -14,7 +14,10 @@ RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_arm64 -c ebpf.c
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf506arm64.o && llvm-strip --strip-debug ebpf506arm64.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf512arm64.o && llvm-strip --strip-debug ebpf512arm64.o
 
-RUN echo -en '// generated - do not edit\npackage ebpftracer\nvar ebpfProg = map[string][]struct{v string; p []byte}{\n' > ebpf.go \
+RUN echo -en '// generated - do not edit\npackage ebpftracer\n\nvar ebpfProg = map[string][]struct {\n' > ebpf.go \
+	&& echo -en '\tv string\n' >> ebpf.go \
+	&& echo -en '\tp []byte\n' >> ebpf.go \
+	&& echo -en '}{\n' >> ebpf.go \
 	&& echo -en '\t"amd64": {\n' >> ebpf.go \
 	&& echo -en '\t\t{"v5.12", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf512x86.o >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"v5.6", []byte("' >> ebpf.go && hexdump -v -e '"\x" 1/1 "%02x"' ebpf506x86.o >> ebpf.go && echo '")},' >> ebpf.go \

ebpftracer/ebpf/ebpf.c

@@ -35,6 +35,7 @@
 #include "tcp/state.c"
 #include "tcp/retransmit.c"
 #include "l7/l7.c"
-#include "l7/tls.c"
+#include "l7/gotls.c"
+#include "l7/openssl.c"
 
 char _license[] SEC("license") = "GPL";

ebpftracer/ebpf/l7/tls.c → ebpftracer/ebpf/l7/gotls.c

@@ -56,7 +56,7 @@ int go_crypto_tls_read_enter(struct pt_regs *ctx) {
     __u64 goroutine_id = GOROUTINE(ctx);
     __u64 pid = pid_tgid >> 32;
     __u64 id = pid << 32 | goroutine_id | IS_TLS_READ_ID;
-    return trace_enter_read(id, fd, buf_ptr);
+    return trace_enter_read(id, fd, buf_ptr, 0);
 }
 
 SEC("uprobe/go_crypto_tls_read_exit")

ebpftracer/ebpf/l7/l7.c

@@ -56,6 +56,7 @@ struct {
 struct read_args {
     __u64 fd;
     char* buf;
+    __u64* ret;
 };
 
 struct {
@@ -229,10 +230,11 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size)
 }
 
 static inline __attribute__((__always_inline__))
-int trace_enter_read(__u64 id, __u64 fd, char *buf) {
+int trace_enter_read(__u64 id, __u64 fd, char *buf, __u64 *ret) {
     struct read_args args = {};
     args.fd = fd;
     args.buf = buf;
+    args.ret = ret;
     bpf_map_update_elem(&active_reads, &id, &args, BPF_ANY);
     return 0;
 }
@@ -254,6 +256,14 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
     if (ret <= 0) {
         return 0;
     }
+    if (args->ret) {
+        if (bpf_probe_read(&ret, sizeof(ret), (void*)args->ret)) {
+            return 0;
+        };
+        if (ret <= 0) {
+            return 0;
+        }
+    }
 
     int zero = 0;
     struct l7_event *e = bpf_map_lookup_elem(&l7_event_heap, &zero);
@@ -363,7 +373,7 @@ int sys_enter_sendto(struct trace_event_raw_sys_enter_rw__stub* ctx) {
 SEC("tracepoint/syscalls/sys_enter_read")
 int sys_enter_read(struct trace_event_raw_sys_enter_rw__stub* ctx) {
     __u64 id = bpf_get_current_pid_tgid();
-    return trace_enter_read(id, ctx->fd, ctx->buf);
+    return trace_enter_read(id, ctx->fd, ctx->buf, 0);
 }
 
 SEC("tracepoint/syscalls/sys_enter_readv")
@@ -373,13 +383,13 @@ int sys_enter_readv(struct trace_event_raw_sys_enter_rw__stub* ctx) {
     if (bpf_probe_read(&iovec0, sizeof(struct iovec), (void *)ctx->buf) < 0) {
         return 0;
     }
-    return trace_enter_read(id, ctx->fd, iovec0.buf);
+    return trace_enter_read(id, ctx->fd, iovec0.buf, 0);
 }
 
 SEC("tracepoint/syscalls/sys_enter_recvfrom")
 int sys_enter_recvfrom(struct trace_event_raw_sys_enter_rw__stub* ctx) {
     __u64 id = bpf_get_current_pid_tgid();
-    return trace_enter_read(id, ctx->fd, ctx->buf);
+    return trace_enter_read(id, ctx->fd, ctx->buf, 0);
 }
 
 SEC("tracepoint/syscalls/sys_exit_read")

ebpftracer/ebpf/l7/openssl.c

@@ -0,0 +1,143 @@
+struct unused {};
+typedef long (*unused_fn)();
+
+struct bio_st {
+    struct unused* method;
+    unused_fn callback;
+    char* cb_arg;
+    int init;
+    int shutdown;
+    int flags;
+    int retry_reason;
+    int num; // fd
+};
+
+struct bio_st_v1_1_1 {
+    struct unused* method;
+    unused_fn callback;
+    unused_fn callback_ex; // new field
+    char* cb_arg;
+    int init;
+    int shutdown;
+    int flags;
+    int retry_reason;
+    int num; // fd
+};
+
+struct bio_st_v3_0 {
+    struct unused* context; // new field
+    struct unused* method;
+    unused_fn callback;
+    unused_fn callback_ex;
+    char* cb_arg;
+    int init;
+    int shutdown;
+    int flags;
+    int retry_reason;
+    int num; // fd
+};
+
+struct ssl_st {
+    __s32 version;
+    struct unused* method;
+    struct bio_st* rbio;  // used by SSL_read
+    struct bio_st* wbio;  // used by SSL_write
+};
+
+#define GET_FD(ctx, bio_t, bio_rw)                                      \
+({                                                                      \
+    struct ssl_st ssl;                                                  \
+    if (bpf_probe_read(&ssl, sizeof(ssl), (void*)PT_REGS_PARM1(ctx))) { \
+        return 0;                                                       \
+    };                                                                  \
+    struct bio_t bio;                                                   \
+    if (bpf_probe_read(&bio, sizeof(bio), (void*)ssl.bio_rw)) {         \
+        return 0;                                                       \
+    };                                                                  \
+    __u32 fd = bio.num;                                                 \
+    if (fd <= 2) {                                                      \
+        return 0;                                                       \
+    }                                                                   \
+    fd;                                                                 \
+})
+
+#define WRITE_ENTER(ctx, bio_t)                              \
+({                                                           \
+    __u32 fd = GET_FD(ctx, bio_t, wbio);                     \
+    char* buf_ptr = (char*)PT_REGS_PARM2(ctx);               \
+    __u64 buf_size = PT_REGS_PARM3(ctx);                     \
+    return trace_enter_write(ctx, fd, 1, buf_ptr, buf_size); \
+})
+
+#define READ_ENTER(ctx, bio_t)                   \
+({                                               \
+    __u32 fd = GET_FD(ctx, bio_t, rbio);         \
+    char* buf_ptr = (char*)PT_REGS_PARM2(ctx);   \
+    __u64 pid_tgid = bpf_get_current_pid_tgid(); \
+    __u64 id = pid_tgid | IS_TLS_READ_ID;        \
+    return trace_enter_read(id, fd, buf_ptr, 0); \
+})
+
+#define READ_EX_ENTER(ctx, bio_t)                      \
+({                                                     \
+    __u32 fd = GET_FD(ctx, bio_t, rbio);               \
+    char* buf_ptr = (char*)PT_REGS_PARM2(ctx);         \
+    __u64 pid_tgid = bpf_get_current_pid_tgid();       \
+    __u64 id = pid_tgid | IS_TLS_READ_ID;              \
+    __u64* ret_ptr = (__u64*)PT_REGS_PARM4(ctx);       \
+    return trace_enter_read(id, fd, buf_ptr, ret_ptr); \
+})
+
+SEC("uprobe/openssl_SSL_write_enter")
+int openssl_SSL_write_enter(struct pt_regs *ctx) {
+    WRITE_ENTER(ctx, bio_st);
+}
+
+SEC("uprobe/openssl_SSL_write_enter_v1_1_1")
+int openssl_SSL_write_enter_v1_1_1(struct pt_regs *ctx) {
+    WRITE_ENTER(ctx, bio_st_v1_1_1);
+}
+
+SEC("uprobe/openssl_SSL_write_enter_v3_0")
+int openssl_SSL_write_enter_v3_0(struct pt_regs *ctx) {
+    WRITE_ENTER(ctx, bio_st_v3_0);
+}
+
+SEC("uprobe/openssl_SSL_read_enter")
+int openssl_SSL_read_enter(struct pt_regs *ctx) {
+    READ_ENTER(ctx, bio_st);
+}
+
+SEC("uprobe/openssl_SSL_read_ex_enter")
+int openssl_SSL_read_ex_enter(struct pt_regs *ctx) {
+    READ_EX_ENTER(ctx, bio_st);
+}
+
+SEC("uprobe/openssl_SSL_read_enter_v1_1_1")
+int openssl_SSL_read_enter_v1_1_1(struct pt_regs *ctx) {
+    READ_ENTER(ctx, bio_st_v1_1_1);
+}
+
+SEC("uprobe/openssl_SSL_read_ex_enter_v1_1_1")
+int openssl_SSL_read_ex_enter_v1_1_1(struct pt_regs *ctx) {
+    READ_EX_ENTER(ctx, bio_st_v1_1_1);
+}
+
+SEC("uprobe/openssl_SSL_read_enter_v3_0")
+int openssl_SSL_read_enter_v3_0(struct pt_regs *ctx) {
+    READ_ENTER(ctx, bio_st_v3_0);
+}
+
+SEC("uprobe/openssl_SSL_read_ex_enter_v3_0")
+int openssl_SSL_read_ex_enter_v3_0(struct pt_regs *ctx) {
+    READ_EX_ENTER(ctx, bio_st_v3_0);
+}
+
+SEC("uprobe/openssl_SSL_read_exit")
+int openssl_SSL_read_exit(struct pt_regs *ctx) {
+    __u64 pid_tgid = bpf_get_current_pid_tgid();
+    __u64 pid = pid_tgid >> 32;
+    __u64 id = pid_tgid | IS_TLS_READ_ID;
+    int ret = (int)PT_REGS_RC(ctx);
+    return trace_exit_read(ctx, id, pid, 1, ret);
+}

ebpftracer/tls.go

@@ -1,6 +1,8 @@
 package ebpftracer
 
 import (
+	"bufio"
+	"bytes"
 	"debug/buildinfo"
 	"debug/elf"
 	"errors"
@@ -12,15 +14,99 @@ import (
 	"golang.org/x/mod/semver"
 	"k8s.io/klog/v2"
 	"os"
+	"regexp"
 	"strings"
 )
 
 const (
 	minSupportedGoVersion = "v1.17.0"
-	writeSymbol           = "crypto/tls.(*Conn).Write"
-	readSymbol            = "crypto/tls.(*Conn).Read"
+	goTlsWriteSymbol      = "crypto/tls.(*Conn).Write"
+	goTlsReadSymbol       = "crypto/tls.(*Conn).Read"
 )
 
+var (
+	opensslVersionRe = regexp.MustCompile(`OpenSSL\s(\d\.\d+\.\d+)`)
+)
+
+func (t *Tracer) AttachOpenSslUprobes(pid uint32) []link.Link {
+	if t.disableL7Tracing {
+		return nil
+	}
+	libPath, version := getSslLibPathAndVersion(pid)
+	if libPath == "" || version == "" {
+		return nil
+	}
+
+	log := func(msg string, err error) {
+		if err != nil {
+			for _, s := range []string{"no such file or directory", "no such process", "permission denied"} {
+				if strings.HasSuffix(err.Error(), s) {
+					return
+				}
+			}
+			klog.ErrorfDepth(1, "pid=%d libssl_version=%s: %s: %s", pid, version, msg, err)
+			return
+		}
+		klog.InfofDepth(1, "pid=%d libssl_version=%s: %s", pid, version, msg)
+	}
+
+	exe, err := link.OpenExecutable(libPath)
+	if err != nil {
+		log("failed to open executable", err)
+		return nil
+	}
+	var links []link.Link
+	writeEnter := "openssl_SSL_write_enter"
+	readEnter := "openssl_SSL_read_enter"
+	readExEnter := "openssl_SSL_read_ex_enter"
+	readExit := "openssl_SSL_read_exit"
+	switch {
+	case semver.Compare(version, "v3.0.0") >= 0:
+		writeEnter = "openssl_SSL_write_enter_v3_0"
+		readEnter = "openssl_SSL_read_enter_v3_0"
+		readExEnter = "openssl_SSL_read_ex_enter_v3_0"
+	case semver.Compare(version, "v1.1.1") >= 0:
+		writeEnter = "openssl_SSL_write_enter_v1_1_1"
+		readEnter = "openssl_SSL_read_enter_v1_1_1"
+		readExEnter = "openssl_SSL_read_ex_enter_v1_1_1"
+	}
+
+	type prog struct {
+		symbol    string
+		uprobe    string
+		uretprobe string
+	}
+	progs := []prog{
+		{symbol: "SSL_write", uprobe: writeEnter},
+		{symbol: "SSL_write_ex", uprobe: writeEnter},
+		{symbol: "SSL_read", uprobe: readEnter},
+		{symbol: "SSL_read_ex", uprobe: readExEnter},
+		{symbol: "SSL_read", uretprobe: readExit},
+		{symbol: "SSL_read_ex", uretprobe: readExit},
+	}
+	for _, p := range progs {
+		if p.uprobe != "" {
+			l, err := exe.Uprobe(p.symbol, t.uprobes[p.uprobe], nil)
+			if err != nil {
+				log("failed to attach uprobe", err)
+				return nil
+			}
+			links = append(links, l)
+		}
+		if p.uretprobe != "" {
+			l, err := exe.Uretprobe(p.symbol, t.uprobes[p.uretprobe], nil)
+			if err != nil {
+				log("failed to attach uretprobe", err)
+				return nil
+			}
+			links = append(links, l)
+		}
+	}
+
+	log("libssl uprobes attached", nil)
+	return links
+}
+
 func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 	if t.disableL7Tracing {
 		return nil
@@ -101,7 +187,7 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 			continue
 		}
 		switch s.Name {
-		case writeSymbol, readSymbol:
+		case goTlsWriteSymbol, goTlsReadSymbol:
 		default:
 			continue
 		}
@@ -117,14 +203,14 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 			}
 		}
 		switch s.Name {
-		case writeSymbol:
+		case goTlsWriteSymbol:
 			l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_write_enter"], &link.UprobeOptions{Address: address})
 			if err != nil {
 				log("failed to attach write_enter uprobe", err)
 				return nil
 			}
 			links = append(links, l)
-		case readSymbol:
+		case goTlsReadSymbol:
 			l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_read_enter"], &link.UprobeOptions{Address: address})
 			if err != nil {
 				log("failed to attach read_enter uprobe", err)
@@ -159,6 +245,72 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 	return links
 }
 
+func getSslLibPathAndVersion(pid uint32) (string, string) {
+	f, err := os.Open(proc.Path(pid, "maps"))
+	if err != nil {
+		return "", ""
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	scanner.Split(bufio.ScanLines)
+	var libsslPath, libcryptoPath string
+	for scanner.Scan() {
+		parts := strings.Fields(scanner.Text())
+		if len(parts) <= 5 {
+			continue
+		}
+		libPath := parts[5]
+		switch {
+		case libsslPath == "" && strings.Contains(libPath, "libssl.so"):
+			fullPath := proc.Path(pid, "root", libPath)
+			if _, err = os.Stat(fullPath); err == nil {
+				libsslPath = fullPath
+			}
+		case libcryptoPath == "" && strings.Contains(libPath, "libcrypto.so"):
+			fullPath := proc.Path(pid, "root", libPath)
+			if _, err = os.Stat(fullPath); err == nil {
+				libcryptoPath = fullPath
+			}
+		default:
+			continue
+		}
+		if libsslPath != "" && libcryptoPath != "" {
+			break
+		}
+	}
+	if libsslPath == "" || libcryptoPath == "" {
+		return "", ""
+	}
+
+	ef, err := elf.Open(libcryptoPath)
+	if err != nil {
+		return "", ""
+	}
+	defer ef.Close()
+	rodataSection := ef.Section(".rodata")
+	if rodataSection == nil {
+		return "", ""
+	}
+	rodataSectionData, err := rodataSection.Data()
+	if err != nil {
+		return "", ""
+	}
+	var version string
+	for _, b := range bytes.Split(rodataSectionData, []byte("\x00")) {
+		if len(b) == 0 {
+			continue
+		}
+		s := string(b)
+		if !strings.HasPrefix(s, "OpenSSL") {
+			continue
+		}
+		if m := opensslVersionRe.FindStringSubmatch(s); len(m) > 1 {
+			version = m[1]
+		}
+	}
+	return libsslPath, "v" + version
+}
+
 func getReturnOffsets(machine elf.Machine, instructions []byte) []int {
 	var res []int
 	switch machine {