Browse Source

Fixed #TASK_GK-2944 横向串联

Carl 2 years ago
parent
commit
17ff96f946

+ 2 - 2
ebpftracer/ebpf/ebpf.c

@@ -63,6 +63,6 @@
 #include "l7/l7.c"
 //#include "l7/gotls.c"
 //#include "l7/openssl.c"
-//#include "utrace/go/net/server.probe.bpf.c"
-//#include "utrace/go/net/client.probe.bpf.c"
+#include "utrace/go/net/server.probe.bpf.c"
+#include "utrace/go/net/client.probe.bpf.c"
 char _license[] SEC("license") = "GPL";

+ 3 - 0
ebpftracer/ebpf/include/socket_trace_common.h

@@ -184,6 +184,9 @@ struct ebpf_proc_info {
 	__u64 net_TCPConn_itab;
 	__u64 crypto_tls_Conn_itab; // TLS_HTTP1,TLS_HTTP2
 	__u64 credentials_syscallConn_itab; // gRPC
+
+//	__u64 start_addr;
+//	__u64 end_addr;
 } __attribute__((packed)) ;
 
 enum {

+ 130 - 86
ebpftracer/ebpf/l7/apm_trace.c

@@ -5,118 +5,162 @@
  * Trace struct
  ***********************************************************/
 struct apm_trace_key_t {
-    __u32 tgid;
-    __u32 pid;
-    __u64 goid;
+	__u32 tgid;
+	__u32 pid;
+	__u64 goid;
+};
+
+struct fd_trace_key_t {
+	__u32 tgid;
+	__u32 fd;
 };
 
-//struct fd_trace_key_t {
-//    __u32 tgid;
-//    __u32 fd;
-//};
 
+#define APM_TRACE_ID_SIZE 16
+#define APM_SPAN_ID_SIZE 8
+
+struct apm_span_context {
+	unsigned char TraceID[APM_TRACE_ID_SIZE];
+	unsigned char SpanID[APM_SPAN_ID_SIZE];
+};
 
 struct apm_trace_info_t {
-    /*
-     * Whether traceID is zero ?
-     * For the client to actively send request, set traceID to zero.
-     */
-    //	bool is_trace_id_zero;
-    //	__u32 update_time; // 从系统开机开始到创建/更新时的间隔时间单位是秒
-    //	__u32 peer_fd;	   // 用于socket之间的关联
-    //	__u64 thread_trace_id; // 线程追踪ID
-    //	__u64 socket_id; // Records the socket associated when tracing was created (记录创建追踪时关联的socket)
-    __u64 trace_id;
+	/*
+	 * Whether traceID is zero ?
+	 * For the client to actively send request, set traceID to zero.
+	 */
+	//	bool is_trace_id_zero;
+	//	__u32 update_time; // 从系统开机开始到创建/更新时的间隔时间单位是秒
+	//	__u32 peer_fd;	   // 用于socket之间的关联
+	//	__u64 thread_trace_id; // 线程追踪ID
+	//	__u64 socket_id; // Records the socket associated when tracing was created (记录创建追踪时关联的socket)
+	__u64 trace_id;
+	struct apm_span_context sc;
 };
 
 struct {
-    __uint(type, BPF_MAP_TYPE_LRU_HASH);
-    __uint(key_size, sizeof(struct apm_trace_key_t));
-    __uint(value_size, sizeof(struct apm_trace_info_t));
-    __uint(max_entries, 32768);
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct apm_trace_key_t));
+	__uint(value_size, sizeof(struct apm_span_context));
+	__uint(max_entries, 1);
+} apm_span_context_map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct apm_trace_key_t));
+	__uint(value_size, sizeof(struct apm_trace_info_t));
+	__uint(max_entries, 32768);
 } trace_info_heap SEC(".maps");
 
-//struct {
-//    __uint(type, BPF_MAP_TYPE_LRU_HASH);
-//    __uint(key_size, sizeof(struct fd_trace_key_t));
-//    __uint(value_size, sizeof(struct apm_trace_info_t));
-//    __uint(max_entries, 32768);
-//} fd_trace_info_heap SEC(".maps");
-
-
-static inline __attribute__((__always_inline__))
-struct apm_trace_key_t get_apm_trace_key_bak(__u32 pid, __u32 tid) {
-    struct apm_trace_key_t trace_key = {};
-    trace_key.tgid = pid;
-    trace_key.pid = tid;
-    trace_key.goid = 0;
-    return trace_key;
-}
+struct {
+	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+	__uint(key_size, sizeof(struct fd_trace_key_t));
+	__uint(value_size, sizeof(struct apm_trace_info_t));
+	__uint(max_entries, 32768);
+} fd_trace_info_heap SEC(".maps");
 
-static __inline struct apm_trace_key_t get_apm_trace_key(__u64 timeout, bool is_socket_io) {
-    __u64 pid_tgid = bpf_get_current_pid_tgid();
-    __u64 goid = 0;
+static __inline __attribute__((__always_inline__))
+struct apm_trace_key_t get_apm_trace_key(__u64 timeout, bool is_socket_io) {
+	__u64 pid_tgid = bpf_get_current_pid_tgid();
+	__u64 goid = 0;
 
-    if (timeout){
-        goid = get_rw_goid(timeout * NS_PER_SEC, is_socket_io);
-    }
+	if (timeout) {
+		goid = get_rw_goid(timeout * NS_PER_SEC, is_socket_io);
+	}
 
-    struct apm_trace_key_t key = {};
+	struct apm_trace_key_t key = {};
 
-    key.tgid = (__u32)(pid_tgid >> 32);
+	key.tgid = (__u32) (pid_tgid >> 32);
 
-    if (goid) {
-        key.goid = goid;
-    } else {
-        key.pid = (__u32)pid_tgid;
-    }
+	if (goid) {
+		key.goid = goid;
+	} else {
+		key.pid = (__u32) pid_tgid;
+	}
 
 //    debug("key.tgid:%llu|%llu",key.tgid,key.goid);
 //    debug("apm key.pid:%d",(__u32)pid_tgid);
 //    debug("apm key.goid:%llu",key.goid);
 //    cw_bpf_debug("[Trace ID] thread id:%d",(__u32)pid_tgid);
-    cw_bpf_debug("[Trace ID] GOID:%llu",key.goid);
-    return key;
+	bpf_printk("[Trace ID] GOID:%llu", key.goid);
+	return key;
 }
 
-static inline __attribute__((__always_inline__))
-__u64 get_trace_id(__u32 pid, __u32 tid) {
-    struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+static __inline __attribute__((__always_inline__))
+__u64 get_apm_trace_id(__u32 pid, __u32 tid) {
+	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
 
-    struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key);
-    if (trace_info) {
-        return trace_info->trace_id;
-    }
-    return 0;
+	struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&trace_info_heap, &trace_key);
+	if (trace_info) {
+		cw_bpf_debug("info_trace_id:%llu", trace_info->trace_id);
+		return trace_info->trace_id;
+	}
+	return 0;
 }
 
-static inline __attribute__((__always_inline__))
-__u64 clear_trace(__u32 pid, __u32 tid, __u32 fd) {
-    struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
-//    struct fd_trace_key_t fd_trace_key = get_fd_trace_key(pid, fd);
-    // 清除trace信息
-    bpf_map_delete_elem(&trace_info_heap, &trace_key);
-//    bpf_map_delete_elem(&fd_trace_info_heap, &fd_trace_key);
-    return 0;
+static __inline __attribute__((__always_inline__))
+struct fd_trace_key_t get_fd_trace_key(__u32 pid, __u32 fd) {
+	struct fd_trace_key_t trace_key = {};
+	trace_key.tgid = pid;
+	trace_key.fd = fd;
+	return trace_key;
 }
 
+static __inline __attribute__((__always_inline__))
+__u64 get_fd_trace_id(__u32 pid, __u32 fd) {
+	struct fd_trace_key_t trace_key = get_fd_trace_key(pid, fd);
+	struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&fd_trace_info_heap, &trace_key);
+	if (trace_info) {
+		cw_bpf_debug("fd_trace_id:%llu", trace_info->trace_id);
+		return trace_info->trace_id;
+	}
+	return 0;
+}
 
-//static __inline __attribute__((__always_inline__))
-//struct fd_trace_key_t get_fd_trace_key(__u32 pid, __u32 fd) {
-//    struct fd_trace_key_t trace_key = {};
-//    trace_key.tgid = pid;
-//    trace_key.fd = fd;
-//    return trace_key;
-//}
+static __inline __attribute__((__always_inline__))
+__u64 clear_trace(__u32 pid, __u32 tid, __u32 fd) {
+	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+	struct fd_trace_key_t fd_trace_key = get_fd_trace_key(pid, fd);
+	// 清除trace信息
+	bpf_map_delete_elem(&trace_info_heap, &trace_key);
+	bpf_map_delete_elem(&fd_trace_info_heap, &fd_trace_key);
+	return 0;
+}
 
-//
-//static inline __attribute__((__always_inline__))
-//__u64 get_fd_trace_id(__u32 pid, __u32 fd) {
-//    struct fd_trace_key_t trace_key = get_fd_trace_key(pid, fd);
-//    struct apm_trace_info_t *trace_info = bpf_map_lookup_elem(&fd_trace_info_heap, &trace_key);
-//    if (trace_info) {
-//        cw_bpf_debug("fd_trace_id:%llu", trace_info->trace_id);
-//        return trace_info->trace_id;
-//    }
-//    return 0;
-//}
+static __inline __attribute__((__always_inline__))
+void save_tracking_span(struct apm_span_context *sc) {
+	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+	long err = 0;
+	err = bpf_map_update_elem(&apm_span_context_map, &trace_key, sc, BPF_ANY);
+	if (err != 0) {
+		bpf_printk("Failed to update tracked_spans map: %ld", err);
+		return;
+	}
+	struct apm_span_context *span_contexts = bpf_map_lookup_elem(&apm_span_context_map, &trace_key);
+	if (span_contexts) {
+		for (int i = 0; i < APM_TRACE_ID_SIZE; i++) {
+			bpf_printk("---%02x", span_contexts->TraceID[i]);
+		}
+
+		for (int i = 0; i < APM_SPAN_ID_SIZE; i++) {
+			bpf_printk("===%02x", span_contexts->SpanID[i]);
+		}
+	}
+}
+
+static __inline __attribute__((__always_inline__))
+struct apm_span_context * get_tracking_span() {
+	struct apm_span_context *apm_sc = {0};
+	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+	struct apm_span_context *span_contexts = bpf_map_lookup_elem(&apm_span_context_map, &trace_key);
+	if (span_contexts) {
+		for (int i = 0; i < APM_TRACE_ID_SIZE; i++) {
+			bpf_printk("get---%02x", span_contexts->TraceID[i]);
+		}
+		for (int i = 0; i < APM_SPAN_ID_SIZE; i++) {
+			bpf_printk("get===%02x", span_contexts->SpanID[i]);
+		}
+		apm_sc = span_contexts;
+	}
+	return apm_sc;
+}

+ 24 - 40
ebpftracer/ebpf/l7/l7.c

@@ -276,12 +276,8 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
 
     if (is_http_response(payload, &http_status))
     {
-//        __u64 goid = get_rw_goid(120 * NS_PER_SEC, 1);
-//        cw_bpf_debug("[Response][HTTP]:thread_id:%d|goid:%d|FD:%d\n", tid, goid, k.fd);
-//        struct trace_key_t trace_key = get_trace_key(pid, tid);
-//        struct fd_trace_key_t fd_trace_key = get_fd_trace_key(pid, fd);
-        __u64 trace_id = get_trace_id(pid, fd);
-//        cw_bpf_debug("trace_id:%llu", trace_id);
+//        __u64 trace_id = get_apm_trace_id(pid, fd);
+        __u64 trace_id = get_fd_trace_id(pid, fd);
         cw_bpf_debug("[trace end][Response][HTTP] pid:%d,fd:%d,trace_id:%llu", pid, fd, trace_id);
         // 清除trace信息
         clear_trace(pid, tid, fd);
@@ -328,18 +324,6 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
 //        }
 
 //        cw_bpf_debug("off->task__files_offset:%x", off->task__files_offset);
-        struct ebpf_proc_info *info =
-                bpf_map_lookup_elem(&proc_info_map, &pid);
-        if (!info) {
-            return 0;
-        }
-
-        if(info->version >= GO_VERSION(1, 18, 0)){
-            cw_bpf_debug("e->GO_VERSION > 1.18:%d",info->version);
-        }
-        int offset_g_goid = info->offsets[OFFSET_IDX_GOID_RUNTIME_G];
-        cw_bpf_debug("offset_g_goid:%d", offset_g_goid);
-
         cw_bpf_debug("e->test_id1111:%d", ttt->test_id);
         cw_bpf_debug("HTTP_END");
         return 0;
@@ -348,9 +332,9 @@ int trace_enter_write(void *ctx, __u64 fd, __u16 is_tls, char *buf, __u64 size,
     if (is_http_request(payload)) {
         cw_bpf_debug("[Enter] [HTTP]:TGID:%d|FD:%d",k.pid,k.fd);
         req->protocol = PROTOCOL_HTTP;
-        struct apm_trace_key_t trace_key = {0};
-        trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
-        cw_bpf_debug("[Enter] [HTTP]:goid:%llu",trace_key.goid);
+//        struct apm_trace_key_t trace_key = {0};
+//        trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+//        cw_bpf_debug("[Enter] [HTTP]:goid:%llu",trace_key.goid);
 
 
     } else if (is_postgres_query(payload, size, &req->request_type)) {
@@ -551,13 +535,20 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 //        __u64 goid = get_rw_goid(120 * NS_PER_SEC, 1);
 //        cw_bpf_debug("[Receive][HTTP]:thread_id:%d|goid:%d|FD:%d\n", tid, goid, k.fd);
 
-        struct apm_trace_key_t trace_key = {0};
-        struct apm_trace_info_t trace_info = {0};
-        trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
-//        trace_key.tgid = pid;
-//        trace_key.pid = tid;
-        __u64 uid_base = bpf_ktime_get_ns();
-        trace_info.trace_id = bpf_get_current_pid_tgid() + uid_base;
+
+		// apm trace
+	    struct apm_trace_key_t trace_key = {0};
+	    trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+
+	    // fd trace
+        struct fd_trace_key_t fd_trace_key = {0};
+	    fd_trace_key = get_fd_trace_key(pid, k.fd);
+
+		// trace info
+	    struct apm_trace_info_t trace_info = {0};
+	    __u64 uid_base = bpf_ktime_get_ns();
+	    trace_info.trace_id = bpf_get_current_pid_tgid() + uid_base;
+
         e->trace_start = 1;
         e->trace_end = 0;
         e->protocol = PROTOCOL_TRACE;
@@ -565,16 +556,9 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
         e->payload_size = ret;
         COPY_PAYLOAD(e->payload, ret, payload);
 
-        // http trace
-//        struct fd_trace_key_t fd_trace_key = {};
-//        fd_trace_key.tgid = pid;
-//        fd_trace_key.fd = k.fd;
-//        bpf_map_update_elem(&fd_trace_info_heap, &fd_trace_key, &trace_info, BPF_NOEXIST);
-//        cw_bpf_debug("[Receive][HTTP] pid:%d,fd:%d,traceid:%llu", fd_trace_key.tgid, fd_trace_key.fd,
-//                   trace_info.trace_id);
-
-        // 入口方法缓存  bpf_map_update_elem(map, key, value, options)
+        // 入口方法缓存
         bpf_map_update_elem(&trace_info_heap, &trace_key, &trace_info, BPF_NOEXIST);
+	    bpf_map_update_elem(&fd_trace_info_heap, &fd_trace_key, &trace_info, BPF_NOEXIST);
         bpf_perf_event_output(ctx, &l7_events, BPF_F_CURRENT_CPU, e, sizeof(*e));
         cw_bpf_debug("[Receive][HTTP] to user space");
         return 0;
@@ -632,7 +616,7 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 
     bpf_map_delete_elem(&active_l7_requests, &k);
     if (e->protocol == PROTOCOL_HTTP) {
-        __u64 trace_id = get_trace_id(pid, tid);
+        __u64 trace_id = get_apm_trace_id(pid, tid);
         e->trace_id = trace_id;
         cw_bpf_debug("[Response][HTTP] [Trace ID] trace_id:%llu", e->trace_id);
         cw_bpf_debug("[Response][HTTP]:TGID:%d|type:%s|FD:%d\n",k.pid,"",k.fd);
@@ -652,7 +636,7 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
         }
     } else if (e->protocol == PROTOCOL_REDIS) {
         cw_bpf_debug("[Response][Redis]:TGID:%d|type:%s|FD:%d\n",k.pid,"",k.fd);
-        __u64 trace_id = get_trace_id(pid, tid);
+        __u64 trace_id = get_apm_trace_id(pid, tid);
         cw_bpf_debug("[Redis] trace_id:%llu", trace_id);
         e->trace_id = trace_id;
         response = is_redis_response(payload, ret, &e->status);
@@ -660,7 +644,7 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
         response = is_memcached_response(payload, ret, &e->status);
     } else if (e->protocol == PROTOCOL_MYSQL) {
         cw_bpf_debug("[Response][Mysql]:thread_id:%d\n",tid);
-        __u64 trace_id = get_trace_id(pid, tid);
+        __u64 trace_id = get_apm_trace_id(pid, tid);
 //        cw_bpf_debug("[Mysql] trace_id:%llu", trace_id);
         e->trace_id = trace_id;
         response = is_mysql_response(payload, ret, req->request_type, &e->statement_id, &e->status);

+ 210 - 0
ebpftracer/ebpf/utrace/go/include/go_context.h

@@ -0,0 +1,210 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef _GO_CONTEXT_H_
+#define _GO_CONTEXT_H_
+
+//#include "bpf_helpers.h"
+#include "go_types.h"
+
+// This limit is used to define the max length of the context.Context chain
+#define MAX_DISTANCE 100
+#define MAX_CONCURRENT_SPANS 1000
+
+struct
+{
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __type(key, void *);
+    __type(value, struct span_context);
+    __uint(max_entries, MAX_CONCURRENT_SPANS);
+//    __uint(pinning, LIBBPF_PIN_BY_NAME);
+} tracked_spans SEC(".maps");
+
+//struct apm_trace_key_t {
+//    __u32 tgid;
+//    __u64 goid;
+//};
+//
+//struct
+//{
+//    __uint(type, BPF_MAP_TYPE_HASH);
+//    __type(key, struct apm_trace_key_t);
+//    __type(value, struct span_context);
+//    __uint(max_entries, MAX_CONCURRENT_SPANS);
+//    __uint(pinning, LIBBPF_PIN_BY_NAME);
+//} tracked_spans_apm SEC(".maps");
+
+struct
+{
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __type(key, struct span_context);
+    __type(value, void *);
+    __uint(max_entries, MAX_CONCURRENT_SPANS);
+//    __uint(pinning, LIBBPF_PIN_BY_NAME);
+} tracked_spans_by_sc SEC(".maps");
+
+static __always_inline void *get_parent_go_context(void *ctx, void *map) {
+    void *data = ctx;
+    for (int i = 0; i < MAX_DISTANCE; i++)
+    {
+        if (data == NULL)
+        {
+            bpf_printk("data is null");
+            break;
+        }
+    
+        void *found_in_map = bpf_map_lookup_elem(map, &data);
+        if (found_in_map != NULL)
+        {
+            return data;
+        }
+        bpf_printk("data is null2");
+        // We assume context.Context implementation contains Parent context.Context member
+        // Since the parent is also an interface, we need to read the data part of it
+        bpf_probe_read(&data, sizeof(data), data + 8);
+    }
+
+    bpf_printk("context %lx not found in context map", ctx);
+    return NULL;
+}
+
+static __always_inline struct span_context *get_parent_span_context(void *ctx) {
+    void *parent_ctx = get_parent_go_context(ctx, &tracked_spans);
+    if (parent_ctx == NULL)
+    {
+        return NULL;
+    }
+
+    struct span_context *parent_sc = bpf_map_lookup_elem(&tracked_spans, &parent_ctx);
+    if (parent_sc == NULL)
+    {
+        return NULL;
+    }
+
+    return parent_sc;
+}
+
+//static __inline struct apm_trace_key_t get_apm_trace_key(__u64 timeout, bool is_socket_io) {
+//    __u64 pid_tgid = bpf_get_current_pid_tgid();
+//    __u64 goid = 0;
+//
+//    if (timeout){
+//        goid = get_rw_goid(timeout * NS_PER_SEC, is_socket_io);
+//    }
+//
+//    struct apm_trace_key_t key = {};
+//
+//    key.tgid = (__u32)(pid_tgid >> 32);
+//
+//    if (goid) {
+//        key.goid = goid;
+//    } else {
+//        key.pid = (__u32)pid_tgid;
+//    }
+//
+////    debug("key.tgid:%llu|%llu",key.tgid,key.goid);
+//    bpf_printk("apm key.pid:%d",(__u32)pid_tgid);
+//    bpf_printk("apm key.goid:%llu",key.goid);
+//    return key;
+//}
+
+static __always_inline void start_tracking_span(void *contextContext, struct span_context *sc) {
+    long err = 0;
+    bpf_printk("Save data");
+    err = bpf_map_update_elem(&tracked_spans, &contextContext, sc, BPF_ANY);
+    if (err != 0)
+    {
+        bpf_printk("Failed to update tracked_spans map: %ld", err);
+        return;
+    }
+//    struct apm_trace_key_t key = {};
+//    key.tgid = 11;
+//    key.goid = 22;
+//    err = bpf_map_update_elem(&tracked_spans_apm, &key, sc, BPF_ANY);
+//    if (err != 0)
+//    {
+//        bpf_printk("Failed to update tracked_spans map: %ld", err);
+//        return;
+//    }
+
+    err = bpf_map_update_elem(&tracked_spans_by_sc, sc, &contextContext, BPF_ANY);
+    if (err != 0)
+    {
+        bpf_printk("Failed to update tracked_spans_by_sc map: %ld", err);
+        return;
+    }
+}
+
+static __always_inline void stop_tracking_span(struct span_context *sc, struct span_context *psc) {
+    if (sc == NULL) {
+        bpf_printk("stop_tracking_span: sc is null");
+        return;
+    }
+
+    void *ctx = bpf_map_lookup_elem(&tracked_spans_by_sc, sc);
+    if (ctx == NULL)
+    {
+        bpf_printk("stop_tracking_span: cant find span context");
+        return;
+    }
+
+    void *parent_ctx = ((psc == NULL) ? NULL : bpf_map_lookup_elem(&tracked_spans_by_sc, psc));
+    if (parent_ctx == NULL)
+    {
+        // No parent span, delete the context
+        bpf_map_delete_elem(&tracked_spans, ctx);
+    } else 
+    {
+        void *ctx_val = 0;
+        bpf_probe_read_user(&ctx_val, sizeof(ctx_val), ctx);
+        void *parent_ctx_val = 0;
+        bpf_probe_read_user(&parent_ctx_val, sizeof(parent_ctx_val), parent_ctx);
+
+        if (ctx_val != parent_ctx_val)
+        {
+            // Parent with different context, delete the context
+            bpf_map_delete_elem(&tracked_spans, ctx);
+        } else {
+            // Parent with the same context, update the entry to point to the parent span
+            bpf_map_update_elem(&tracked_spans, ctx, psc, BPF_ANY);
+        }
+    }
+
+    bpf_map_delete_elem(&tracked_spans_by_sc, sc);
+}
+
+// Extract the go context.Context data pointer from the function arguments
+// context_pos:
+    // The argument position of the context.Context data pointer
+    // In case the context.Context is passed as an argument,
+    // this is the argument index of the pointer (starting from 1).
+    // In case the context.Context is a member of a struct,
+    // this is the argument index of the struct pointer (starting from 1). 
+// context_offset:
+    // In case the context.Context is a member of a struct,
+    // this is the offset of the context.Context member in the struct.
+// passed_as_arg:
+    // Indicates whether context.Context is passed as an argument or is a member of a struct
+static __always_inline void *get_Go_context(void *ctx, int context_pos, const volatile u64 context_offset, bool passed_as_arg) {
+    void *arg = get_argument(ctx, context_pos);
+    if (passed_as_arg) {
+        return arg;
+    }
+    void *ctx_addr = get_go_interface_instance((void*)(arg + context_offset));
+    void *ctx_val = 0;
+    bpf_probe_read_user(&ctx_val, sizeof(ctx_val), ctx_addr);
+    return ctx_val;
+}
+
+#endif

+ 121 - 0
ebpftracer/ebpf/utrace/go/include/span_context.h

@@ -0,0 +1,121 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef _SPAN_CONTEXT_H_
+#define _SPAN_CONTEXT_H_
+
+#include "utils.h"
+
+#define SPAN_CONTEXT_STRING_SIZE 55
+#define W3C_KEY_LENGTH 11 // length of the "traceparent" key
+#define W3C_VAL_LENGTH 55
+
+struct span_context {
+	unsigned char TraceID[TRACE_ID_SIZE];
+	unsigned char SpanID[SPAN_ID_SIZE];
+};
+
+static __always_inline struct span_context generate_span_context() {
+	struct span_context context = {};
+	generate_random_bytes(context.TraceID, TRACE_ID_SIZE);
+	generate_random_bytes(context.SpanID, SPAN_ID_SIZE);
+	return context;
+}
+
+static __always_inline void span_context_to_w3c_string(struct span_context *ctx, char *buff) {
+	// W3C format: version (2 chars) - trace id (32 chars) - span id (16 chars) - sampled (2 chars)
+	char *out = buff;
+
+	// Write version
+	*out++ = '0';
+	*out++ = '0';
+	*out++ = '-';
+
+	// Write trace id
+	bytes_to_hex_string(ctx->TraceID, TRACE_ID_SIZE, out);
+	out += TRACE_ID_STRING_SIZE;
+	*out++ = '-';
+
+	// Write span id
+	bytes_to_hex_string(ctx->SpanID, SPAN_ID_SIZE, out);
+	out += SPAN_ID_STRING_SIZE;
+	*out++ = '-';
+
+	// Write sampled
+	*out++ = '0';
+	*out = '1';
+}
+
+#define MAX_SIZE 120
+#define MAX_TOKENS 8
+
+struct Token {
+	u32 start;
+	u32 len;
+};
+struct cw_header_token {
+	struct Token token[MAX_TOKENS];
+	u32 start[MAX_TOKENS];
+	u32 a;
+};
+
+static __always_inline void w3c_string_to_span_context(char *str, struct span_context *ctx) {
+//	GO:0:10154813500555812:5450531005555981:5610250100539899:304775019cd3218a:1001025098564810:140acc88cde8773f:110
+//	00-95b5ec2b81e2374a4a27ce36ab71d349-18f65a5a3ab22213-02
+	u32 trace_id_start_pos = 3;
+	u32 span_id_start_pod = 36;
+//
+//	u32 idx = 0;
+//	u32 strLen = 0;
+//	struct cw_header_token range = {0};
+//	for (u32 k = 0; k < MAX_SIZE; ++k) {
+//		if (str[k] == '\0') {
+//			break;
+//		}
+////		bpf_printk("k---%d", k);
+//		if (str[k] == ':') {
+//			strLen = 0;
+//			range.start[idx] = k + 1;
+//			bpf_printk("range.start:%d", range.start[idx]);
+//			idx++;
+//		}
+////		bpf_printk("range.len token[%d]:[%d]", token, strLen);
+//	}
+//
+//	for (u32 i = 0; i < idx; ++i) {
+//		bpf_printk("Token %u: start = %u, len = %u\n", i, range.start[i], range.len[i]);
+//	}
+
+//	struct Range range2;
+////
+//	bpf_probe_read(&range2, sizeof(range2), &range);
+//	bpf_printk("traceparent_header_value_go_str.str:%s", range2.len);
+
+//	if (ret != 0) {
+//		bpf_printk("Failed to read Range struct: %d\n", ret);
+//	} else {
+//
+//		for (int i = 1; i < 9; ++i) {
+//			if (i > 1) {
+//				range2.len[i] = range2.start[i - 1];
+//			}
+//		}
+//	}
+
+
+	hex_string_to_bytes(str + trace_id_start_pos, TRACE_ID_STRING_SIZE, ctx->TraceID);
+	hex_string_to_bytes(str + span_id_start_pod, SPAN_ID_STRING_SIZE, ctx->SpanID);
+}
+
+#endif

+ 279 - 0
ebpftracer/ebpf/utrace/go/net/client.probe.bpf.c

@@ -0,0 +1,279 @@
+#include "arguments.h"
+#include "span_context.h"
+#include "go_context.h"
+#include "go_types.h"
+#include "uprobe.h"
+
+//char __license[] SEC("license") = "Dual MIT/GPL";
+
+#define MAX_HOSTNAME_SIZE 256
+#define MAX_PROTO_SIZE 8
+#define MAX_PATH_SIZE 100
+#define MAX_METHOD_SIZE 10
+#define MAX_CONCURRENT 50
+
+struct http_request_t {
+    BASE_SPAN_PROPERTIES
+    char host[MAX_HOSTNAME_SIZE];
+    char proto[MAX_PROTO_SIZE];
+    u64 status_code;
+    char method[MAX_METHOD_SIZE];
+    char path[MAX_PATH_SIZE];
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__type(key, void*);
+	__type(value, struct http_request_t);
+	__uint(max_entries, MAX_CONCURRENT);
+} http_events SEC(".maps");
+
+//struct {
+//	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+//	__uint(key_size, sizeof(u32));
+//	__uint(value_size, sizeof(struct map_bucket));
+//	__uint(max_entries, 1);
+//} golang_mapbucket_storage_map SEC(".maps");
+
+struct
+{
+    __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+    __uint(key_size, sizeof(u32));
+    __uint(value_size, sizeof(struct http_request_t));
+    __uint(max_entries, 1);
+} http_client_uprobe_storage_map SEC(".maps");
+
+//struct {
+//	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+//} events SEC(".maps");
+
+// Injected in init
+volatile const u64 method_ptr_pos;
+volatile const u64 url_ptr_pos;
+volatile const u64 path_ptr_pos;
+volatile const u64 headers_ptr_pos;
+volatile const u64 ctx_ptr_pos;
+volatile const u64 buckets_ptr_pos;
+volatile const u64 status_code_pos;
+volatile const u64 request_host_pos;
+volatile const u64 request_proto_pos;
+
+static __always_inline long inject_header(void* headers_ptr, struct span_context* propagated_ctx) {
+    // Read the key-value count - this field must be the first one in the hmap struct as documented in src/runtime/map.go
+    u64 curr_keyvalue_count = 0;
+    long res = bpf_probe_read_user(&curr_keyvalue_count, sizeof(curr_keyvalue_count), headers_ptr);
+
+    if (res < 0) {
+        bpf_printk("Couldn't read map key-value count from user");
+        return -1;
+    }
+
+    if (curr_keyvalue_count >= 8) {
+        bpf_printk("Map size is bigger than 8, skipping context propagation");
+        return -1;
+    }
+
+    // Get pointer to temp bucket struct we store in a map (avoiding large local variable on the stack)
+    // Performing read-modify-write on the bucket
+    u32 map_id = 0;
+    struct map_bucket *bucket_map_value = bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id);
+    if (!bucket_map_value) {
+        return -1;
+    }
+
+    void *buckets_ptr_ptr = (void*) (headers_ptr + buckets_ptr_pos);
+    void *bucket_ptr = 0; // The actual pointer to the buckets
+
+    if (curr_keyvalue_count == 0) {
+        // No key-value pairs in the Go map, need to "allocate" memory for the user
+        bucket_ptr = write_target_data(bucket_map_value, sizeof(struct map_bucket));
+        if (bucket_ptr == NULL) {
+            bpf_printk("inject_header: Failed to write bucket to user");
+            return -1;
+        }
+        // Update the buckets pointer in the hmap struct to point to newly allocated bucket
+        res = bpf_probe_write_user(buckets_ptr_ptr, &bucket_ptr, sizeof(bucket_ptr));
+        if (res < 0) {
+            bpf_printk("Failed to update the map bucket pointer for the user");
+            return -1;
+        }
+    } else {
+        // There is at least 1 key-value pair, hence the bucket pointer from the user is valid
+        // Read the bucket pointer
+        res = bpf_probe_read_user(&bucket_ptr, sizeof(bucket_ptr), buckets_ptr_ptr);
+        // Read the user's bucket to the eBPF map entry
+        bpf_probe_read_user(bucket_map_value, sizeof(struct map_bucket), bucket_ptr);
+    }
+
+    u8 bucket_index = curr_keyvalue_count & 0x7;
+
+    char traceparent_tophash = 0xee;
+    bucket_map_value->tophash[bucket_index] = traceparent_tophash;
+
+    // Prepare the key string for the user
+    char key[W3C_KEY_LENGTH] = "traceparent";
+    void *ptr = write_target_data(key, W3C_KEY_LENGTH);
+    if (ptr == NULL) {
+        bpf_printk("inject_header: Failed to write key to user");
+        return -1;
+    }
+    bucket_map_value->keys[bucket_index] = (struct go_string_ot) {.len = W3C_KEY_LENGTH, .str = ptr};
+
+    // Prepare the value string slice
+    // First the value string which constains the span context
+    char val[W3C_VAL_LENGTH];
+    span_context_to_w3c_string(propagated_ctx, val);
+    ptr = write_target_data(val, sizeof(val));
+    if(ptr == NULL) {
+        bpf_printk("inject_header: Failed to write value to user");
+        return -1;
+    }
+
+    // The go string pointing to the above val
+    struct go_string_ot header_value = {.len = W3C_VAL_LENGTH, .str = ptr};
+    ptr = write_target_data((void*)&header_value, sizeof(header_value));
+    if(ptr == NULL) {
+        bpf_printk("inject_header: Failed to write go_string to user");
+        return -1;
+    }
+
+    // Last, go_slice pointing to the above go_string
+    bucket_map_value->values[bucket_index] = (struct go_slice_ot) {.array = ptr, .cap = 1, .len = 1};
+
+    // Update the map header count field
+    curr_keyvalue_count += 1;
+    res = bpf_probe_write_user(headers_ptr, &curr_keyvalue_count, sizeof(curr_keyvalue_count));
+    if (res < 0) {
+        bpf_printk("Failed to update key-value count in map header");
+        return -1;
+    }
+
+    // Update the bucket
+    res = bpf_probe_write_user(bucket_ptr, bucket_map_value, sizeof(struct map_bucket));
+    if (res < 0) {
+        bpf_printk("Failed to update bucket content");
+        return -1;
+    }
+
+    bpf_memset((unsigned char *)bucket_map_value, sizeof(struct map_bucket), 0);
+    return 0;
+}
+
+// This instrumentation attaches uprobe to the following function:
+// func net/http/transport.roundTrip(req *Request) (*Response, error)
+SEC("uprobe/Transport_roundTrip")
+int uprobe_Transport_roundTrip(struct pt_regs *ctx) {
+    bpf_printk("[HTTP Client] start");
+    u64 request_pos = 2;
+    void *req_ptr = get_argument(ctx, request_pos);
+
+    // Get parent if exists
+    void *context_ptr_val = get_Go_context(ctx, 2, ctx_ptr_pos, false);
+    if (context_ptr_val == NULL)
+    {
+        return 0;
+    }
+    void *key = get_consistent_key(ctx, context_ptr_val);
+    void *httpReq_ptr = bpf_map_lookup_elem(&http_events, &key);
+    if (httpReq_ptr != NULL)
+    {
+        bpf_printk("uprobe/Transport_RoundTrip already tracked with the current context");
+        return 0;
+    }
+
+    u32 map_id = 0;
+    struct http_request_t *httpReq = bpf_map_lookup_elem(&http_client_uprobe_storage_map, &map_id);
+    if (httpReq == NULL)
+    {
+        bpf_printk("uprobe/Transport_roundTrip: httpReq is NULL");
+        return 0;
+    }
+
+    __builtin_memset(httpReq, 0, sizeof(struct http_request_t));
+    httpReq->start_time = bpf_ktime_get_ns();
+
+    struct span_context *parent_span_ctx = get_parent_span_context(context_ptr_val);
+    if (parent_span_ctx != NULL) {
+        bpf_printk("[Client] parent_span_ctx != NULL");
+
+        bpf_probe_read(&httpReq->psc, sizeof(httpReq->psc), parent_span_ctx);
+        copy_byte_arrays(httpReq->psc.TraceID, httpReq->sc.TraceID, TRACE_ID_SIZE);
+        generate_random_bytes(httpReq->sc.SpanID, SPAN_ID_SIZE);
+    } else {
+        bpf_printk("[Client] parent_span_ctx == NULL");
+	    struct apm_span_context *apm_sc = get_tracking_span();
+	    if (!apm_sc) {
+		    httpReq->sc = generate_span_context();
+		} else {
+		    httpReq->sc = *(struct span_context *) apm_sc;
+	    }
+    }
+
+    if (!get_go_string_from_user_ptr((void *)(req_ptr+method_ptr_pos), httpReq->method, sizeof(httpReq->method))) {
+        bpf_printk("uprobe_Transport_roundTrip: Failed to get method from request");
+        return 0;
+    }
+
+    // get path from Request.URL
+    void *url_ptr = 0;
+    bpf_probe_read(&url_ptr, sizeof(url_ptr), (void *)(req_ptr+url_ptr_pos));
+    if (!get_go_string_from_user_ptr((void *)(url_ptr+path_ptr_pos), httpReq->path, sizeof(httpReq->path))) {
+        bpf_printk("uprobe_Transport_roundTrip: Failed to get path from Request.URL");
+        return 0;
+    }
+
+    // get host from Request
+    if (!get_go_string_from_user_ptr((void *)(req_ptr+request_host_pos), httpReq->host, sizeof(httpReq->host))) {
+        bpf_printk("uprobe_Transport_roundTrip: Failed to get host from Request");
+    }
+
+    // get proto from Request
+    if (!get_go_string_from_user_ptr((void *)(req_ptr+request_proto_pos), httpReq->proto, sizeof(httpReq->proto))) {
+        bpf_printk("uprobe_Transport_roundTrip: Failed to get proto from Request");
+    }
+
+    // get headers from Request
+    void *headers_ptr = 0;
+    bpf_probe_read(&headers_ptr, sizeof(headers_ptr), (void *)(req_ptr+headers_ptr_pos));
+    long res = inject_header(headers_ptr, &httpReq->sc);
+    if (res < 0) {
+        bpf_printk("uprobe_Transport_roundTrip: Failed to inject header");
+    }
+
+    // Write event
+    bpf_map_update_elem(&http_events, &key, httpReq, 0);
+    start_tracking_span(context_ptr_val, &httpReq->sc);
+    return 0;
+}
+
+// This instrumentation attaches uretprobe to the following function:
+// func net/http/transport.roundTrip(req *Request) (*Response, error)
+SEC("uprobe/Transport_roundTrip")
+int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) {
+    bpf_printk("[HTTP Client] Return start");
+
+    u64 end_time = bpf_ktime_get_ns();
+    void *req_ctx_ptr = get_Go_context(ctx, 2, ctx_ptr_pos, false);
+    void *key = get_consistent_key(ctx, req_ctx_ptr);
+
+    struct http_request_t *http_req_span = bpf_map_lookup_elem(&http_events, &key);
+    if (http_req_span == NULL) {
+        bpf_printk("probe_Transport_roundTrip_Returns: entry_state is NULL");
+        return 0;
+    }
+
+    if (is_register_abi()) {
+        // Getting the returned response
+        void *resp_ptr = get_argument(ctx, 1);
+        // Get status code from response
+        bpf_probe_read(&http_req_span->status_code, sizeof(http_req_span->status_code), (void *)(resp_ptr + status_code_pos));
+    }
+
+    http_req_span->end_time = end_time;
+
+//    bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, http_req_span, sizeof(*http_req_span));
+    stop_tracking_span(&http_req_span->sc, &http_req_span->psc);
+
+    bpf_map_delete_elem(&http_events, &key);
+    return 0;
+}

+ 442 - 0
ebpftracer/ebpf/utrace/go/net/server.probe.bpf.c

@@ -0,0 +1,442 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "arguments.h"
+#include "span_context.h"
+#include "go_context.h"
+#include "go_types.h"
+#include "uprobe.h"
+
+//char __license[] SEC("license") = "Dual MIT/GPL";
+
+#define PATH_MAX_LEN 128
+#define MAX_BUCKETS 8
+#define METHOD_MAX_LEN 8
+#define MAX_CONCURRENT 50
+#define REMOTE_ADDR_MAX_LEN 256
+#define HOST_MAX_LEN 256
+#define PROTO_MAX_LEN 8
+
+struct http_server_span_t {
+	BASE_SPAN_PROPERTIES
+	u64 status_code;
+	char method[METHOD_MAX_LEN];
+	char path[PATH_MAX_LEN];
+	char remote_addr[REMOTE_ADDR_MAX_LEN];
+	char host[HOST_MAX_LEN];
+	char proto[PROTO_MAX_LEN];
+};
+
+struct uprobe_data_t {
+	struct http_server_span_t span;
+	// bpf2go doesn't support pointers fields
+	// saving the response pointer in the entry probe
+	// and using it in the return probe
+	u64 resp_ptr;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__type(key, void *);
+	__type(value, struct uprobe_data_t);
+	__uint(max_entries, MAX_CONCURRENT);
+} http_server_uprobes SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__uint(key_size, sizeof(u32));
+	__uint(value_size, sizeof(struct map_bucket));
+	__uint(max_entries, 1);
+} golang_mapbucket_storage_map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__uint(key_size, sizeof(u32));
+	__uint(value_size, sizeof(struct cw_header_token));
+	__uint(max_entries, 1);
+} header_range SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__uint(key_size, sizeof(u32));
+	__uint(value_size, sizeof(struct span_context));
+	__uint(max_entries, 1);
+} parent_span_context_storage_map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__uint(key_size, sizeof(u32));
+	__uint(value_size, sizeof(struct uprobe_data_t));
+	__uint(max_entries, 1);
+} http_server_uprobe_storage_map SEC(".maps");
+
+//struct
+//{
+//	__uint(type, BPF_MAP_TYPE_LRU_HASH);
+//	__uint(key_size, sizeof(struct apm_trace_key_t));
+//	__uint(value_size, sizeof(struct span_context));
+//	__uint(max_entries, 1);
+//} apm_span_context_map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+} events SEC(".maps");
+
+// Injected in init
+volatile const u64 method_ptr_pos;
+volatile const u64 url_ptr_pos;
+volatile const u64 path_ptr_pos;
+volatile const u64 ctx_ptr_pos;
+volatile const u64 headers_ptr_pos;
+volatile const u64 buckets_ptr_pos;
+volatile const u64 req_ptr_pos;
+volatile const u64 status_code_pos;
+volatile const u64 remote_addr_pos;
+volatile const u64 host_pos;
+volatile const u64 proto_pos;
+//for (int i = 0; i < W3C_KEY_LENGTH; i++)
+//{
+//if (current_header_key[i] != "traceparent"[i]){
+//goto outer_loop;
+//}
+//if (current_header_key[i] != "Traceparent"[i]){
+//goto outer_loop;
+//}
+//}
+
+static __always_inline struct span_context *extract_context_from_req_headers(void *headers_ptr_ptr) {
+	void *headers_ptr;
+	long res;
+	res = bpf_probe_read(&headers_ptr, sizeof(headers_ptr), headers_ptr_ptr);
+	if (res < 0) {
+		return NULL;
+	}
+	u64 headers_count = 0;
+	res = bpf_probe_read(&headers_count, sizeof(headers_count), headers_ptr);
+	if (res < 0) {
+		return NULL;
+	}
+	if (headers_count == 0) {
+		return NULL;
+	}
+	unsigned char log_2_bucket_count;
+	res = bpf_probe_read(&log_2_bucket_count, sizeof(log_2_bucket_count), headers_ptr + 9);
+	if (res < 0) {
+		return NULL;
+	}
+	u64 bucket_count = 1 << log_2_bucket_count;
+	void *header_buckets;
+	res = bpf_probe_read(&header_buckets, sizeof(header_buckets), (void *) (headers_ptr + buckets_ptr_pos));
+	if (res < 0) {
+		return NULL;
+	}
+	u32 map_id = 0;
+	struct map_bucket *map_value = bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id);
+	if (!map_value) {
+		return NULL;
+	}
+	for (u32 j = 0; j < MAX_BUCKETS; j++) {
+		if (j >= bucket_count) {
+			break;
+		}
+		res = bpf_probe_read(map_value, sizeof(struct map_bucket), header_buckets + (j * sizeof(struct map_bucket)));
+		if (res < 0) {
+			continue;
+		}
+		for (u32 i = 0; i < 8; i++) {
+			if (map_value->tophash[i] == 0) {
+				continue;
+			}
+			if (map_value->keys[i].len != W3C_KEY_LENGTH) {
+				continue;
+			}
+			char current_header_key[W3C_KEY_LENGTH];
+			bpf_probe_read(current_header_key, sizeof(current_header_key), map_value->keys[i].str);
+			for (int i = 0; i < W3C_KEY_LENGTH; i++) {
+				if (current_header_key[i] != "Traceparent"[i] && current_header_key[i] != "traceparent"[i]) {
+					goto outer_loop;
+				}
+			}
+//			if (!bpf_memcmp(current_header_key, "traceparent", W3C_KEY_LENGTH) && !bpf_memcmp(current_header_key, "Traceparent", W3C_KEY_LENGTH))
+//			{
+//				bpf_printk("not find it");
+//				continue;
+//			}
+//			bpf_printk("find it:%s", map_value->keys[i].str);
+			void *traceparent_header_value_ptr = map_value->values[i].array;
+			struct go_string_ot traceparent_header_value_go_str;
+			res = bpf_probe_read(&traceparent_header_value_go_str, sizeof(traceparent_header_value_go_str),
+			                     traceparent_header_value_ptr);
+			// 00-95b5ec2b81e2374a4a27ce36ab71d349-18f65a5a3ab22213-02
+//			bpf_printk("traceparent_header_value_go_str.str:%s", traceparent_header_value_go_str.str);
+
+			if (res < 0) {
+				return NULL;
+			}
+//			if (traceparent_header_value_go_str.len != W3C_VAL_LENGTH) {
+//				continue;
+//			}
+			char traceparent_header_value[120];
+			res = bpf_probe_read(&traceparent_header_value, sizeof(traceparent_header_value),
+			                     traceparent_header_value_go_str.str);
+			// 00-95b5ec2b81e2374a4a27ce36ab71d349-18f65a5a3ab22213-02��c�����c����@c;
+//			bpf_printk("traceparent_header_value11:[%s]", traceparent_header_value);
+
+//			bpf_printk("111111111111111111:[%d]", range.start[1]);
+
+
+			if (res < 0) {
+				return NULL;
+			}
+
+			struct span_context *parent_span_context = bpf_map_lookup_elem(&parent_span_context_storage_map, &map_id);
+			if (!parent_span_context) {
+//				bpf_printk("no parent_span_context");
+				return NULL;
+			}
+//			bpf_printk("parent_span_context-TraceID1:%s", parent_span_context->TraceID);
+//			bpf_printk("parent_span_context-SpanID1:%s", parent_span_context->SpanID);
+//			struct cw_header_token *header_range_data = bpf_map_lookup_elem(&header_range, &map_id);
+//			if (!header_range_data) {
+//				return NULL;
+//			}
+//			u32 idx = 0;
+//			u32 str_len = 0;
+//			struct cw_header_token tmp = *header_range_data;
+//			tmp.a = 4;
+////			struct Token t = {.start=1,.len=2};
+//			for (int k = 0; k < MAX_TOKENS; ++k) {
+////				header_range_data->token[k] = t ;
+//				header_range_data->start[k] = 1 ;
+//
+////				tmp.token[k] = t ;
+//			}
+//			for (int k = 0; k < MAX_TOKENS; ++k) {
+////					struct Token t = {.start=1, .len=2};
+////					header_range_data->token[k] = t;
+//				bpf_printk("header_range_data->token---%d", header_range_data->token[k].len);
+//
+//			}
+//
+////			header_range_data->token[0].len = 1;
+//
+//			bpf_printk("header_range_data->token---%d", header_range_data->token[0].len);
+//
+//			for (u32 k = 0; k < MAX_SIZE; ++k) {
+//				if (traceparent_header_value[k] == '\0') {
+////					bpf_map_update_elem(&header_range, &map_id, &tmp, BPF_ANY);
+//					break;
+//				}
+//				bpf_printk("k---%d", k);
+//				if (traceparent_header_value[k] == ':') {
+//					str_len = 0;
+//					tmp.token[idx].start = k + 1;
+//					header_range_data->start[idx] = k + 1;
+//					bpf_printk("range.start:%d", tmp.token[idx].start);
+//					idx++;
+//				}
+////				else {
+////					str_len++;
+////				}
+////				tmp.token[idx].len = str_len;
+//				bpf_printk("range.len token[%d]:[%d]", idx, str_len);
+//			}
+
+
+//			for (int k = 0; k < MAX_TOKENS; ++k) {
+//
+////					struct Token tt[8] = tmp.token;
+////					header_range_data->token[k] = t;
+//				bpf_printk("header_range_data->token---%d", header_range_data->token[k].len);
+////				bpf_printk("header_range_data->token---%d", tmp.token[k].len);
+//			}
+//			for (int k = 0; k < MAX_TOKENS; ++k) {
+//
+////					struct Token tt[8] = tmp.token;
+////					header_range_data->token[k] = t;
+//				bpf_printk("header_range_data->token---%d", header_range_data->token[k].len);
+////				bpf_printk("header_range_data->token---%d", tmp.token[k].len);
+//			}
+
+//			if (idx == MAX_TOKENS) {
+//				for (int k = 0; k < idx; ++k) {
+////					struct Token t = {.start=1, .len=2};
+////					header_range_data->token[k] = t;
+//					bpf_printk("header_range_data->token---%d", header_range_data->token[k].len);
+//
+//				}
+//			}
+//			for (int k = 0; k < idx ; ++k) {
+//				struct Token t = {.start=1,.len=2};
+//				header_range_data->token[k]= t;
+//			}
+//			header_range_data = &tmp;
+
+
+//			bpf_printk("header_range_data---%d", header_range_data->a);
+//			bpf_printk("header_range_data->token---%d", header_range_data->token[0].len);
+//			__builtin_memcpy(&header_range_data->token, &tmp.token, sizeof(header_range_data->token));
+//			struct cw_header_token *header_range_data2 = bpf_map_lookup_elem(&header_range, &map_id);
+//			if (!header_range_data2) {
+//				return NULL;
+//			}
+//			bpf_printk("header_range_data2---%d", header_range_data2->a);
+
+//			for (int kl = 0; kl < idx; ++kl) {
+//				u32 start = tmp.token[kl].start;
+//				u32 len = tmp.token[kl].len;
+//				bpf_printk("Token [%d][%d]", kl,start);
+//			}
+			w3c_string_to_span_context(traceparent_header_value, parent_span_context);
+//			bpf_printk("traceparent_header_value2:%s", traceparent_header_value);
+//            bpf_printk("parent_span_context-TraceID: ");
+//            for (int i = 0; i < TRACE_ID_SIZE; i++) {
+//                bpf_printk("%02x", parent_span_context->TraceID[i]);
+//            }
+//			bpf_printk("parent_span_context-TraceID2:%s", parent_span_context->TraceID);
+//			bpf_printk("parent_span_context-SpanID2:%s", parent_span_context->SpanID);
+			return parent_span_context;
+			outer_loop:
+			continue;
+		}
+	}
+	return NULL;
+}
+
+// This instrumentation attaches uprobe to the following function:
+// func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request)
+SEC("uprobe/HandlerFunc_ServeHTTP")
+int uprobe_HandlerFunc_ServeHTTP(struct pt_regs *ctx) {
+	bpf_printk("[uprobe_HandlerFunc_ServeHTTP]");
+	void *req_ctx_ptr = get_Go_context(ctx, 4, ctx_ptr_pos, false);
+	void *key = get_consistent_key(ctx, req_ctx_ptr);
+	void *httpReq_ptr = bpf_map_lookup_elem(&http_server_uprobes, &key);
+	if (httpReq_ptr != NULL) {
+		bpf_printk("uprobe/HandlerFunc_ServeHTTP already tracked with the current request");
+		return 0;
+	}
+
+	u32 map_id = 0;
+	struct uprobe_data_t *uprobe_data = bpf_map_lookup_elem(&http_server_uprobe_storage_map, &map_id);
+	if (uprobe_data == NULL) {
+		bpf_printk("uprobe/HandlerFunc_ServeHTTP: http_server_span is NULL");
+		return 0;
+	}
+
+	__builtin_memset(uprobe_data, 0, sizeof(struct uprobe_data_t));
+
+	// Save response writer
+	void *resp_impl = get_argument(ctx, 3);
+	uprobe_data->resp_ptr = (u64) resp_impl;
+
+	struct http_server_span_t *http_server_span = &uprobe_data->span;
+	http_server_span->start_time = bpf_ktime_get_ns();
+
+	// Propagate context
+	void *req_ptr = get_argument(ctx, 4);
+	struct span_context *parent_ctx = extract_context_from_req_headers((void *) (req_ptr + headers_ptr_pos));
+	if (parent_ctx != NULL) {
+		// found parent context in http headers
+		http_server_span->psc = *parent_ctx;
+		copy_byte_arrays(http_server_span->psc.TraceID, http_server_span->sc.TraceID, TRACE_ID_SIZE);
+		generate_random_bytes(http_server_span->sc.SpanID, SPAN_ID_SIZE);
+	} else {
+		http_server_span->sc = generate_span_context();
+	}
+
+	if (req_ctx_ptr == NULL) {
+//		bpf_printk("uprobe/HandlerFunc_ServeHTTP: req_ctx_ptr is NULL");
+		return 0;
+	}
+//	bpf_printk("found parent context in http headers");
+
+//	bpf_printk("parent_span_context-TraceID---");
+
+
+//    for (int i = 0; i < TRACE_ID_SIZE; i++) {
+//        bpf_printk("%02x", uprobe_data->span.sc.TraceID[i]);
+//    }
+
+//    bpf_printk("parent_span_context-SpanID---");
+//    for (int i = 0; i < SPAN_ID_SIZE; i++) {
+//        bpf_printk("%02x", uprobe_data->span.sc.SpanID[i]);
+//    }
+
+//	bpf_map_update_elem(&http_server_uprobes, &key, uprobe_data, 0);
+//	start_tracking_span(req_ctx_ptr, &http_server_span->sc);
+
+//	void save_tracking_span(struct apm_span_context *sc)
+	struct apm_span_context apm_sc = *(struct apm_span_context *) &http_server_span->sc;
+	save_tracking_span(&apm_sc);
+
+	return 0;
+}
+
+void read_go_string(void *base, int offset, char *output, int maxLen, const char *errorMsg) {
+	void *ptr = (void *) (base + offset);
+	if (!get_go_string_from_user_ptr(ptr, output, maxLen)) {
+		bpf_printk("Failed to get %s", errorMsg);
+	}
+
+}
+
+// This instrumentation attaches uprobe to the following function:
+// func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request)
+SEC("uprobe/HandlerFunc_ServeHTTP")
+int uprobe_HandlerFunc_ServeHTTP_Returns(struct pt_regs *ctx) {
+	bpf_printk("[uprobe_HandlerFunc_ServeHTTP_Returns]");
+	u64 end_time = bpf_ktime_get_ns();
+	void *req_ctx_ptr = get_Go_context(ctx, 4, ctx_ptr_pos, false);
+	void *key = get_consistent_key(ctx, req_ctx_ptr);
+
+	struct uprobe_data_t *uprobe_data = bpf_map_lookup_elem(&http_server_uprobes, &key);
+	if (uprobe_data == NULL) {
+		bpf_printk("uprobe/HandlerFunc_ServeHTTP_Returns: entry_state is NULL");
+		return 0;
+	}
+	bpf_map_delete_elem(&http_server_uprobes, &key);
+
+	struct http_server_span_t *http_server_span = &uprobe_data->span;
+	void *resp_ptr = (void *) uprobe_data->resp_ptr;
+	void *req_ptr = NULL;
+	bpf_probe_read(&req_ptr, sizeof(req_ptr), (void *) (resp_ptr + req_ptr_pos));
+
+	http_server_span->end_time = end_time;
+
+	void *url_ptr = 0;
+	bpf_probe_read(&url_ptr, sizeof(url_ptr), (void *) (req_ptr + url_ptr_pos));
+	// Collect fields from response
+	read_go_string(req_ptr, method_ptr_pos, http_server_span->method, sizeof(http_server_span->method),
+	               "method from request");
+	read_go_string(url_ptr, path_ptr_pos, http_server_span->path, sizeof(http_server_span->path),
+	               "path from Request.URL");
+	read_go_string(req_ptr, remote_addr_pos, http_server_span->remote_addr, sizeof(http_server_span->remote_addr),
+	               "remote addr from Request.RemoteAddr");
+	read_go_string(req_ptr, host_pos, http_server_span->host, sizeof(http_server_span->host), "host from Request.Host");
+	read_go_string(req_ptr, proto_pos, http_server_span->proto, sizeof(http_server_span->proto),
+	               "proto from Request.Proto");
+
+	// status code
+	bpf_probe_read(&http_server_span->status_code, sizeof(http_server_span->status_code),
+	               (void *) (resp_ptr + status_code_pos));
+
+	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, http_server_span, sizeof(*http_server_span));
+	stop_tracking_span(&http_server_span->sc, &http_server_span->psc);
+
+	struct apm_trace_key_t trace_key = get_apm_trace_key(120 * NS_PER_SEC, true);
+	bpf_map_delete_elem(&apm_span_context_map, &trace_key);
+
+	bpf_printk("HTTP_END");
+	return 0;
+}

+ 40 - 5
ebpftracer/tls.go

@@ -25,6 +25,10 @@ const (
 	minSupportedGoVersion = "v1.17.0"
 	goTlsWriteSymbol      = "crypto/tls.(*Conn).Write"
 	goTlsReadSymbol       = "crypto/tls.(*Conn).Read"
+	goExecute             = "runtime.execute"
+	goNewproc1            = "runtime.newproc1"
+	goServeHTTP           = "net/http.serverHandler.ServeHTTP"
+	goTransport           = "net/http.(*Transport).roundTrip"
 )
 
 var (
@@ -228,8 +232,8 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 			continue
 		}
 		switch s.Name {
-		//case goTlsWriteSymbol, goTlsReadSymbol, "runtime.execute", "runtime.newproc1":
-		case "runtime.execute", "runtime.newproc1", "net/http.serverHandler.ServeHTTP":
+		//case goTlsWriteSymbol, goTlsReadSymbol:
+		case goExecute, goNewproc1, goServeHTTP, goTransport:
 		default:
 			continue
 		}
@@ -247,7 +251,7 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 		fmt.Println("s.Name-----:", s.Name)
 
 		switch s.Name {
-		case "runtime.execute":
+		case goExecute:
 			l, err := exe.Uprobe(s.Name, t.uprobes["runtime_execute"], &link.UprobeOptions{Address: address})
 			if err != nil {
 				log("failed to attach write_enter uprobe", err)
@@ -258,7 +262,7 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 			}
 			links = append(links, l)
 
-		case "runtime.newproc1":
+		case goNewproc1:
 			l, err := exe.Uprobe(s.Name, t.uprobes["enter_runtime_newproc1"], &link.UprobeOptions{Address: address})
 			if err != nil {
 				log("failed to attach newproc1 uprobe", err)
@@ -284,7 +288,7 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 				}
 				links = append(links, l)
 			}
-		case "net/http.serverHandler.ServeHTTP":
+		case goServeHTTP:
 			l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_HandlerFunc_ServeHTTP"], &link.UprobeOptions{Address: address})
 			if err != nil {
 				log("failed to attach write_enter uprobe", err)
@@ -315,6 +319,37 @@ func (t *Tracer) AttachGoTlsUprobes(pid uint32) []link.Link {
 				links = append(links, l)
 			}
 
+		case goTransport:
+			l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_Transport_roundTrip"], &link.UprobeOptions{Address: address})
+			if err != nil {
+				log("failed to attach write_enter uprobe", err)
+				fmt.Println("net/http.uprobe_Transport_roundTrip no")
+				fmt.Println(err)
+				continue
+			} else {
+				fmt.Println("net/http.uprobe_Transport_roundTrip ok")
+			}
+			links = append(links, l)
+			sStart := s.Value - textSection.Addr
+			sEnd := sStart + s.Size
+			if sEnd > textSectionLen {
+				continue
+			}
+			sBytes := textSectionData[sStart:sEnd]
+			returnOffsets := getReturnOffsets(ef.Machine, sBytes)
+			if len(returnOffsets) == 0 {
+				log("failed to attach uprobe_Transport_roundTrip uprobe", fmt.Errorf("no return offsets found"))
+				return nil
+			}
+			for _, offset := range returnOffsets {
+				l, err := exe.Uprobe(s.Name, t.uprobes["uprobe_Transport_roundTrip_Returns"], &link.UprobeOptions{Address: address, Offset: uint64(offset)})
+				if err != nil {
+					log("failed to attach exit_runtime_newproc1 uprobe", err)
+					return nil
+				}
+				links = append(links, l)
+			}
+
 		case goTlsWriteSymbol:
 			l, err := exe.Uprobe(s.Name, t.uprobes["go_crypto_tls_write_enter"], &link.UprobeOptions{Address: address})
 			if err != nil {

+ 4 - 0
ebpftracer/tracer/socket.go

@@ -325,6 +325,10 @@ func SetConstants(collectionSpec *ebpf.CollectionSpec) {
 		"req_ptr_pos":      int64(8),
 		"status_code_pos":  int64(120),
 		"url_ptr_pos":      int64(16),
+		// todo in process
+		"total_cpus": int64(2),
+		"start_addr": int64(140143583477760),
+		"end_addr":   int64(140143583543296),
 	}
 	err := collectionSpec.RewriteConstants(consts)
 	fmt.Println("err----------------", err)

+ 6 - 5
pkg/go.opentelemetry.io/otel/exporters/otlp/otlptrace/apm_exporter.go

@@ -27,14 +27,15 @@ const (
 
 type apmTraceSpan tracesdk.ReadOnlySpan
 
+// GO:0:10154813500555812:5450531005555981:5610250100539899:ee022542c3940f1b:1001025098564810:888ceb3df1bdbe2c:110
 type RootDataT struct {
 	AccountId      int           `json:"account_id"`
 	AgentId        int64         `json:"agent_id"`
 	AgentVersion   string        `json:"agent_version"`
 	AppId          int64         `json:"app_id"`
-	AppIdFrom      int           `json:"app_id_from"`
+	AppIdFrom      int           `json:"app_id_from"` // from header
 	AppName        string        `json:"app_name"`
-	CalledId       int           `json:"called_id"`
+	CalledId       int           `json:"called_id"` // from header
 	ClientIp       string        `json:"client_ip"`
 	CollTime       uint64        `json:"coll_time"`
 	Cpu            int           `json:"cpu"`
@@ -44,7 +45,7 @@ type RootDataT struct {
 	HttpCode       int64         `json:"http_code"`
 	HttpMethod     string        `json:"http_method"`
 	InstanceId     int64         `json:"instance_id"`
-	InstanceIdFrom int           `json:"instance_id_from"`
+	InstanceIdFrom int           `json:"instance_id_from"` // from header
 	LocalPort      int64         `json:"local_port"`
 	Maps           []MapInfoT    `json:"maps"`
 	MemU           int           `json:"mem_u"`
@@ -59,11 +60,11 @@ type RootDataT struct {
 	ServiceType    string        `json:"service_type"`
 	Sip            string        `json:"sip"`
 	Sn             string        `json:"sn"`
-	SpanIdFrom     string        `json:"span_id_from"`
+	SpanIdFrom     string        `json:"span_id_from"` // from header
 	Sport          int64         `json:"sport"`
 	TId            int           `json:"t_id"`
 	TName          string        `json:"t_name"`
-	TraceId        string        `json:"trace_id"`
+	TraceId        string        `json:"trace_id"` // from header
 	TransIds       []interface{} `json:"trans_ids"`
 	TypeFrom       string        `json:"type_from"`
 	Uri            string        `json:"uri"`