Fixed #TASK_QT-9810 支持l7层关联tcp信息。

Makefile2

@@ -14,7 +14,7 @@ endif
 debug:
 	exit
 build:
-	CGO_ENABLED=1 go build -gcflags="all=-N -l" -buildvcs=false -o euspace
+	CGO_ENABLED=1 go build -gcflags="all=-N -l" -buildvcs=false -o dist/package_dir/bin/euspace
 c:
 	#docker exec -it 9d928d96d4d0 sh -c 'cd /opt/github/euspace/ebpftracer && sh build.sh${PARAMS}'
 	docker exec -it 0fec3217d6da sh -c 'cd /opt/github/euspace/ebpftracer && make all ${PARAMS}'

containers/metrics.go

@@ -101,9 +101,9 @@ var metrics = struct {
 	NetAcceptsSuccessful: 	  metric("process_net_tcp_successful_accept_total", "Total number of successful TCP accepts", "destination", "actual_destination"),
 	NetAcceptBytesSent:       metric("process_net_tcp_bytes_sent_accept_total", "Total number of bytes sent to the peer", "destination", "actual_destination"),
 	NetAcceptBytesReceived:   metric("process_net_tcp_bytes_received_accept_total", "Total number of bytes received from the peer", "destination", "actual_destination"),
-	NetDataLatency:      	  metric("process_net_tcp_data_latency", "Data latency", "instance_id","app_id","app_name","src","destination", "actual_destination"),
-	NetDataDuration:      	  metric("process_net_tcp_data_duration", "Data duration", "instance_id","app_id","app_name","src","destination", "actual_destination"),
-	NetEstTime:      	  	  metric("process_net_tcp_est_time", "Established time", "instance_id","app_id","app_name","src","destination", "actual_destination"),
+	NetDataLatency:      	  metric("process_net_tcp_data_latency_time", "Data latency", "instance_id","app_id","app_name","src","destination", "actual_destination"),
+	NetDataDuration:      	  metric("process_net_tcp_flow_duration_time", "Data duration", "instance_id","app_id","app_name","src","destination", "actual_destination"),
+	NetEstTime:      	  	  metric("process_net_tcp_connection_establish_time", "Established time", "instance_id","app_id","app_name","src","destination", "actual_destination"),
 	
 	LogMessages: metric("process_log_messages_total", "Number of messages grouped by the automatically extracted repeated pattern", "source", "level", "pattern_hash", "sample"),
 

dist/scripts/uninstall.sh

@@ -21,7 +21,7 @@ readonly BRAND_AGENT_DEFAULT_USER_AND_GROUP_NAME="cloudwise"
 #== **********************************************************
 #== smartagentd
 readonly AGENT_PROC="euspace"
-readonly AGENT_INSTALLER_VERSION=0.0.1
+readonly AGENT_INSTALLER_VERSION=1.0.0
 #== **********************************************************
 #==  配置目录
 #== **********************************************************

ebpftracer/ebpf/l7/l7.c

@@ -83,6 +83,10 @@ struct l7_event {
     __u16 dport;
     __u8 saddr[16];
     __u8 daddr[16];
+    __u16 component_sport;
+    __u16 component_dport;
+    __u8 component_saddr[16];
+    __u8 component_daddr[16];
 	unsigned char assumed_app_id[APM_ASSUMED_APP_ID_SIZE];
 	unsigned char span_id[APM_SPAN_ID_SIZE];
 	unsigned char trace_id_from[APM_TRACE_ID_SIZE];
@@ -907,6 +911,11 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 	if (e->protocol == PROTOCOL_HTTP) {
 		__u64 trace_id = req->trace_id;
 		e->trace_id = trace_id;
+        // cw_bpf_debug("l7.c addr is --------:%d,%s,%d,%s",conn->sport,conn->saddr,conn->dport,conn->daddr);
+        e->component_sport = conn->sport;
+        e->component_dport = conn->dport;
+        __builtin_memcpy(&e->component_saddr, &conn->saddr, sizeof(e->component_saddr));
+        __builtin_memcpy(&e->component_daddr, &conn->daddr, sizeof(e->component_daddr));
 //	    struct  apm_span_context * sc = cw_get_current_tracking_span();
 //	    if (sc) {
 		cw_copy_byte_arrays(req->assumed_app_id, e->assumed_app_id, APM_ASSUMED_APP_ID_SIZE);
@@ -938,6 +947,10 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 		__u64 trace_id = get_apm_trace_id(pid, tid);
 		cw_bpf_debug("[Redis] trace_id:%llu", trace_id);
 		e->trace_id = trace_id;
+        e->component_sport = conn->sport;
+        e->component_dport = conn->dport;
+        __builtin_memcpy(&e->component_saddr, &conn->saddr, sizeof(e->component_saddr));
+        __builtin_memcpy(&e->component_daddr, &conn->daddr, sizeof(e->component_daddr));
 		response = is_redis_response(payload, ret, &e->status);
 	} else if (e->protocol == PROTOCOL_MEMCACHED) {
 		response = is_memcached_response(payload, ret, &e->status);
@@ -946,6 +959,10 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 		__u64 trace_id = get_apm_trace_id(pid, tid);
 //        cw_bpf_debug("[Mysql] trace_id:%llu", trace_id);
 		e->trace_id = trace_id;
+        e->component_sport = conn->sport;
+        e->component_dport = conn->dport;
+        __builtin_memcpy(&e->component_saddr, &conn->saddr, sizeof(e->component_saddr));
+        __builtin_memcpy(&e->component_daddr, &conn->daddr, sizeof(e->component_daddr));
 		response = is_mysql_response(payload, ret, req->request_type, &e->statement_id, &e->status);
 		if (req->request_type == MYSQL_COM_STMT_PREPARE) {
 			e->method = METHOD_STATEMENT_PREPARE;
@@ -1080,6 +1097,10 @@ int trace_exit_read(void *ctx, __u64 id, __u32 pid, __u16 is_tls, long int ret)
 
 		__u64 trace_id = get_apm_trace_id(pid, tid);
 		e->trace_id = trace_id;
+        e->component_sport = conn->sport;
+        e->component_dport = conn->dport;
+        __builtin_memcpy(&e->component_saddr, &conn->saddr, sizeof(e->component_saddr));
+        __builtin_memcpy(&e->component_daddr, &conn->daddr, sizeof(e->component_daddr));
 //	    cw_bpf_debug("[Response][DM] trace_id:%llu", trace_id);
 
 		response = is_dm_response(payload, ret, req->request_type, &dm_ctx->status);

ebpftracer/ebpf/tcp/state.c

@@ -83,6 +83,10 @@ struct connection {
     __u64 first_read_time;
     __u64 first_write_time;
     __u64 new_read_time;
+    __u16 sport;
+    __u16 dport;
+    __u8 saddr[16];
+    __u8 daddr[16];
 };
 
 struct accept_connection {
@@ -138,6 +142,10 @@ int inet_sock_set_state(void *ctx)
         conn.first_read_time = 0;
         conn.first_write_time = 0;
         conn.new_read_time = 0;
+        conn.sport = args.sport;
+        conn.dport = args.dport;
+        __builtin_memcpy(&conn.saddr, &args.saddr_v6, sizeof(conn.saddr));
+        __builtin_memcpy(&conn.daddr, &args.daddr_v6, sizeof(conn.saddr));
 
         bpf_map_delete_elem(&fd_by_pid_tgid, &id);
         bpf_map_update_elem(&connection_id_by_socket, &args.skaddr, &cid, BPF_ANY);

ebpftracer/l7/l7.go

@@ -158,6 +158,8 @@ type RequestData struct {
 	EndAt             uint64
     SAddr          	  netaddr.IPPort
 	DAddr          	  netaddr.IPPort
+	ComponentSAddr    netaddr.IPPort
+	ComponentDAddr    netaddr.IPPort
 	ParentSpanContext struct {
 		TraceIdFrom    string
 		CalledId       string

ebpftracer/tracer.go

@@ -475,6 +475,10 @@ type l7Event struct {
     Dport				uint16
     SAddr          		[16]byte
 	DAddr          		[16]byte
+	ComponentSport		uint16
+    ComponentDport		uint16
+    ComponentSAddr      [16]byte
+	ComponentDAddr      [16]byte
 	AssumedAppId        HashByte
 	SpanId              HashByte
 	TraceIdFrom         HashByte16
@@ -708,6 +712,12 @@ func runEventsReader(name string, r *perf.Reader, ch chan<- Event, typ perfMapTy
 				SpanId:       hex.EncodeToString(v.SpanId[:]),
 				StartAt:      v.StartAt,
 				EndAt:        v.EndtAt,
+				ComponentSAddr: ipPort(v.SAddr,v.Sport),
+				ComponentDAddr: ipPort(v.DAddr,v.Dport),
+			}
+			if req.Protocol == l7.ProtocolHTTP{
+				klog.Infof("runEventsReader ComponentSAddr.String %s", req.ComponentSAddr.String())
+				klog.Infof("runEventsReader ComponentDAddr.String %s", req.ComponentDAddr.String())
 			}
 			if v.TraceEnd == 1 {
 				req.ParentSpanContext.TraceIdFrom = hex.EncodeToString(v.TraceIdFrom[:])

logs/release.log

@@ -0,0 +1,35 @@
+
+2024-12-05 02:42:07 UTC [INFO] package script run path: /root/code/euspace/dist/scripts
+2024-12-05 02:42:07 UTC [INFO] package logs path: /root/code/euspace/logs
+2024-12-05 02:42:07 UTC [INFO] -----params PARAM_RELEASE: false, PARAM_DEBUG: false, PARAM_VERSION: 
+2024-12-05 02:42:07 UTC [INFO] ---> edit BRAND_PRODUCT_NAME="euspace" to BRAND_PRODUCT_NAME="euspace", configFile: /root/code/euspace/dist/scripts/package.sh
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly BRAND_PRODUCT_NAME="euspace" to readonly BRAND_PRODUCT_NAME="euspace", configFile: /root/code/euspace/dist/scripts/install_temp.sh
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly BRAND_PRODUCT_NAME_LOWER="euspace" to readonly BRAND_PRODUCT_NAME_LOWER="euspace", configFile: /root/code/euspace/dist/scripts/install_temp.sh
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly BRAND_PRODUCT_NAME="euspace" to readonly BRAND_PRODUCT_NAME="euspace", configFile: /root/code/euspace/dist/package_dir/bin/agentctl
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly BRAND_PRODUCT_NAME_LOWER="euspace" to readonly BRAND_PRODUCT_NAME_LOWER="euspace", configFile: /root/code/euspace/dist/package_dir/bin/agentctl
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly AGENT_CONFIG_PATH="" to readonly AGENT_CONFIG_PATH="", configFile: /root/code/euspace/dist/package_dir/bin/agentctl
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly BRAND_PRODUCT_NAME="euspace" to readonly BRAND_PRODUCT_NAME="euspace", configFile: /root/code/euspace/dist/scripts/uninstall.sh
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly BRAND_PRODUCT_NAME_LOWER="euspace" to readonly BRAND_PRODUCT_NAME_LOWER="euspace", configFile: /root/code/euspace/dist/scripts/uninstall.sh
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly AGENT_PROC="euspace" to readonly AGENT_PROC="euspace", configFile: /root/code/euspace/dist/scripts/install_temp.sh
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly AGENT_PROC="euspace" to readonly AGENT_PROC="euspace", configFile: /root/code/euspace/dist/scripts/uninstall.sh
+2024-12-05 02:42:07 UTC [INFO] ---> edit readonly AGENT_PROC="euspace" to readonly AGENT_PROC="euspace", configFile: /root/code/euspace/dist/package_dir/bin/agentctl
+2024-12-05 02:42:08 UTC [INFO] Delete file /root/code/euspace/dist/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh.
+2024-12-05 02:42:08 UTC [INFO] -----Write release version 1.0.0 to /root/code/euspace/dist/installer.version file.
+2024-12-05 02:42:08 UTC [INFO] ---> edit readonly AGENT_INSTALLER_VERSION=1.0.0 to readonly AGENT_INSTALLER_VERSION=1.0.0, configFile: /root/code/euspace/dist/scripts/uninstall.sh
+2024-12-05 02:42:08 UTC [INFO] There are 5 steps to packing, begin package Cloudwise-euspace....
+2024-12-05 02:42:08 UTC [INFO] Delete the type [#==] comment ...
+2024-12-05 02:42:08 UTC [INFO] Delete the type [#DEBUG==] comment ...
+2024-12-05 02:42:08 UTC [INFO] step【1】start【tar -cpf /root/code/euspace/Cloudwise-euspace.tar ./package_dir/*】to Cloudwise-euspace.tar
+2024-12-05 02:42:08 UTC [INFO] step【2】start【xz -T0 -z Cloudwise-euspace.tar】to Cloudwise-euspace.tar.zx
+2024-12-05 02:42:13 UTC [INFO] step【3】start【tar -cpf Cloudwise-euspace.tar Cloudwise-euspace.tar.xz xzdec】 to Cloudwise-euspace.tar
+2024-12-05 02:42:13 UTC [INFO] step【4】start【base64 Cloudwise-euspace.tar > Cloudwise-euspace.tar.base64】 to Cloudwise-euspace.tar.base64
+2024-12-05 02:42:13 UTC [INFO] step【5】start write 【Cloudwise-euspace.tar.base64】 to /root/code/euspace/dist/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh
+2024-12-05 02:42:13 UTC [INFO] ---> edit readonly AGENT_BUILD_DATE=25.09.2020 to readonly AGENT_BUILD_DATE=2024-12-05, configFile: /root/code/euspace/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh.tmp
+2024-12-05 02:42:13 UTC [INFO] ---> edit readonly AGENT_INSTALLER_VERSION=1.2.0 to readonly AGENT_INSTALLER_VERSION=1.0.0, configFile: /root/code/euspace/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh.tmp
+2024-12-05 02:42:13 UTC [INFO] ---> edit readonly AGENT_BUILD_TAG="" to readonly AGENT_BUILD_TAG='d707430', configFile: /root/code/euspace/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh.tmp
+2024-12-05 02:42:13 UTC [INFO] ---> edit readonly AGENT_BUILD_DATE_INFO="" to readonly AGENT_BUILD_DATE_INFO='2024-12-05 02:42:13', configFile: /root/code/euspace/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh.tmp
+2024-12-05 02:42:13 UTC [INFO] ---> edit readonly AGENT_BUILD_UPLOADER="" to readonly AGENT_BUILD_UPLOADER='ubuntu22/root', configFile: /root/code/euspace/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh.tmp
+2024-12-05 02:42:13 UTC [INFO] ---> edit readonly EXTERNAL_TAR_SIZE=233820160 to readonly EXTERNAL_TAR_SIZE=142069760, configFile: /root/code/euspace/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh.tmp
+2024-12-05 02:42:13 UTC [INFO] ---> edit readonly ARTIFACTS_SIZE=953271934 to readonly ARTIFACTS_SIZE=200866999, configFile: /root/code/euspace/Cloudwise-euspace-installer-Linux-x86_64-1.0.0.sh.tmp
+2024-12-05 02:42:13 UTC [INFO] Cloudwise-euspace packaging success.
+2024-12-05 02:42:13 UTC [INFO] Installation finished, PID 920378, exit code: 0.

main.go

@@ -255,9 +255,9 @@ func main() {
 				metric.GetName() != "process_net_tcp_bytes_sent_per" &&
 				metric.GetName() != "process_net_tcp_bytes_received_total" &&
 				metric.GetName() != "process_net_tcp_bytes_sent_total" &&
-				metric.GetName() != "process_net_tcp_data_latency" &&
-				metric.GetName() != "process_net_tcp_data_duration" &&
-				metric.GetName() != "process_net_tcp_est_time"{
+				metric.GetName() != "process_net_tcp_data_latency_time" &&
+				metric.GetName() != "process_net_tcp_flow_duration_time" &&
+				metric.GetName() != "process_net_tcp_connection_establish_time"{
 				continue
 			}
 
@@ -304,7 +304,7 @@ func main() {
 		if err != nil {
 			return
 		}
-
+		log.Infoln("netdata is:", string(jsonData))
 		// 创建请求  
 		urlRoute := "/api/v2/ebpf/receive"
 		// req, err := http.NewRequest("POST", "http://10.0.7.115:18080/api/v2/ebpf/receive", bytes.NewBuffer(jsonData))
@@ -336,13 +336,13 @@ func main() {
 		// 读取响应内容  
 		responseData, err := io.ReadAll(response.Body)  
 		if err != nil {  
-			fmt.Println("Error:", err)  
+			log.Infoln("Error:", err)  
 			return  
 		}  
 
 		// 输出响应状态码和响应正文  
-		fmt.Println("Status Code:", response.StatusCode)  
-		fmt.Println("Response Body:", string(responseData)) 
+		log.Infoln("Status Code:", response.StatusCode)  
+		log.Infoln("Response Body:", string(responseData)) 
 	}
 	sendNetDataDone := make(chan struct{})
 	go func() {

pkg/go.opentelemetry.io/otel/exporters/otlp/otlptrace/apm_exporter.go

@@ -107,6 +107,8 @@ type MapInfoT struct {
 	AssumedAppId   int64    `json:"assumed_app_id,omitempty"`
 	Uri            string   `json:"uri,omitempty"`
 	SpanId         string   `json:"span_id,omitempty"`
+	SrcAddr		   string   `json:"src_addr,omitempty"`
+	DestinationAddr	string  `json:"destination_addr,omitempty"`
 }
 
 type TraceMapT struct {
@@ -790,6 +792,10 @@ func buildHttpMapFromEvent(mNode *MapInfoT, event tracesdk.Event) {
 		case "time.duration":
 			//mNode.PureTime = uint64(attr.Value.AsInt64()) / 1e3
 			mNode.WallTime = uint64(attr.Value.AsInt64()) / 1e3
+		case "http.src_addr":
+			mNode.SrcAddr = attr.Value.AsString()
+		case "http.destination_addr":
+			mNode.DestinationAddr = attr.Value.AsString()
 		}
 	}
 	//mNode.AssumedAppId = Md5ToInt64(descAddr, 16)
@@ -837,6 +843,10 @@ func buildMysqlMapEvent(mNode *MapInfoT, event tracesdk.Event) {
 			if len(words) > 0 {
 				mNode.OperType = strings.ToUpper(words[0])
 			}
+		case "sql.src_addr":
+			mNode.SrcAddr = attr.Value.AsString()
+		case "sql.destination_addr":
+			mNode.DestinationAddr = attr.Value.AsString()
 		}
 	}
 }
@@ -866,6 +876,10 @@ func buildDMMapEvent(mNode *MapInfoT, event tracesdk.Event) {
 			} else {
 				mNode.Exception = 0
 			}
+		case "sql.src_addr":
+			mNode.SrcAddr = attr.Value.AsString()
+		case "sql.destination_addr":
+			mNode.DestinationAddr = attr.Value.AsString()
 		}
 	}
 }
@@ -889,6 +903,10 @@ func buildRedisMap(mNode *MapInfoT, sd apmTraceSpan) {
 			if len(words) > 0 {
 				mNode.OperType = strings.ToUpper(words[0])
 			}
+		case "nosql.src_addr":
+			mNode.SrcAddr = attr.Value.AsString()
+		case "nosql.destination_addr":
+			mNode.DestinationAddr = attr.Value.AsString()
 		}
 	}
 }
@@ -912,6 +930,10 @@ func buildRedisMapEvent(mNode *MapInfoT, event tracesdk.Event) {
 			if len(words) > 0 {
 				mNode.OperType = strings.ToUpper(words[0])
 			}
+		case "nosql.src_addr":
+			mNode.SrcAddr = attr.Value.AsString()
+		case "nosql.destination_addr":
+			mNode.DestinationAddr = attr.Value.AsString()
 		}
 	}
 }

tracing/apm_tracing.go

@@ -320,6 +320,8 @@ func (t *Trace) MysqlTraceQueryEvent(query string, r *l7.RequestData, destinatio
 		semconv.DBStatement(query),
 		semconv.NetPeerName(destination.IP().String()),
 		semconv.NetPeerPort(int(destination.Port())),
+		attribute.String("sql.src_addr", r.ComponentSAddr.String()),
+		attribute.String("sql.destination_addr", r.ComponentDAddr.String()),
 	)
 	t.appendTimestamp(&attr, r.StartAt, r.EndAt, r.Duration.Nanoseconds())
 	t.createTraceEvent(l7.ProtocolMysql.String(), int(ebpftracer.EventTypeL7Request), int(l7.ProtocolMysql), attr...)
@@ -336,6 +338,8 @@ func (t *Trace) DmTraceQueryEvent(query string, r *l7.RequestData, destination n
 		semconv.DBStatement(query),
 		semconv.NetPeerName(destination.IP().String()),
 		semconv.NetPeerPort(int(destination.Port())),
+		attribute.String("sql.src_addr", r.ComponentSAddr.String()),
+		attribute.String("sql.destination_addr", r.ComponentDAddr.String()),
 	)
 	t.appendTimestamp(&attr, r.StartAt, r.EndAt, r.Duration.Nanoseconds())
 
@@ -376,6 +380,8 @@ func (t *Trace) RedisTraceQueryEvent(cmd, args string, r *l7.RequestData, destin
 		semconv.DBStatement(statement),
 		semconv.NetPeerName(destination.IP().String()),
 		semconv.NetPeerPort(int(destination.Port())),
+		attribute.String("nosql.src_addr", r.ComponentSAddr.String()),
+		attribute.String("nosql.destination_addr", r.ComponentDAddr.String()),
 	)
 	t.appendTimestamp(&attr, r.StartAt, r.EndAt, r.Duration.Nanoseconds())
 	t.createTraceEvent(l7.ProtocolRedis.String(), int(ebpftracer.EventTypeL7Request), int(l7.ProtocolRedis), attr...)
@@ -438,6 +444,8 @@ func (t *Trace) HttpTraceRequestEvent(method, path, ip string, port uint16, r *l
 		attribute.Int64("http.assumed_app_id", assumedAppID),
 		attribute.String("http.span_id", r.SpanId),
 		attribute.Int("http.port", int(port)),
+		attribute.String("http.src_addr", r.ComponentSAddr.String()),
+		attribute.String("http.destination_addr", r.ComponentDAddr.String()),
 	)
 	t.appendTimestamp(&attr, r.StartAt, r.EndAt, r.Duration.Nanoseconds())
 	t.createTraceEvent(l7.ProtocolHTTP.String(), int(ebpftracer.EventTypeL7Request), int(l7.ProtocolHTTP), attr...)