get better profileo on worker process

Signed-off-by: Ava Hahn <a.hahn@f5.com>
fixes for worker process profiling
2025-10-06 16:19:20 +00:00 · 2025-09-18 00:04:23 +00:00
8 changed files with 411 additions and 47 deletions
--- a/compose.yaml
+++ b/compose.yaml
@ -50,6 +50,286 @@ services:
    <<: *kasvc
    ports:
      - 8089:8080
+  kasvc-10:
+    <<: *kasvc
+    ports:
+      - 8090:8080
+  kasvc-11:
+    <<: *kasvc
+    ports:
+      - 8091:8080
+  kasvc-12:
+    <<: *kasvc
+    ports:
+      - 8092:8080
+  kasvc-13:
+    <<: *kasvc
+    ports:
+      - 8093:8080
+  kasvc-14:
+    <<: *kasvc
+    ports:
+      - 8094:8080
+  kasvc-15:
+    <<: *kasvc
+    ports:
+      - 8095:8080
+  kasvc-16:
+    <<: *kasvc
+    ports:
+      - 8096:8080
+  kasvc-17:
+    <<: *kasvc
+    ports:
+      - 8097:8080
+  kasvc-18:
+    <<: *kasvc
+    ports:
+      - 8098:8080
+  kasvc-19:
+    <<: *kasvc
+    ports:
+      - 8099:8080
+  kasvc-20:
+    <<: *kasvc
+    ports:
+      - 8100:8080
+  kasvc-21:
+    <<: *kasvc
+    ports:
+      - 8101:8080
+  kasvc-22:
+    <<: *kasvc
+    ports:
+      - 8102:8080
+  kasvc-23:
+    <<: *kasvc
+    ports:
+      - 8103:8080
+  kasvc-24:
+    <<: *kasvc
+    ports:
+      - 8104:8080
+  kasvc-25:
+    <<: *kasvc
+    ports:
+      - 8105:8080
+  kasvc-26:
+    <<: *kasvc
+    ports:
+      - 8106:8080
+  kasvc-27:
+    <<: *kasvc
+    ports:
+      - 8107:8080
+  kasvc-28:
+    <<: *kasvc
+    ports:
+      - 8108:8080
+  kasvc-29:
+    <<: *kasvc
+    ports:
+      - 8109:8080
+  kasvc-30:
+    <<: *kasvc
+    ports:
+      - 8110:8080
+  kasvc-31:
+    <<: *kasvc
+    ports:
+      - 8111:8080
+  kasvc-32:
+    <<: *kasvc
+    ports:
+      - 8112:8080
+  kasvc-33:
+    <<: *kasvc
+    ports:
+      - 8113:8080
+  kasvc-34:
+    <<: *kasvc
+    ports:
+      - 8114:8080
+  kasvc-35:
+    <<: *kasvc
+    ports:
+      - 8115:8080
+  kasvc-36:
+    <<: *kasvc
+    ports:
+      - 8116:8080
+  kasvc-37:
+    <<: *kasvc
+    ports:
+      - 8117:8080
+  kasvc-38:
+    <<: *kasvc
+    ports:
+      - 8118:8080
+  kasvc-39:
+    <<: *kasvc
+    ports:
+      - 8119:8080
+  kasvc-40:
+    <<: *kasvc
+    ports:
+      - 8120:8080
+  kasvc-41:
+    <<: *kasvc
+    ports:
+      - 8121:8080
+  kasvc-42:
+    <<: *kasvc
+    ports:
+      - 8122:8080
+  kasvc-43:
+    <<: *kasvc
+    ports:
+      - 8123:8080
+  kasvc-44:
+    <<: *kasvc
+    ports:
+      - 8124:8080
+  kasvc-45:
+    <<: *kasvc
+    ports:
+      - 8125:8080
+  kasvc-46:
+    <<: *kasvc
+    ports:
+      - 8126:8080
+  kasvc-47:
+    <<: *kasvc
+    ports:
+      - 8127:8080
+  kasvc-48:
+    <<: *kasvc
+    ports:
+      - 8128:8080
+  kasvc-49:
+    <<: *kasvc
+    ports:
+      - 8129:8080
+  kasvc-50:
+    <<: *kasvc
+    ports:
+      - 8130:8080
+  kasvc-51:
+    <<: *kasvc
+    ports:
+      - 8131:8080
+  kasvc-52:
+    <<: *kasvc
+    ports:
+      - 8132:8080
+  kasvc-53:
+    <<: *kasvc
+    ports:
+      - 8133:8080
+  kasvc-54:
+    <<: *kasvc
+    ports:
+      - 8134:8080
+  kasvc-55:
+    <<: *kasvc
+    ports:
+      - 8135:8080
+  kasvc-56:
+    <<: *kasvc
+    ports:
+      - 8136:8080
+  kasvc-57:
+    <<: *kasvc
+    ports:
+      - 8137:8080
+  kasvc-58:
+    <<: *kasvc
+    ports:
+      - 8138:8080
+  kasvc-59:
+    <<: *kasvc
+    ports:
+      - 8139:8080
+  kasvc-60:
+    <<: *kasvc
+    ports:
+      - 8140:8080
+  kasvc-61:
+    <<: *kasvc
+    ports:
+      - 8141:8080
+  kasvc-62:
+    <<: *kasvc
+    ports:
+      - 8142:8080
+  kasvc-63:
+    <<: *kasvc
+    ports:
+      - 8143:8080
+  kasvc-64:
+    <<: *kasvc
+    ports:
+      - 8144:8080
+  kasvc-65:
+    <<: *kasvc
+    ports:
+      - 8145:8080
+  kasvc-66:
+    <<: *kasvc
+    ports:
+      - 8146:8080
+  kasvc-67:
+    <<: *kasvc
+    ports:
+      - 8147:8080
+  kasvc-68:
+    <<: *kasvc
+    ports:
+      - 8148:8080
+  kasvc-69:
+    <<: *kasvc
+    ports:
+      - 8149:8080
+  kasvc-70:
+    <<: *kasvc
+    ports:
+      - 8150:8080
+  kasvc-71:
+    <<: *kasvc
+    ports:
+      - 8151:8080
+  kasvc-72:
+    <<: *kasvc
+    ports:
+      - 8152:8080
+  kasvc-73:
+    <<: *kasvc
+    ports:
+      - 8153:8080
+  kasvc-74:
+    <<: *kasvc
+    ports:
+      - 8154:8080
+  kasvc-75:
+    <<: *kasvc
+    ports:
+      - 8155:8080
+  kasvc-76:
+    <<: *kasvc
+    ports:
+      - 8156:8080
+  kasvc-77:
+    <<: *kasvc
+    ports:
+      - 8157:8080
+  kasvc-78:
+    <<: *kasvc
+    ports:
+      - 8158:8080
+  kasvc-79:
+    <<: *kasvc
+    ports:
+      - 8159:8080

  kaproxy:
    build:
@ -59,6 +339,8 @@ services:
        - linux/x86_64
    platform: linux/x86_64
    privileged: true
+    cap_add:
+      - SYS_PTRACE
    ports:
      - 8079:8080
    networks:
--- a/kaclient/run.sh
+++ b/kaclient/run.sh
@ -1,5 +1,7 @@
 #!/bin/bash

+N=$(nproc --all)
+
 function log_request_to () {
 	return_code=$(curl -Sikl -o /dev/null -w "%{http_code}" $1 2>/dev/null)
 	case ${return_code:0:1} in
@ -13,10 +15,6 @@ function log_request_to () {
 	esac
 }

-function do_wrk_on () {
-	/wrk/wrk -t1 -c10 $1 &
-}
-
 function sigint_handler() {
 	jobs -p | xargs kill -9
 	exit
@ -26,10 +24,13 @@ trap 'sigint_handler' INT

 # TODO: make this a more elegant item
 # maybe a while loop with curl
-sleep 0.5
+sleep 2

-for iter in {0.999}; do
-	do_wrk_on "https://kaproxy:8080/$iter"
+echo "[+] client making request loop"
+for iter in {0..80}; do
+	((i=i%N)); ((i++==0)) && wait
+	echo "request to $iter"
+	log_request_to "https://kaproxy:8080/$iter" &
 done

 wait $(jobs -p)
--- a/kaproxy/Dockerfile
+++ b/kaproxy/Dockerfile
@ -6,8 +6,8 @@ RUN echo "deb http://deb.debian.org/debian-debug/ bookworm-proposed-updates-debu

 RUN apt update -y
 RUN apt install libssl3 libssl3-dbgsym openssl openssl-dbgsym libssl-dev zlib1g-dev \
-	libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym \
-	--allow-downgrades -y
+	libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym strace \
+	procps --allow-downgrades -y

 COPY nginx.conf /
 WORKDIR /
@ -24,8 +24,9 @@ WORKDIR /nginx
 RUN auto/configure \
 	--with-debug \
 	--with-http_ssl_module \
+	--with-file-aio \
 	--with-cc-opt="-gdwarf-4 -fno-omit-frame-pointer"
-RUN make
+RUN make -j $(nproc --all)
 RUN make install

 COPY run.sh /
--- a/kaproxy/gencerts.sh
+++ b/kaproxy/gencerts.sh
@ -1,6 +1,8 @@
 #!/bin/bash

-for iter in {0..999}; do
+N=$(nproc --all)
+for iter in {0..79}; do
+	((i=i%N)); ((i++==0)) && wait
 	echo "minting cert $iter"
 	openssl req -x509         \
 		-newkey rsa:4096      \
@ -8,18 +10,21 @@ for iter in {0..999}; do
 		-out cert$iter.pem    \
 		-sha256 -nodes        \
 		-days 3650            \
-		-quiet                \
-		-subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter"
+		-subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" &
+done

-	upstr=$(($iter%10))
-	echo '
+for iter in {0..79}; do
+	echo "
 		location /$iter {
 			proxy_ssl_certificate /cert$iter.pem;
 			proxy_ssl_certificate_key /key$iter.pem;
-			proxy_pass http://kasvc-$upstr:8080;
-		}' >> /nginx.conf
+			proxy_pass https://kasvc-$iter:8080;
+		}" >> /nginx.conf
 done
+
 echo '
 	}
 }
 ' >> /nginx.conf
+
+wait
--- a/kaproxy/nginx.conf
+++ b/kaproxy/nginx.conf
@ -1,19 +1,26 @@
-worker_processes 10;
-error_log /dev/stdout notice;
+worker_processes 1;
+error_log /dev/stdout debug;
 pid /tmp/pid;

+# callgrind in worker processes must be able to do things
+user root;
+
 events {
 	worker_connections 10;
 }

 http {
 	keepalive_timeout 300;
+	aio on; # blocking io blocks tracing
+	directio 4m;

 	server {
 		listen 8080 ssl;
 		server_name www.example.com;
 		ssl_certificate /www.example.com.crt;
 		ssl_certificate_key /www.example.com.key;
+		ssl_certificate_cache max=1000;
+		ssl_session_cache shared:SSL:10m;
 		access_log /tmp/access.log;
 		proxy_socket_keepalive on;

--- a/kaproxy/run.sh
+++ b/kaproxy/run.sh
@ -3,10 +3,10 @@
 function p_invoke() {
 	valgrind --tool=callgrind                         \
 		--trace-children=yes                          \
-		--callgrind-out-file=/tmp/callgrind.output    \
+		--callgrind-out-file=/tmp/callgrind.out.%p    \
 		--cache-sim=yes                               \
+		--instr-atstart=no                            \
 		/nginx/objs/nginx                             \
-			-p /tmp                                   \
 			-e /tmp/error.log                         \
 			-c /nginx.conf                            \
 			-g "daemon off;"
@ -14,7 +14,6 @@ function p_invoke() {

 function invoke() {
 	/nginx/objs/nginx     \
-		-p /tmp           \
 		-e /tmp/error.log \
 		-c /nginx.conf    \
 		-g "daemon off;"  \
@ -35,7 +34,14 @@ function sigcont_handler() {
 trap 'sigint_handler' INT
 trap 'sigcont_handler' CONT

+# enable tracing
+echo 1 > /proc/sys/kernel/yama/ptrace_scope
+
 p_invoke &
+
 wait
-echo "NGINX down. waiting until signalled..."
+echo "NGINX down. waiting to find it again"
+sleep 0.5
+wait $(cat /tmp/pid)
+echo "NGINX is GONE. waiting until signalled"
 sleep infinity
--- a/keepalive-svc.go
+++ b/keepalive-svc.go
@ -1,12 +1,57 @@
 package main

+
 import (
-	"fmt"
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"math/big"
 	"net"
 	"net/http"
+	"os"
 	"time"
+	"fmt"
 )

+func generateSelfSignedCert(host string) (tls.Certificate, error) {
+	cert := &x509.Certificate{
+		SerialNumber:          big.NewInt(0),
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().AddDate(10, 0, 0),
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		KeyUsage:              x509.KeyUsageDigitalSignature,
+		BasicConstraintsValid: true,
+	}
+
+	cert.DNSNames = []string{host}
+	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivKey.PublicKey, certPrivKey)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	certPEM := new(bytes.Buffer)
+	pem.Encode(certPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
+	})
+	certPrivKeyPEM := new(bytes.Buffer)
+	pem.Encode(certPrivKeyPEM, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
+	})
+	serverCert, err := tls.X509KeyPair(certPEM.Bytes(), certPrivKeyPEM.Bytes())
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	return serverCert, err
+}
+
 type myHandler struct{
 	nreq int
 }
@ -17,16 +62,34 @@ func (h myHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }

 func main() {
+	hostname, err := os.Hostname()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	cert, err := generateSelfSignedCert(hostname)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ClientAuth: tls.RequireAnyClientCert,
+	}
+
 	srv := &http.Server{
 		Addr:         ":8080",
 		Handler:      myHandler{nreq: 0},
-		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 10 * time.Second,
-		IdleTimeout:  300 * time.Second,
+		ReadTimeout:  5 * time.Minute,
+		WriteTimeout: 5 * time.Minute,
+		IdleTimeout:  5 * time.Minute,
 		ConnState: func(conn net.Conn, event http.ConnState) {
 			fmt.Printf("addr: %s, changed state to: %s", conn.RemoteAddr(), event.String())
 		},
+		TLSConfig: tlsConfig,
 	}

-	srv.ListenAndServe()
+	srv.ListenAndServeTLS("", "")
 }
--- a/run.sh
+++ b/run.sh
@ -35,36 +35,35 @@ echo "[+] building and deploying containers"
 go build keepalive-svc.go
 mv keepalive-svc kasvc/
 rsync -avz $1 kaproxy/
-sudo docker-compose up --build -d
-sudo docker exec -it $KAPROXY callgrind_control -i off
+docker-compose up --build -d

-sudo docker wait $KACLIENT
+docker wait $KACLIENT
 echo "[+] client finished, triggering reload"
-sudo docker exec -it $KAPROXY callgrind_control -i on
-sudo docker kill -s CONT $KAPROXY
+docker exec $KAPROXY callgrind_control -i on
+docker kill -s CONT $KAPROXY

 echo "[+] wait five seconds for reload complete"
 sleep 5
-sudo docker exec -it $KAPROXY callgrind_control -i off

 echo "    > restarting client"
-sudo docker-compose restart kaclient
-sudo docker wait $KACLIENT
+docker-compose restart kaclient
+docker wait $KACLIENT

-echo "[+] client finished again. Killing NGINX and fetching profile data"
-sudo docker kill -s INT $KAPROXY
-sudo docker exec -it $KAPROXY callgrind_control -d
+echo "[+] client finished again. reloading NGINX and fetching profile data"
+docker kill -s CONT $KAPROXY
+#docker exec $KAPROXY callgrind_control -i off
+docker kill -s INT $KAPROXY
+sleep 10

 echo "[+] building profiling report"
-sudo docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file
+docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file
 do
 	echo "    > processing: " $file
 	F=$(basename $file)
-	sudo docker cp $KAPROXY:$file $F;
+	docker cp $KAPROXY:$file $F;
 	sudo chmod 777 $F
 	echo "Output file: $F" >> $PROFILE_OUTPUT
 	callgrind_annotate \
-		--include=kaproxy \
 		--auto=yes \
 		$F >> $PROFILE_OUTPUT
 	echo "End of profile: $F\n\n\n" >> $PROFILE_OUTPUT
Author	SHA1	Message	Date
Ava Hahn	4acd439e8a	get better profileo on worker process Signed-off-by: Ava Hahn <a.hahn@f5.com>	2025-10-06 16:19:20 +00:00
Ava Hahn	78baa850a8	fixes for worker process profiling Signed-off-by: Ava Hahn <a.hahn@f5.com>	2025-09-18 00:04:23 +00:00