From 78baa850a8f9363bbe2b2607d62a12f6e35fafbd Mon Sep 17 00:00:00 2001 From: Ava Hahn Date: Thu, 18 Sep 2025 00:04:23 +0000 Subject: [PATCH] fixes for worker process profiling Signed-off-by: Ava Hahn --- compose.yaml | 2 ++ kaclient/run.sh | 15 ++++++++------- kaproxy/Dockerfile | 7 ++++--- kaproxy/gencerts.sh | 12 ++++++++---- kaproxy/nginx.conf | 9 ++++++++- kaproxy/run.sh | 14 ++++++++++---- run.sh | 12 ++++++------ 7 files changed, 46 insertions(+), 25 deletions(-) diff --git a/compose.yaml b/compose.yaml index 3d5b71c..6ab393b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -59,6 +59,8 @@ services: - linux/x86_64 platform: linux/x86_64 privileged: true + cap_add: + - SYS_PTRACE ports: - 8079:8080 networks: diff --git a/kaclient/run.sh b/kaclient/run.sh index 6fa2756..b244fc2 100755 --- a/kaclient/run.sh +++ b/kaclient/run.sh @@ -1,5 +1,7 @@ #!/bin/bash +N=$(nproc --all) + function log_request_to () { return_code=$(curl -Sikl -o /dev/null -w "%{http_code}" $1 2>/dev/null) case ${return_code:0:1} in @@ -13,10 +15,6 @@ function log_request_to () { esac } -function do_wrk_on () { - /wrk/wrk -t1 -c10 $1 & -} - function sigint_handler() { jobs -p | xargs kill -9 exit @@ -26,10 +24,13 @@ trap 'sigint_handler' INT # TODO: make this a more elegant item # maybe a while loop with curl -sleep 0.5 +sleep 2 -for iter in {0.999}; do - do_wrk_on "https://kaproxy:8080/$iter" +echo "[+] client making request loop" +for iter in {0..999}; do + ((i=i%N)); ((i++==0)) && wait + echo "request to $iter" + log_request_to "https://kaproxy:8080/$iter" & done wait $(jobs -p) diff --git a/kaproxy/Dockerfile b/kaproxy/Dockerfile index 23fdc4e..b9fe88c 100644 --- a/kaproxy/Dockerfile +++ b/kaproxy/Dockerfile @@ -6,8 +6,8 @@ RUN echo "deb http://deb.debian.org/debian-debug/ bookworm-proposed-updates-debu RUN apt update -y RUN apt install libssl3 libssl3-dbgsym openssl openssl-dbgsym libssl-dev zlib1g-dev \ - libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym \ - --allow-downgrades -y + libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym strace \ + procps --allow-downgrades -y COPY nginx.conf / WORKDIR / @@ -24,8 +24,9 @@ WORKDIR /nginx RUN auto/configure \ --with-debug \ --with-http_ssl_module \ + --with-file-aio \ --with-cc-opt="-gdwarf-4 -fno-omit-frame-pointer" -RUN make +RUN make -j $(nproc --all) RUN make install COPY run.sh / diff --git a/kaproxy/gencerts.sh b/kaproxy/gencerts.sh index d133a8e..717deab 100755 --- a/kaproxy/gencerts.sh +++ b/kaproxy/gencerts.sh @@ -1,6 +1,8 @@ #!/bin/bash +N=$(nproc --all) for iter in {0..999}; do + ((i=i%N)); ((i++==0)) && wait echo "minting cert $iter" openssl req -x509 \ -newkey rsa:4096 \ @@ -8,17 +10,19 @@ for iter in {0..999}; do -out cert$iter.pem \ -sha256 -nodes \ -days 3650 \ - -quiet \ - -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" + -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" & +done +for iter in {0..999}; do upstr=$(($iter%10)) - echo ' + echo " location /$iter { proxy_ssl_certificate /cert$iter.pem; proxy_ssl_certificate_key /key$iter.pem; proxy_pass http://kasvc-$upstr:8080; - }' >> /nginx.conf + }" >> /nginx.conf done + echo ' } } diff --git a/kaproxy/nginx.conf b/kaproxy/nginx.conf index 5f3adf7..34250d4 100644 --- a/kaproxy/nginx.conf +++ b/kaproxy/nginx.conf @@ -1,19 +1,26 @@ -worker_processes 10; +worker_processes 1; error_log /dev/stdout notice; pid /tmp/pid; +# callgrind in worker processes must be able to do things +user root; + events { worker_connections 10; } http { keepalive_timeout 300; + aio on; # blocking io blocks tracing + directio 4m; server { listen 8080 ssl; server_name www.example.com; ssl_certificate /www.example.com.crt; ssl_certificate_key /www.example.com.key; + #ssl_certificate_cache max=1000; + ssl_session_cache shared:SSL:10m; access_log /tmp/access.log; proxy_socket_keepalive on; diff --git a/kaproxy/run.sh b/kaproxy/run.sh index 7a32bec..fe1fa67 100755 --- a/kaproxy/run.sh +++ b/kaproxy/run.sh @@ -3,10 +3,10 @@ function p_invoke() { valgrind --tool=callgrind \ --trace-children=yes \ - --callgrind-out-file=/tmp/callgrind.output \ + --callgrind-out-file=/tmp/callgrind.out.%p \ --cache-sim=yes \ + --instr-atstart=no \ /nginx/objs/nginx \ - -p /tmp \ -e /tmp/error.log \ -c /nginx.conf \ -g "daemon off;" @@ -14,7 +14,6 @@ function p_invoke() { function invoke() { /nginx/objs/nginx \ - -p /tmp \ -e /tmp/error.log \ -c /nginx.conf \ -g "daemon off;" \ @@ -35,7 +34,14 @@ function sigcont_handler() { trap 'sigint_handler' INT trap 'sigcont_handler' CONT +# enable tracing +echo 1 > /proc/sys/kernel/yama/ptrace_scope + p_invoke & + wait -echo "NGINX down. waiting until signalled..." +echo "NGINX down. waiting to find it again" +sleep 0.5 +wait $(cat /tmp/pid) +echo "NGINX is GONE. waiting until signalled" sleep infinity diff --git a/run.sh b/run.sh index f989958..10818a7 100755 --- a/run.sh +++ b/run.sh @@ -36,24 +36,25 @@ go build keepalive-svc.go mv keepalive-svc kasvc/ rsync -avz $1 kaproxy/ sudo docker-compose up --build -d -sudo docker exec -it $KAPROXY callgrind_control -i off +sudo docker exec $KAPROXY callgrind_control -i off sudo docker wait $KACLIENT echo "[+] client finished, triggering reload" -sudo docker exec -it $KAPROXY callgrind_control -i on +sudo docker exec $KAPROXY callgrind_control -i on sudo docker kill -s CONT $KAPROXY echo "[+] wait five seconds for reload complete" sleep 5 -sudo docker exec -it $KAPROXY callgrind_control -i off echo " > restarting client" sudo docker-compose restart kaclient sudo docker wait $KACLIENT -echo "[+] client finished again. Killing NGINX and fetching profile data" +echo "[+] client finished again. reloading NGINX and fetching profile data" +sudo docker kill -s CONT $KAPROXY +sudo docker exec $KAPROXY callgrind_control -i off sudo docker kill -s INT $KAPROXY -sudo docker exec -it $KAPROXY callgrind_control -d +sleep 10 echo "[+] building profiling report" sudo docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file @@ -64,7 +65,6 @@ do sudo chmod 777 $F echo "Output file: $F" >> $PROFILE_OUTPUT callgrind_annotate \ - --include=kaproxy \ --auto=yes \ $F >> $PROFILE_OUTPUT echo "End of profile: $F\n\n\n" >> $PROFILE_OUTPUT