fixes for worker process profiling

Signed-off-by: Ava Hahn <a.hahn@f5.com>

compose.yaml

@@ -59,6 +59,8 @@ services:
         - linux/x86_64
     platform: linux/x86_64
     privileged: true
+    cap_add:
+      - SYS_PTRACE
     ports:
       - 8079:8080
     networks:

kaclient/run.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+N=$(nproc --all)
+
 function log_request_to () {
 	return_code=$(curl -Sikl -o /dev/null -w "%{http_code}" $1 2>/dev/null)
 	case ${return_code:0:1} in
@@ -13,10 +15,6 @@ function log_request_to () {
 	esac
 }
 
-function do_wrk_on () {
-	/wrk/wrk -t1 -c10 $1 &
-}
-
 function sigint_handler() {
 	jobs -p | xargs kill -9
 	exit
@@ -26,10 +24,13 @@ trap 'sigint_handler' INT
 
 # TODO: make this a more elegant item
 # maybe a while loop with curl
-sleep 0.5
+sleep 2
 
-for iter in {0.999}; do
-	do_wrk_on "https://kaproxy:8080/$iter"
+echo "[+] client making request loop"
+for iter in {0..999}; do
+	((i=i%N)); ((i++==0)) && wait
+	echo "request to $iter"
+	log_request_to "https://kaproxy:8080/$iter" &
 done
 
 wait $(jobs -p)

kaproxy/Dockerfile

@@ -6,8 +6,8 @@ RUN echo "deb http://deb.debian.org/debian-debug/ bookworm-proposed-updates-debu
 
 RUN apt update -y
 RUN apt install libssl3 libssl3-dbgsym openssl openssl-dbgsym libssl-dev zlib1g-dev \
-	libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym \
-	--allow-downgrades -y
+	libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym strace \
+	procps --allow-downgrades -y
 
 COPY nginx.conf /
 WORKDIR /
@@ -24,8 +24,9 @@ WORKDIR /nginx
 RUN auto/configure \
 	--with-debug \
 	--with-http_ssl_module \
+	--with-file-aio \
 	--with-cc-opt="-gdwarf-4 -fno-omit-frame-pointer"
-RUN make
+RUN make -j $(nproc --all)
 RUN make install
 
 COPY run.sh /

kaproxy/gencerts.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
+N=$(nproc --all)
 for iter in {0..999}; do
+	((i=i%N)); ((i++==0)) && wait
 	echo "minting cert $iter"
 	openssl req -x509         \
 		-newkey rsa:4096      \
@@ -8,17 +10,19 @@ for iter in {0..999}; do
 		-out cert$iter.pem    \
 		-sha256 -nodes        \
 		-days 3650            \
-		-quiet                \
-		-subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter"
+		-subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" &
+done
 
+for iter in {0..999}; do
 	upstr=$(($iter%10))
-	echo '
+	echo "
 		location /$iter {
 			proxy_ssl_certificate /cert$iter.pem;
 			proxy_ssl_certificate_key /key$iter.pem;
 			proxy_pass http://kasvc-$upstr:8080;
-		}' >> /nginx.conf
+		}" >> /nginx.conf
 done
+
 echo '
 	}
 }

kaproxy/nginx.conf

@@ -1,19 +1,26 @@
-worker_processes 10;
+worker_processes 1;
 error_log /dev/stdout notice;
 pid /tmp/pid;
 
+# callgrind in worker processes must be able to do things
+user root;
+
 events {
 	worker_connections 10;
 }
 
 http {
 	keepalive_timeout 300;
+	aio on; # blocking io blocks tracing
+	directio 4m;
 
 	server {
 		listen 8080 ssl;
 		server_name www.example.com;
 		ssl_certificate /www.example.com.crt;
 		ssl_certificate_key /www.example.com.key;
+		#ssl_certificate_cache max=1000;
+		ssl_session_cache shared:SSL:10m;
 		access_log /tmp/access.log;
 		proxy_socket_keepalive on;
 

kaproxy/run.sh

@@ -3,10 +3,10 @@
 function p_invoke() {
 	valgrind --tool=callgrind                         \
 		--trace-children=yes                          \
-		--callgrind-out-file=/tmp/callgrind.output    \
+		--callgrind-out-file=/tmp/callgrind.out.%p    \
 		--cache-sim=yes                               \
+		--instr-atstart=no                            \
 		/nginx/objs/nginx                             \
-			-p /tmp                                   \
 			-e /tmp/error.log                         \
 			-c /nginx.conf                            \
 			-g "daemon off;"
@@ -14,7 +14,6 @@ function p_invoke() {
 
 function invoke() {
 	/nginx/objs/nginx     \
-		-p /tmp           \
 		-e /tmp/error.log \
 		-c /nginx.conf    \
 		-g "daemon off;"  \
@@ -35,7 +34,14 @@ function sigcont_handler() {
 trap 'sigint_handler' INT
 trap 'sigcont_handler' CONT
 
+# enable tracing
+echo 1 > /proc/sys/kernel/yama/ptrace_scope
+
 p_invoke &
+
 wait
-echo "NGINX down. waiting until signalled..."
+echo "NGINX down. waiting to find it again"
+sleep 0.5
+wait $(cat /tmp/pid)
+echo "NGINX is GONE. waiting until signalled"
 sleep infinity

run.sh

@@ -36,24 +36,25 @@ go build keepalive-svc.go
 mv keepalive-svc kasvc/
 rsync -avz $1 kaproxy/
 sudo docker-compose up --build -d
-sudo docker exec -it $KAPROXY callgrind_control -i off
+sudo docker exec $KAPROXY callgrind_control -i off
 
 sudo docker wait $KACLIENT
 echo "[+] client finished, triggering reload"
-sudo docker exec -it $KAPROXY callgrind_control -i on
+sudo docker exec $KAPROXY callgrind_control -i on
 sudo docker kill -s CONT $KAPROXY
 
 echo "[+] wait five seconds for reload complete"
 sleep 5
-sudo docker exec -it $KAPROXY callgrind_control -i off
 
 echo "    > restarting client"
 sudo docker-compose restart kaclient
 sudo docker wait $KACLIENT
 
-echo "[+] client finished again. Killing NGINX and fetching profile data"
+echo "[+] client finished again. reloading NGINX and fetching profile data"
+sudo docker kill -s CONT $KAPROXY
+sudo docker exec $KAPROXY callgrind_control -i off
 sudo docker kill -s INT $KAPROXY
-sudo docker exec -it $KAPROXY callgrind_control -d
+sleep 10
 
 echo "[+] building profiling report"
 sudo docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file
@@ -64,7 +65,6 @@ do
 	sudo chmod 777 $F
 	echo "Output file: $F" >> $PROFILE_OUTPUT
 	callgrind_annotate \
-		--include=kaproxy \
 		--auto=yes \
 		$F >> $PROFILE_OUTPUT
 	echo "End of profile: $F\n\n\n" >> $PROFILE_OUTPUT