fixes for worker process profiling

Signed-off-by: Ava Hahn <a.hahn@f5.com>

kaproxy/Dockerfile

@@ -6,8 +6,8 @@ RUN echo "deb http://deb.debian.org/debian-debug/ bookworm-proposed-updates-debu
 
 RUN apt update -y
 RUN apt install libssl3 libssl3-dbgsym openssl openssl-dbgsym libssl-dev zlib1g-dev \
-	libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym \
-	--allow-downgrades -y
+	libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym strace \
+	procps --allow-downgrades -y
 
 COPY nginx.conf /
 WORKDIR /
@@ -24,8 +24,9 @@ WORKDIR /nginx
 RUN auto/configure \
 	--with-debug \
 	--with-http_ssl_module \
+	--with-file-aio \
 	--with-cc-opt="-gdwarf-4 -fno-omit-frame-pointer"
-RUN make
+RUN make -j $(nproc --all)
 RUN make install
 
 COPY run.sh /

kaproxy/gencerts.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
+N=$(nproc --all)
 for iter in {0..999}; do
+	((i=i%N)); ((i++==0)) && wait
 	echo "minting cert $iter"
 	openssl req -x509         \
 		-newkey rsa:4096      \
@@ -8,17 +10,19 @@ for iter in {0..999}; do
 		-out cert$iter.pem    \
 		-sha256 -nodes        \
 		-days 3650            \
-		-quiet                \
-		-subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter"
+		-subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" &
+done
 
+for iter in {0..999}; do
 	upstr=$(($iter%10))
-	echo '
+	echo "
 		location /$iter {
 			proxy_ssl_certificate /cert$iter.pem;
 			proxy_ssl_certificate_key /key$iter.pem;
 			proxy_pass http://kasvc-$upstr:8080;
-		}' >> /nginx.conf
+		}" >> /nginx.conf
 done
+
 echo '
 	}
 }

kaproxy/nginx.conf

@@ -1,19 +1,26 @@
-worker_processes 10;
+worker_processes 1;
 error_log /dev/stdout notice;
 pid /tmp/pid;
 
+# callgrind in worker processes must be able to do things
+user root;
+
 events {
 	worker_connections 10;
 }
 
 http {
 	keepalive_timeout 300;
+	aio on; # blocking io blocks tracing
+	directio 4m;
 
 	server {
 		listen 8080 ssl;
 		server_name www.example.com;
 		ssl_certificate /www.example.com.crt;
 		ssl_certificate_key /www.example.com.key;
+		#ssl_certificate_cache max=1000;
+		ssl_session_cache shared:SSL:10m;
 		access_log /tmp/access.log;
 		proxy_socket_keepalive on;
 

kaproxy/run.sh

@@ -3,10 +3,10 @@
 function p_invoke() {
 	valgrind --tool=callgrind                         \
 		--trace-children=yes                          \
-		--callgrind-out-file=/tmp/callgrind.output    \
+		--callgrind-out-file=/tmp/callgrind.out.%p    \
 		--cache-sim=yes                               \
+		--instr-atstart=no                            \
 		/nginx/objs/nginx                             \
-			-p /tmp                                   \
 			-e /tmp/error.log                         \
 			-c /nginx.conf                            \
 			-g "daemon off;"
@@ -14,7 +14,6 @@ function p_invoke() {
 
 function invoke() {
 	/nginx/objs/nginx     \
-		-p /tmp           \
 		-e /tmp/error.log \
 		-c /nginx.conf    \
 		-g "daemon off;"  \
@@ -35,7 +34,14 @@ function sigcont_handler() {
 trap 'sigint_handler' INT
 trap 'sigcont_handler' CONT
 
+# enable tracing
+echo 1 > /proc/sys/kernel/yama/ptrace_scope
+
 p_invoke &
+
 wait
-echo "NGINX down. waiting until signalled..."
+echo "NGINX down. waiting to find it again"
+sleep 0.5
+wait $(cat /tmp/pid)
+echo "NGINX is GONE. waiting until signalled"
 sleep infinity