compose.yaml

@@ -50,286 +50,6 @@ services:
     <<: *kasvc
     ports:
       - 8089:8080
-  kasvc-10:
-    <<: *kasvc
-    ports:
-      - 8090:8080
-  kasvc-11:
-    <<: *kasvc
-    ports:
-      - 8091:8080
-  kasvc-12:
-    <<: *kasvc
-    ports:
-      - 8092:8080
-  kasvc-13:
-    <<: *kasvc
-    ports:
-      - 8093:8080
-  kasvc-14:
-    <<: *kasvc
-    ports:
-      - 8094:8080
-  kasvc-15:
-    <<: *kasvc
-    ports:
-      - 8095:8080
-  kasvc-16:
-    <<: *kasvc
-    ports:
-      - 8096:8080
-  kasvc-17:
-    <<: *kasvc
-    ports:
-      - 8097:8080
-  kasvc-18:
-    <<: *kasvc
-    ports:
-      - 8098:8080
-  kasvc-19:
-    <<: *kasvc
-    ports:
-      - 8099:8080
-  kasvc-20:
-    <<: *kasvc
-    ports:
-      - 8100:8080
-  kasvc-21:
-    <<: *kasvc
-    ports:
-      - 8101:8080
-  kasvc-22:
-    <<: *kasvc
-    ports:
-      - 8102:8080
-  kasvc-23:
-    <<: *kasvc
-    ports:
-      - 8103:8080
-  kasvc-24:
-    <<: *kasvc
-    ports:
-      - 8104:8080
-  kasvc-25:
-    <<: *kasvc
-    ports:
-      - 8105:8080
-  kasvc-26:
-    <<: *kasvc
-    ports:
-      - 8106:8080
-  kasvc-27:
-    <<: *kasvc
-    ports:
-      - 8107:8080
-  kasvc-28:
-    <<: *kasvc
-    ports:
-      - 8108:8080
-  kasvc-29:
-    <<: *kasvc
-    ports:
-      - 8109:8080
-  kasvc-30:
-    <<: *kasvc
-    ports:
-      - 8110:8080
-  kasvc-31:
-    <<: *kasvc
-    ports:
-      - 8111:8080
-  kasvc-32:
-    <<: *kasvc
-    ports:
-      - 8112:8080
-  kasvc-33:
-    <<: *kasvc
-    ports:
-      - 8113:8080
-  kasvc-34:
-    <<: *kasvc
-    ports:
-      - 8114:8080
-  kasvc-35:
-    <<: *kasvc
-    ports:
-      - 8115:8080
-  kasvc-36:
-    <<: *kasvc
-    ports:
-      - 8116:8080
-  kasvc-37:
-    <<: *kasvc
-    ports:
-      - 8117:8080
-  kasvc-38:
-    <<: *kasvc
-    ports:
-      - 8118:8080
-  kasvc-39:
-    <<: *kasvc
-    ports:
-      - 8119:8080
-  kasvc-40:
-    <<: *kasvc
-    ports:
-      - 8120:8080
-  kasvc-41:
-    <<: *kasvc
-    ports:
-      - 8121:8080
-  kasvc-42:
-    <<: *kasvc
-    ports:
-      - 8122:8080
-  kasvc-43:
-    <<: *kasvc
-    ports:
-      - 8123:8080
-  kasvc-44:
-    <<: *kasvc
-    ports:
-      - 8124:8080
-  kasvc-45:
-    <<: *kasvc
-    ports:
-      - 8125:8080
-  kasvc-46:
-    <<: *kasvc
-    ports:
-      - 8126:8080
-  kasvc-47:
-    <<: *kasvc
-    ports:
-      - 8127:8080
-  kasvc-48:
-    <<: *kasvc
-    ports:
-      - 8128:8080
-  kasvc-49:
-    <<: *kasvc
-    ports:
-      - 8129:8080
-  kasvc-50:
-    <<: *kasvc
-    ports:
-      - 8130:8080
-  kasvc-51:
-    <<: *kasvc
-    ports:
-      - 8131:8080
-  kasvc-52:
-    <<: *kasvc
-    ports:
-      - 8132:8080
-  kasvc-53:
-    <<: *kasvc
-    ports:
-      - 8133:8080
-  kasvc-54:
-    <<: *kasvc
-    ports:
-      - 8134:8080
-  kasvc-55:
-    <<: *kasvc
-    ports:
-      - 8135:8080
-  kasvc-56:
-    <<: *kasvc
-    ports:
-      - 8136:8080
-  kasvc-57:
-    <<: *kasvc
-    ports:
-      - 8137:8080
-  kasvc-58:
-    <<: *kasvc
-    ports:
-      - 8138:8080
-  kasvc-59:
-    <<: *kasvc
-    ports:
-      - 8139:8080
-  kasvc-60:
-    <<: *kasvc
-    ports:
-      - 8140:8080
-  kasvc-61:
-    <<: *kasvc
-    ports:
-      - 8141:8080
-  kasvc-62:
-    <<: *kasvc
-    ports:
-      - 8142:8080
-  kasvc-63:
-    <<: *kasvc
-    ports:
-      - 8143:8080
-  kasvc-64:
-    <<: *kasvc
-    ports:
-      - 8144:8080
-  kasvc-65:
-    <<: *kasvc
-    ports:
-      - 8145:8080
-  kasvc-66:
-    <<: *kasvc
-    ports:
-      - 8146:8080
-  kasvc-67:
-    <<: *kasvc
-    ports:
-      - 8147:8080
-  kasvc-68:
-    <<: *kasvc
-    ports:
-      - 8148:8080
-  kasvc-69:
-    <<: *kasvc
-    ports:
-      - 8149:8080
-  kasvc-70:
-    <<: *kasvc
-    ports:
-      - 8150:8080
-  kasvc-71:
-    <<: *kasvc
-    ports:
-      - 8151:8080
-  kasvc-72:
-    <<: *kasvc
-    ports:
-      - 8152:8080
-  kasvc-73:
-    <<: *kasvc
-    ports:
-      - 8153:8080
-  kasvc-74:
-    <<: *kasvc
-    ports:
-      - 8154:8080
-  kasvc-75:
-    <<: *kasvc
-    ports:
-      - 8155:8080
-  kasvc-76:
-    <<: *kasvc
-    ports:
-      - 8156:8080
-  kasvc-77:
-    <<: *kasvc
-    ports:
-      - 8157:8080
-  kasvc-78:
-    <<: *kasvc
-    ports:
-      - 8158:8080
-  kasvc-79:
-    <<: *kasvc
-    ports:
-      - 8159:8080
 
   kaproxy:
     build:
@@ -339,8 +59,6 @@ services:
         - linux/x86_64
     platform: linux/x86_64
     privileged: true
-    cap_add:
-      - SYS_PTRACE
     ports:
       - 8079:8080
     networks:

kaclient/run.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-N=$(nproc --all)
-
 function log_request_to () {
 	return_code=$(curl -Sikl -o /dev/null -w "%{http_code}" $1 2>/dev/null)
 	case ${return_code:0:1} in
@@ -15,6 +13,10 @@ function log_request_to () {
 	esac
 }
 
+function do_wrk_on () {
+	/wrk/wrk -t1 -c10 $1 &
+}
+
 function sigint_handler() {
 	jobs -p | xargs kill -9
 	exit
@@ -24,13 +26,10 @@ trap 'sigint_handler' INT
 
 # TODO: make this a more elegant item
 # maybe a while loop with curl
-sleep 2
+sleep 0.5
 
-echo "[+] client making request loop"
-for iter in {0..80}; do
-	((i=i%N)); ((i++==0)) && wait
-	echo "request to $iter"
-	log_request_to "https://kaproxy:8080/$iter" &
+for iter in {0.999}; do
+	do_wrk_on "https://kaproxy:8080/$iter"
 done
 
 wait $(jobs -p)

kaproxy/Dockerfile

@@ -6,8 +6,8 @@ RUN echo "deb http://deb.debian.org/debian-debug/ bookworm-proposed-updates-debu
 
 RUN apt update -y
 RUN apt install libssl3 libssl3-dbgsym openssl openssl-dbgsym libssl-dev zlib1g-dev \
-	libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym strace \
-	procps --allow-downgrades -y
+	libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym \
+	--allow-downgrades -y
 
 COPY nginx.conf /
 WORKDIR /
@@ -24,9 +24,8 @@ WORKDIR /nginx
 RUN auto/configure \
 	--with-debug \
 	--with-http_ssl_module \
-	--with-file-aio \
 	--with-cc-opt="-gdwarf-4 -fno-omit-frame-pointer"
-RUN make -j $(nproc --all)
+RUN make
 RUN make install
 
 COPY run.sh /

kaproxy/gencerts.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
 
-N=$(nproc --all)
-for iter in {0..79}; do
-	((i=i%N)); ((i++==0)) && wait
+for iter in {0..999}; do
 	echo "minting cert $iter"
 	openssl req -x509         \
 		-newkey rsa:4096      \
@@ -10,21 +8,18 @@ for iter in {0..79}; do
 		-out cert$iter.pem    \
 		-sha256 -nodes        \
 		-days 3650            \
-		-subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" &
-done
+		-quiet                \
+		-subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter"
 
-for iter in {0..79}; do
-	echo "
+	upstr=$(($iter%10))
+	echo '
 		location /$iter {
 			proxy_ssl_certificate /cert$iter.pem;
 			proxy_ssl_certificate_key /key$iter.pem;
-			proxy_pass https://kasvc-$iter:8080;
-		}" >> /nginx.conf
+			proxy_pass http://kasvc-$upstr:8080;
+		}' >> /nginx.conf
 done
-
 echo '
 	}
 }
 ' >> /nginx.conf
-
-wait

kaproxy/nginx.conf

@@ -1,26 +1,19 @@
-worker_processes 1;
-error_log /dev/stdout debug;
+worker_processes 10;
+error_log /dev/stdout notice;
 pid /tmp/pid;
 
-# callgrind in worker processes must be able to do things
-user root;
-
 events {
 	worker_connections 10;
 }
 
 http {
 	keepalive_timeout 300;
-	aio on; # blocking io blocks tracing
-	directio 4m;
 
 	server {
 		listen 8080 ssl;
 		server_name www.example.com;
 		ssl_certificate /www.example.com.crt;
 		ssl_certificate_key /www.example.com.key;
-		ssl_certificate_cache max=1000;
-		ssl_session_cache shared:SSL:10m;
 		access_log /tmp/access.log;
 		proxy_socket_keepalive on;
 

kaproxy/run.sh

@@ -3,10 +3,10 @@
 function p_invoke() {
 	valgrind --tool=callgrind                         \
 		--trace-children=yes                          \
-		--callgrind-out-file=/tmp/callgrind.out.%p    \
+		--callgrind-out-file=/tmp/callgrind.output    \
 		--cache-sim=yes                               \
-		--instr-atstart=no                            \
 		/nginx/objs/nginx                             \
+			-p /tmp                                   \
 			-e /tmp/error.log                         \
 			-c /nginx.conf                            \
 			-g "daemon off;"
@@ -14,6 +14,7 @@ function p_invoke() {
 
 function invoke() {
 	/nginx/objs/nginx     \
+		-p /tmp           \
 		-e /tmp/error.log \
 		-c /nginx.conf    \
 		-g "daemon off;"  \
@@ -34,14 +35,7 @@ function sigcont_handler() {
 trap 'sigint_handler' INT
 trap 'sigcont_handler' CONT
 
-# enable tracing
-echo 1 > /proc/sys/kernel/yama/ptrace_scope
-
 p_invoke &
-
 wait
-echo "NGINX down. waiting to find it again"
-sleep 0.5
-wait $(cat /tmp/pid)
-echo "NGINX is GONE. waiting until signalled"
+echo "NGINX down. waiting until signalled..."
 sleep infinity

keepalive-svc.go

@@ -1,95 +1,32 @@
 package main
 
-
 import (
-	"bytes"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
-	"math/big"
-	"net"
-	"net/http"
-	"os"
-	"time"
 	"fmt"
+  "net"
+	"net/http"
+	"time"
 )
 
-func generateSelfSignedCert(host string) (tls.Certificate, error) {
-	cert := &x509.Certificate{
-		SerialNumber:          big.NewInt(0),
-		NotBefore:             time.Now(),
-		NotAfter:              time.Now().AddDate(10, 0, 0),
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		KeyUsage:              x509.KeyUsageDigitalSignature,
-		BasicConstraintsValid: true,
-	}
-
-	cert.DNSNames = []string{host}
-	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return tls.Certificate{}, err
-	}
-	certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivKey.PublicKey, certPrivKey)
-	if err != nil {
-		return tls.Certificate{}, err
-	}
-	certPEM := new(bytes.Buffer)
-	pem.Encode(certPEM, &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: certBytes,
-	})
-	certPrivKeyPEM := new(bytes.Buffer)
-	pem.Encode(certPrivKeyPEM, &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
-	})
-	serverCert, err := tls.X509KeyPair(certPEM.Bytes(), certPrivKeyPEM.Bytes())
-	if err != nil {
-		return tls.Certificate{}, err
-	}
-	return serverCert, err
-}
-
 type myHandler struct{
 	nreq int
 }
 
 func (h myHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	fmt.Printf("accepted request number %d", h.nreq)
+  fmt.Printf("accepted request number %d", h.nreq)
 	fmt.Fprintf(w, "request number: %d", h.nreq)
 }
 
 func main() {
-	hostname, err := os.Hostname()
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	cert, err := generateSelfSignedCert(hostname)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	tlsConfig := &tls.Config{
-		Certificates: []tls.Certificate{cert},
-		ClientAuth: tls.RequireAnyClientCert,
-	}
-
 	srv := &http.Server{
 		Addr:         ":8080",
 		Handler:      myHandler{nreq: 0},
-		ReadTimeout:  5 * time.Minute,
-		WriteTimeout: 5 * time.Minute,
-		IdleTimeout:  5 * time.Minute,
-		ConnState: func(conn net.Conn, event http.ConnState) {
-			fmt.Printf("addr: %s, changed state to: %s", conn.RemoteAddr(), event.String())
-		},
-		TLSConfig: tlsConfig,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 10 * time.Second,
+		IdleTimeout:  300 * time.Second,
+    ConnState: func(conn net.Conn, event http.ConnState) {
+      fmt.Printf("addr: %s, changed state to: %s", conn.RemoteAddr(), event.String())
+    },
 	}
 
-	srv.ListenAndServeTLS("", "")
+	srv.ListenAndServe()
 }

run.sh

@@ -35,35 +35,36 @@ echo "[+] building and deploying containers"
 go build keepalive-svc.go
 mv keepalive-svc kasvc/
 rsync -avz $1 kaproxy/
-docker-compose up --build -d
+sudo docker-compose up --build -d
+sudo docker exec -it $KAPROXY callgrind_control -i off
 
-docker wait $KACLIENT
+sudo docker wait $KACLIENT
 echo "[+] client finished, triggering reload"
-docker exec $KAPROXY callgrind_control -i on
-docker kill -s CONT $KAPROXY
+sudo docker exec -it $KAPROXY callgrind_control -i on
+sudo docker kill -s CONT $KAPROXY
 
 echo "[+] wait five seconds for reload complete"
 sleep 5
+sudo docker exec -it $KAPROXY callgrind_control -i off
 
 echo "    > restarting client"
-docker-compose restart kaclient
-docker wait $KACLIENT
+sudo docker-compose restart kaclient
+sudo docker wait $KACLIENT
 
-echo "[+] client finished again. reloading NGINX and fetching profile data"
-docker kill -s CONT $KAPROXY
-#docker exec $KAPROXY callgrind_control -i off
-docker kill -s INT $KAPROXY
-sleep 10
+echo "[+] client finished again. Killing NGINX and fetching profile data"
+sudo docker kill -s INT $KAPROXY
+sudo docker exec -it $KAPROXY callgrind_control -d
 
 echo "[+] building profiling report"
-docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file
+sudo docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file
 do
 	echo "    > processing: " $file
 	F=$(basename $file)
-	docker cp $KAPROXY:$file $F;
+	sudo docker cp $KAPROXY:$file $F;
 	sudo chmod 777 $F
 	echo "Output file: $F" >> $PROFILE_OUTPUT
 	callgrind_annotate \
+		--include=kaproxy \
 		--auto=yes \
 		$F >> $PROFILE_OUTPUT
 	echo "End of profile: $F\n\n\n" >> $PROFILE_OUTPUT