diff --git a/compose.yaml b/compose.yaml index 3d5b71c..ab02a86 100644 --- a/compose.yaml +++ b/compose.yaml @@ -50,6 +50,286 @@ services: <<: *kasvc ports: - 8089:8080 + kasvc-10: + <<: *kasvc + ports: + - 8090:8080 + kasvc-11: + <<: *kasvc + ports: + - 8091:8080 + kasvc-12: + <<: *kasvc + ports: + - 8092:8080 + kasvc-13: + <<: *kasvc + ports: + - 8093:8080 + kasvc-14: + <<: *kasvc + ports: + - 8094:8080 + kasvc-15: + <<: *kasvc + ports: + - 8095:8080 + kasvc-16: + <<: *kasvc + ports: + - 8096:8080 + kasvc-17: + <<: *kasvc + ports: + - 8097:8080 + kasvc-18: + <<: *kasvc + ports: + - 8098:8080 + kasvc-19: + <<: *kasvc + ports: + - 8099:8080 + kasvc-20: + <<: *kasvc + ports: + - 8100:8080 + kasvc-21: + <<: *kasvc + ports: + - 8101:8080 + kasvc-22: + <<: *kasvc + ports: + - 8102:8080 + kasvc-23: + <<: *kasvc + ports: + - 8103:8080 + kasvc-24: + <<: *kasvc + ports: + - 8104:8080 + kasvc-25: + <<: *kasvc + ports: + - 8105:8080 + kasvc-26: + <<: *kasvc + ports: + - 8106:8080 + kasvc-27: + <<: *kasvc + ports: + - 8107:8080 + kasvc-28: + <<: *kasvc + ports: + - 8108:8080 + kasvc-29: + <<: *kasvc + ports: + - 8109:8080 + kasvc-30: + <<: *kasvc + ports: + - 8110:8080 + kasvc-31: + <<: *kasvc + ports: + - 8111:8080 + kasvc-32: + <<: *kasvc + ports: + - 8112:8080 + kasvc-33: + <<: *kasvc + ports: + - 8113:8080 + kasvc-34: + <<: *kasvc + ports: + - 8114:8080 + kasvc-35: + <<: *kasvc + ports: + - 8115:8080 + kasvc-36: + <<: *kasvc + ports: + - 8116:8080 + kasvc-37: + <<: *kasvc + ports: + - 8117:8080 + kasvc-38: + <<: *kasvc + ports: + - 8118:8080 + kasvc-39: + <<: *kasvc + ports: + - 8119:8080 + kasvc-40: + <<: *kasvc + ports: + - 8120:8080 + kasvc-41: + <<: *kasvc + ports: + - 8121:8080 + kasvc-42: + <<: *kasvc + ports: + - 8122:8080 + kasvc-43: + <<: *kasvc + ports: + - 8123:8080 + kasvc-44: + <<: *kasvc + ports: + - 8124:8080 + kasvc-45: + <<: *kasvc + ports: + - 8125:8080 + kasvc-46: + <<: *kasvc + ports: + - 8126:8080 + kasvc-47: + <<: *kasvc + ports: + - 8127:8080 + kasvc-48: + <<: *kasvc + ports: + - 8128:8080 + kasvc-49: + <<: *kasvc + ports: + - 8129:8080 + kasvc-50: + <<: *kasvc + ports: + - 8130:8080 + kasvc-51: + <<: *kasvc + ports: + - 8131:8080 + kasvc-52: + <<: *kasvc + ports: + - 8132:8080 + kasvc-53: + <<: *kasvc + ports: + - 8133:8080 + kasvc-54: + <<: *kasvc + ports: + - 8134:8080 + kasvc-55: + <<: *kasvc + ports: + - 8135:8080 + kasvc-56: + <<: *kasvc + ports: + - 8136:8080 + kasvc-57: + <<: *kasvc + ports: + - 8137:8080 + kasvc-58: + <<: *kasvc + ports: + - 8138:8080 + kasvc-59: + <<: *kasvc + ports: + - 8139:8080 + kasvc-60: + <<: *kasvc + ports: + - 8140:8080 + kasvc-61: + <<: *kasvc + ports: + - 8141:8080 + kasvc-62: + <<: *kasvc + ports: + - 8142:8080 + kasvc-63: + <<: *kasvc + ports: + - 8143:8080 + kasvc-64: + <<: *kasvc + ports: + - 8144:8080 + kasvc-65: + <<: *kasvc + ports: + - 8145:8080 + kasvc-66: + <<: *kasvc + ports: + - 8146:8080 + kasvc-67: + <<: *kasvc + ports: + - 8147:8080 + kasvc-68: + <<: *kasvc + ports: + - 8148:8080 + kasvc-69: + <<: *kasvc + ports: + - 8149:8080 + kasvc-70: + <<: *kasvc + ports: + - 8150:8080 + kasvc-71: + <<: *kasvc + ports: + - 8151:8080 + kasvc-72: + <<: *kasvc + ports: + - 8152:8080 + kasvc-73: + <<: *kasvc + ports: + - 8153:8080 + kasvc-74: + <<: *kasvc + ports: + - 8154:8080 + kasvc-75: + <<: *kasvc + ports: + - 8155:8080 + kasvc-76: + <<: *kasvc + ports: + - 8156:8080 + kasvc-77: + <<: *kasvc + ports: + - 8157:8080 + kasvc-78: + <<: *kasvc + ports: + - 8158:8080 + kasvc-79: + <<: *kasvc + ports: + - 8159:8080 kaproxy: build: @@ -59,6 +339,8 @@ services: - linux/x86_64 platform: linux/x86_64 privileged: true + cap_add: + - SYS_PTRACE ports: - 8079:8080 networks: diff --git a/kaclient/run.sh b/kaclient/run.sh index 6fa2756..198c459 100755 --- a/kaclient/run.sh +++ b/kaclient/run.sh @@ -1,5 +1,7 @@ #!/bin/bash +N=$(nproc --all) + function log_request_to () { return_code=$(curl -Sikl -o /dev/null -w "%{http_code}" $1 2>/dev/null) case ${return_code:0:1} in @@ -13,10 +15,6 @@ function log_request_to () { esac } -function do_wrk_on () { - /wrk/wrk -t1 -c10 $1 & -} - function sigint_handler() { jobs -p | xargs kill -9 exit @@ -26,10 +24,13 @@ trap 'sigint_handler' INT # TODO: make this a more elegant item # maybe a while loop with curl -sleep 0.5 +sleep 2 -for iter in {0.999}; do - do_wrk_on "https://kaproxy:8080/$iter" +echo "[+] client making request loop" +for iter in {0..80}; do + ((i=i%N)); ((i++==0)) && wait + echo "request to $iter" + log_request_to "https://kaproxy:8080/$iter" & done wait $(jobs -p) diff --git a/kaproxy/Dockerfile b/kaproxy/Dockerfile index 23fdc4e..b9fe88c 100644 --- a/kaproxy/Dockerfile +++ b/kaproxy/Dockerfile @@ -6,8 +6,8 @@ RUN echo "deb http://deb.debian.org/debian-debug/ bookworm-proposed-updates-debu RUN apt update -y RUN apt install libssl3 libssl3-dbgsym openssl openssl-dbgsym libssl-dev zlib1g-dev \ - libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym \ - --allow-downgrades -y + libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym strace \ + procps --allow-downgrades -y COPY nginx.conf / WORKDIR / @@ -24,8 +24,9 @@ WORKDIR /nginx RUN auto/configure \ --with-debug \ --with-http_ssl_module \ + --with-file-aio \ --with-cc-opt="-gdwarf-4 -fno-omit-frame-pointer" -RUN make +RUN make -j $(nproc --all) RUN make install COPY run.sh / diff --git a/kaproxy/gencerts.sh b/kaproxy/gencerts.sh index d133a8e..51615e2 100755 --- a/kaproxy/gencerts.sh +++ b/kaproxy/gencerts.sh @@ -1,6 +1,8 @@ #!/bin/bash -for iter in {0..999}; do +N=$(nproc --all) +for iter in {0..79}; do + ((i=i%N)); ((i++==0)) && wait echo "minting cert $iter" openssl req -x509 \ -newkey rsa:4096 \ @@ -8,18 +10,21 @@ for iter in {0..999}; do -out cert$iter.pem \ -sha256 -nodes \ -days 3650 \ - -quiet \ - -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" + -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" & +done - upstr=$(($iter%10)) - echo ' +for iter in {0..79}; do + echo " location /$iter { proxy_ssl_certificate /cert$iter.pem; proxy_ssl_certificate_key /key$iter.pem; - proxy_pass http://kasvc-$upstr:8080; - }' >> /nginx.conf + proxy_pass https://kasvc-$iter:8080; + }" >> /nginx.conf done + echo ' } } ' >> /nginx.conf + +wait diff --git a/kaproxy/nginx.conf b/kaproxy/nginx.conf index 5f3adf7..b738036 100644 --- a/kaproxy/nginx.conf +++ b/kaproxy/nginx.conf @@ -1,19 +1,26 @@ -worker_processes 10; -error_log /dev/stdout notice; +worker_processes 1; +error_log /dev/stdout debug; pid /tmp/pid; +# callgrind in worker processes must be able to do things +user root; + events { worker_connections 10; } http { keepalive_timeout 300; + aio on; # blocking io blocks tracing + directio 4m; server { listen 8080 ssl; server_name www.example.com; ssl_certificate /www.example.com.crt; ssl_certificate_key /www.example.com.key; + ssl_certificate_cache max=1000; + ssl_session_cache shared:SSL:10m; access_log /tmp/access.log; proxy_socket_keepalive on; diff --git a/kaproxy/run.sh b/kaproxy/run.sh index 7a32bec..fe1fa67 100755 --- a/kaproxy/run.sh +++ b/kaproxy/run.sh @@ -3,10 +3,10 @@ function p_invoke() { valgrind --tool=callgrind \ --trace-children=yes \ - --callgrind-out-file=/tmp/callgrind.output \ + --callgrind-out-file=/tmp/callgrind.out.%p \ --cache-sim=yes \ + --instr-atstart=no \ /nginx/objs/nginx \ - -p /tmp \ -e /tmp/error.log \ -c /nginx.conf \ -g "daemon off;" @@ -14,7 +14,6 @@ function p_invoke() { function invoke() { /nginx/objs/nginx \ - -p /tmp \ -e /tmp/error.log \ -c /nginx.conf \ -g "daemon off;" \ @@ -35,7 +34,14 @@ function sigcont_handler() { trap 'sigint_handler' INT trap 'sigcont_handler' CONT +# enable tracing +echo 1 > /proc/sys/kernel/yama/ptrace_scope + p_invoke & + wait -echo "NGINX down. waiting until signalled..." +echo "NGINX down. waiting to find it again" +sleep 0.5 +wait $(cat /tmp/pid) +echo "NGINX is GONE. waiting until signalled" sleep infinity diff --git a/keepalive-svc.go b/keepalive-svc.go index b092078..e5ab40c 100755 --- a/keepalive-svc.go +++ b/keepalive-svc.go @@ -1,32 +1,95 @@ package main + import ( - "fmt" - "net" + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "math/big" + "net" "net/http" + "os" "time" + "fmt" ) +func generateSelfSignedCert(host string) (tls.Certificate, error) { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(0), + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + } + + cert.DNSNames = []string{host} + certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return tls.Certificate{}, err + } + certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivKey.PublicKey, certPrivKey) + if err != nil { + return tls.Certificate{}, err + } + certPEM := new(bytes.Buffer) + pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + certPrivKeyPEM := new(bytes.Buffer) + pem.Encode(certPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + serverCert, err := tls.X509KeyPair(certPEM.Bytes(), certPrivKeyPEM.Bytes()) + if err != nil { + return tls.Certificate{}, err + } + return serverCert, err +} + type myHandler struct{ nreq int } func (h myHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - fmt.Printf("accepted request number %d", h.nreq) + fmt.Printf("accepted request number %d", h.nreq) fmt.Fprintf(w, "request number: %d", h.nreq) } func main() { + hostname, err := os.Hostname() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + cert, err := generateSelfSignedCert(hostname) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientAuth: tls.RequireAnyClientCert, + } + srv := &http.Server{ Addr: ":8080", Handler: myHandler{nreq: 0}, - ReadTimeout: 5 * time.Second, - WriteTimeout: 10 * time.Second, - IdleTimeout: 300 * time.Second, - ConnState: func(conn net.Conn, event http.ConnState) { - fmt.Printf("addr: %s, changed state to: %s", conn.RemoteAddr(), event.String()) - }, + ReadTimeout: 5 * time.Minute, + WriteTimeout: 5 * time.Minute, + IdleTimeout: 5 * time.Minute, + ConnState: func(conn net.Conn, event http.ConnState) { + fmt.Printf("addr: %s, changed state to: %s", conn.RemoteAddr(), event.String()) + }, + TLSConfig: tlsConfig, } - srv.ListenAndServe() + srv.ListenAndServeTLS("", "") } diff --git a/run.sh b/run.sh index f989958..e5c55bf 100755 --- a/run.sh +++ b/run.sh @@ -35,36 +35,35 @@ echo "[+] building and deploying containers" go build keepalive-svc.go mv keepalive-svc kasvc/ rsync -avz $1 kaproxy/ -sudo docker-compose up --build -d -sudo docker exec -it $KAPROXY callgrind_control -i off +docker-compose up --build -d -sudo docker wait $KACLIENT +docker wait $KACLIENT echo "[+] client finished, triggering reload" -sudo docker exec -it $KAPROXY callgrind_control -i on -sudo docker kill -s CONT $KAPROXY +docker exec $KAPROXY callgrind_control -i on +docker kill -s CONT $KAPROXY echo "[+] wait five seconds for reload complete" sleep 5 -sudo docker exec -it $KAPROXY callgrind_control -i off echo " > restarting client" -sudo docker-compose restart kaclient -sudo docker wait $KACLIENT +docker-compose restart kaclient +docker wait $KACLIENT -echo "[+] client finished again. Killing NGINX and fetching profile data" -sudo docker kill -s INT $KAPROXY -sudo docker exec -it $KAPROXY callgrind_control -d +echo "[+] client finished again. reloading NGINX and fetching profile data" +docker kill -s CONT $KAPROXY +#docker exec $KAPROXY callgrind_control -i off +docker kill -s INT $KAPROXY +sleep 10 echo "[+] building profiling report" -sudo docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file +docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file do echo " > processing: " $file F=$(basename $file) - sudo docker cp $KAPROXY:$file $F; + docker cp $KAPROXY:$file $F; sudo chmod 777 $F echo "Output file: $F" >> $PROFILE_OUTPUT callgrind_annotate \ - --include=kaproxy \ --auto=yes \ $F >> $PROFILE_OUTPUT echo "End of profile: $F\n\n\n" >> $PROFILE_OUTPUT