diff --git a/compose.yaml b/compose.yaml index ab02a86..3d5b71c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -50,286 +50,6 @@ services: <<: *kasvc ports: - 8089:8080 - kasvc-10: - <<: *kasvc - ports: - - 8090:8080 - kasvc-11: - <<: *kasvc - ports: - - 8091:8080 - kasvc-12: - <<: *kasvc - ports: - - 8092:8080 - kasvc-13: - <<: *kasvc - ports: - - 8093:8080 - kasvc-14: - <<: *kasvc - ports: - - 8094:8080 - kasvc-15: - <<: *kasvc - ports: - - 8095:8080 - kasvc-16: - <<: *kasvc - ports: - - 8096:8080 - kasvc-17: - <<: *kasvc - ports: - - 8097:8080 - kasvc-18: - <<: *kasvc - ports: - - 8098:8080 - kasvc-19: - <<: *kasvc - ports: - - 8099:8080 - kasvc-20: - <<: *kasvc - ports: - - 8100:8080 - kasvc-21: - <<: *kasvc - ports: - - 8101:8080 - kasvc-22: - <<: *kasvc - ports: - - 8102:8080 - kasvc-23: - <<: *kasvc - ports: - - 8103:8080 - kasvc-24: - <<: *kasvc - ports: - - 8104:8080 - kasvc-25: - <<: *kasvc - ports: - - 8105:8080 - kasvc-26: - <<: *kasvc - ports: - - 8106:8080 - kasvc-27: - <<: *kasvc - ports: - - 8107:8080 - kasvc-28: - <<: *kasvc - ports: - - 8108:8080 - kasvc-29: - <<: *kasvc - ports: - - 8109:8080 - kasvc-30: - <<: *kasvc - ports: - - 8110:8080 - kasvc-31: - <<: *kasvc - ports: - - 8111:8080 - kasvc-32: - <<: *kasvc - ports: - - 8112:8080 - kasvc-33: - <<: *kasvc - ports: - - 8113:8080 - kasvc-34: - <<: *kasvc - ports: - - 8114:8080 - kasvc-35: - <<: *kasvc - ports: - - 8115:8080 - kasvc-36: - <<: *kasvc - ports: - - 8116:8080 - kasvc-37: - <<: *kasvc - ports: - - 8117:8080 - kasvc-38: - <<: *kasvc - ports: - - 8118:8080 - kasvc-39: - <<: *kasvc - ports: - - 8119:8080 - kasvc-40: - <<: *kasvc - ports: - - 8120:8080 - kasvc-41: - <<: *kasvc - ports: - - 8121:8080 - kasvc-42: - <<: *kasvc - ports: - - 8122:8080 - kasvc-43: - <<: *kasvc - ports: - - 8123:8080 - kasvc-44: - <<: *kasvc - ports: - - 8124:8080 - kasvc-45: - <<: *kasvc - ports: - - 8125:8080 - kasvc-46: - <<: *kasvc - ports: - - 8126:8080 - kasvc-47: - <<: *kasvc - ports: - - 8127:8080 - kasvc-48: - <<: *kasvc - ports: - - 8128:8080 - kasvc-49: - <<: *kasvc - ports: - - 8129:8080 - kasvc-50: - <<: *kasvc - ports: - - 8130:8080 - kasvc-51: - <<: *kasvc - ports: - - 8131:8080 - kasvc-52: - <<: *kasvc - ports: - - 8132:8080 - kasvc-53: - <<: *kasvc - ports: - - 8133:8080 - kasvc-54: - <<: *kasvc - ports: - - 8134:8080 - kasvc-55: - <<: *kasvc - ports: - - 8135:8080 - kasvc-56: - <<: *kasvc - ports: - - 8136:8080 - kasvc-57: - <<: *kasvc - ports: - - 8137:8080 - kasvc-58: - <<: *kasvc - ports: - - 8138:8080 - kasvc-59: - <<: *kasvc - ports: - - 8139:8080 - kasvc-60: - <<: *kasvc - ports: - - 8140:8080 - kasvc-61: - <<: *kasvc - ports: - - 8141:8080 - kasvc-62: - <<: *kasvc - ports: - - 8142:8080 - kasvc-63: - <<: *kasvc - ports: - - 8143:8080 - kasvc-64: - <<: *kasvc - ports: - - 8144:8080 - kasvc-65: - <<: *kasvc - ports: - - 8145:8080 - kasvc-66: - <<: *kasvc - ports: - - 8146:8080 - kasvc-67: - <<: *kasvc - ports: - - 8147:8080 - kasvc-68: - <<: *kasvc - ports: - - 8148:8080 - kasvc-69: - <<: *kasvc - ports: - - 8149:8080 - kasvc-70: - <<: *kasvc - ports: - - 8150:8080 - kasvc-71: - <<: *kasvc - ports: - - 8151:8080 - kasvc-72: - <<: *kasvc - ports: - - 8152:8080 - kasvc-73: - <<: *kasvc - ports: - - 8153:8080 - kasvc-74: - <<: *kasvc - ports: - - 8154:8080 - kasvc-75: - <<: *kasvc - ports: - - 8155:8080 - kasvc-76: - <<: *kasvc - ports: - - 8156:8080 - kasvc-77: - <<: *kasvc - ports: - - 8157:8080 - kasvc-78: - <<: *kasvc - ports: - - 8158:8080 - kasvc-79: - <<: *kasvc - ports: - - 8159:8080 kaproxy: build: @@ -339,8 +59,6 @@ services: - linux/x86_64 platform: linux/x86_64 privileged: true - cap_add: - - SYS_PTRACE ports: - 8079:8080 networks: diff --git a/kaclient/run.sh b/kaclient/run.sh index 198c459..6fa2756 100755 --- a/kaclient/run.sh +++ b/kaclient/run.sh @@ -1,7 +1,5 @@ #!/bin/bash -N=$(nproc --all) - function log_request_to () { return_code=$(curl -Sikl -o /dev/null -w "%{http_code}" $1 2>/dev/null) case ${return_code:0:1} in @@ -15,6 +13,10 @@ function log_request_to () { esac } +function do_wrk_on () { + /wrk/wrk -t1 -c10 $1 & +} + function sigint_handler() { jobs -p | xargs kill -9 exit @@ -24,13 +26,10 @@ trap 'sigint_handler' INT # TODO: make this a more elegant item # maybe a while loop with curl -sleep 2 +sleep 0.5 -echo "[+] client making request loop" -for iter in {0..80}; do - ((i=i%N)); ((i++==0)) && wait - echo "request to $iter" - log_request_to "https://kaproxy:8080/$iter" & +for iter in {0.999}; do + do_wrk_on "https://kaproxy:8080/$iter" done wait $(jobs -p) diff --git a/kaproxy/Dockerfile b/kaproxy/Dockerfile index b9fe88c..23fdc4e 100644 --- a/kaproxy/Dockerfile +++ b/kaproxy/Dockerfile @@ -6,8 +6,8 @@ RUN echo "deb http://deb.debian.org/debian-debug/ bookworm-proposed-updates-debu RUN apt update -y RUN apt install libssl3 libssl3-dbgsym openssl openssl-dbgsym libssl-dev zlib1g-dev \ - libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym strace \ - procps --allow-downgrades -y + libc6-dbg gcc make mk-configure valgrind libpcre2-dev libgcrypt20-dbgsym \ + --allow-downgrades -y COPY nginx.conf / WORKDIR / @@ -24,9 +24,8 @@ WORKDIR /nginx RUN auto/configure \ --with-debug \ --with-http_ssl_module \ - --with-file-aio \ --with-cc-opt="-gdwarf-4 -fno-omit-frame-pointer" -RUN make -j $(nproc --all) +RUN make RUN make install COPY run.sh / diff --git a/kaproxy/gencerts.sh b/kaproxy/gencerts.sh index 51615e2..d133a8e 100755 --- a/kaproxy/gencerts.sh +++ b/kaproxy/gencerts.sh @@ -1,8 +1,6 @@ #!/bin/bash -N=$(nproc --all) -for iter in {0..79}; do - ((i=i%N)); ((i++==0)) && wait +for iter in {0..999}; do echo "minting cert $iter" openssl req -x509 \ -newkey rsa:4096 \ @@ -10,21 +8,18 @@ for iter in {0..79}; do -out cert$iter.pem \ -sha256 -nodes \ -days 3650 \ - -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" & -done + -quiet \ + -subj "/C=XX/ST=StateName/L=CityName/O=CompanyName/OU=CompanySectionName/CN=kaproxy-$iter" -for iter in {0..79}; do - echo " + upstr=$(($iter%10)) + echo ' location /$iter { proxy_ssl_certificate /cert$iter.pem; proxy_ssl_certificate_key /key$iter.pem; - proxy_pass https://kasvc-$iter:8080; - }" >> /nginx.conf + proxy_pass http://kasvc-$upstr:8080; + }' >> /nginx.conf done - echo ' } } ' >> /nginx.conf - -wait diff --git a/kaproxy/nginx.conf b/kaproxy/nginx.conf index b738036..5f3adf7 100644 --- a/kaproxy/nginx.conf +++ b/kaproxy/nginx.conf @@ -1,26 +1,19 @@ -worker_processes 1; -error_log /dev/stdout debug; +worker_processes 10; +error_log /dev/stdout notice; pid /tmp/pid; -# callgrind in worker processes must be able to do things -user root; - events { worker_connections 10; } http { keepalive_timeout 300; - aio on; # blocking io blocks tracing - directio 4m; server { listen 8080 ssl; server_name www.example.com; ssl_certificate /www.example.com.crt; ssl_certificate_key /www.example.com.key; - ssl_certificate_cache max=1000; - ssl_session_cache shared:SSL:10m; access_log /tmp/access.log; proxy_socket_keepalive on; diff --git a/kaproxy/run.sh b/kaproxy/run.sh index fe1fa67..7a32bec 100755 --- a/kaproxy/run.sh +++ b/kaproxy/run.sh @@ -3,10 +3,10 @@ function p_invoke() { valgrind --tool=callgrind \ --trace-children=yes \ - --callgrind-out-file=/tmp/callgrind.out.%p \ + --callgrind-out-file=/tmp/callgrind.output \ --cache-sim=yes \ - --instr-atstart=no \ /nginx/objs/nginx \ + -p /tmp \ -e /tmp/error.log \ -c /nginx.conf \ -g "daemon off;" @@ -14,6 +14,7 @@ function p_invoke() { function invoke() { /nginx/objs/nginx \ + -p /tmp \ -e /tmp/error.log \ -c /nginx.conf \ -g "daemon off;" \ @@ -34,14 +35,7 @@ function sigcont_handler() { trap 'sigint_handler' INT trap 'sigcont_handler' CONT -# enable tracing -echo 1 > /proc/sys/kernel/yama/ptrace_scope - p_invoke & - wait -echo "NGINX down. waiting to find it again" -sleep 0.5 -wait $(cat /tmp/pid) -echo "NGINX is GONE. waiting until signalled" +echo "NGINX down. waiting until signalled..." sleep infinity diff --git a/keepalive-svc.go b/keepalive-svc.go index e5ab40c..b092078 100755 --- a/keepalive-svc.go +++ b/keepalive-svc.go @@ -1,95 +1,32 @@ package main - import ( - "bytes" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "math/big" - "net" - "net/http" - "os" - "time" "fmt" + "net" + "net/http" + "time" ) -func generateSelfSignedCert(host string) (tls.Certificate, error) { - cert := &x509.Certificate{ - SerialNumber: big.NewInt(0), - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(10, 0, 0), - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature, - BasicConstraintsValid: true, - } - - cert.DNSNames = []string{host} - certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - return tls.Certificate{}, err - } - certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivKey.PublicKey, certPrivKey) - if err != nil { - return tls.Certificate{}, err - } - certPEM := new(bytes.Buffer) - pem.Encode(certPEM, &pem.Block{ - Type: "CERTIFICATE", - Bytes: certBytes, - }) - certPrivKeyPEM := new(bytes.Buffer) - pem.Encode(certPrivKeyPEM, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), - }) - serverCert, err := tls.X509KeyPair(certPEM.Bytes(), certPrivKeyPEM.Bytes()) - if err != nil { - return tls.Certificate{}, err - } - return serverCert, err -} - type myHandler struct{ nreq int } func (h myHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - fmt.Printf("accepted request number %d", h.nreq) + fmt.Printf("accepted request number %d", h.nreq) fmt.Fprintf(w, "request number: %d", h.nreq) } func main() { - hostname, err := os.Hostname() - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - cert, err := generateSelfSignedCert(hostname) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - ClientAuth: tls.RequireAnyClientCert, - } - srv := &http.Server{ Addr: ":8080", Handler: myHandler{nreq: 0}, - ReadTimeout: 5 * time.Minute, - WriteTimeout: 5 * time.Minute, - IdleTimeout: 5 * time.Minute, - ConnState: func(conn net.Conn, event http.ConnState) { - fmt.Printf("addr: %s, changed state to: %s", conn.RemoteAddr(), event.String()) - }, - TLSConfig: tlsConfig, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 300 * time.Second, + ConnState: func(conn net.Conn, event http.ConnState) { + fmt.Printf("addr: %s, changed state to: %s", conn.RemoteAddr(), event.String()) + }, } - srv.ListenAndServeTLS("", "") + srv.ListenAndServe() } diff --git a/run.sh b/run.sh index e5c55bf..f989958 100755 --- a/run.sh +++ b/run.sh @@ -35,35 +35,36 @@ echo "[+] building and deploying containers" go build keepalive-svc.go mv keepalive-svc kasvc/ rsync -avz $1 kaproxy/ -docker-compose up --build -d +sudo docker-compose up --build -d +sudo docker exec -it $KAPROXY callgrind_control -i off -docker wait $KACLIENT +sudo docker wait $KACLIENT echo "[+] client finished, triggering reload" -docker exec $KAPROXY callgrind_control -i on -docker kill -s CONT $KAPROXY +sudo docker exec -it $KAPROXY callgrind_control -i on +sudo docker kill -s CONT $KAPROXY echo "[+] wait five seconds for reload complete" sleep 5 +sudo docker exec -it $KAPROXY callgrind_control -i off echo " > restarting client" -docker-compose restart kaclient -docker wait $KACLIENT +sudo docker-compose restart kaclient +sudo docker wait $KACLIENT -echo "[+] client finished again. reloading NGINX and fetching profile data" -docker kill -s CONT $KAPROXY -#docker exec $KAPROXY callgrind_control -i off -docker kill -s INT $KAPROXY -sleep 10 +echo "[+] client finished again. Killing NGINX and fetching profile data" +sudo docker kill -s INT $KAPROXY +sudo docker exec -it $KAPROXY callgrind_control -d echo "[+] building profiling report" -docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file +sudo docker exec $KAPROXY bash -c "find /tmp -iname \"callgrind.out*\"" | while read file do echo " > processing: " $file F=$(basename $file) - docker cp $KAPROXY:$file $F; + sudo docker cp $KAPROXY:$file $F; sudo chmod 777 $F echo "Output file: $F" >> $PROFILE_OUTPUT callgrind_annotate \ + --include=kaproxy \ --auto=yes \ $F >> $PROFILE_OUTPUT echo "End of profile: $F\n\n\n" >> $PROFILE_OUTPUT