affine/nginx-vm-tests
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

vm flag and wip vsh rework

Browse source 
Signed-off-by: Ava Hahn <a.hahn@f5.com>
This commit is contained in:
Ava Hahn 2025-01-21 14:25:05 -08:00
parent 8c91778fe1
commit b47476bd69
5 changed files with 266 additions and 195 deletions
Download patch file Download diff file Expand all files Collapse all files

31
README.md
View file

@ -8,8 +8,12 @@ This is a set of scripts that roughly does the following:
## Prerequisites ## Prerequisites
- SECRET.sh must contain the following - SECRET.sh must contain the following
- USERN=<username on your VMs> ```sh
- PASSP=<password for said user> USERN= # username on your VMs
PASSP= # password for said user
VM_PARALLEL= # number of CPU cores a VM has
HOST_PARALLEL= # number of CPU cores to build with locally on the hypervisor
```
- sshpass, virsh, libvirt, etc - sshpass, virsh, libvirt, etc
- cloned repos of nginx, nginx-tests, and nginx-otel - cloned repos of nginx, nginx-tests, and nginx-otel
@ -53,3 +57,26 @@ Invoke `test.sh` with some or all of the following flags:
Logs are in logging directory shown. They are split out into files per VM per phase. Logs are in logging directory shown. They are split out into files per VM per phase.
User may set test_log_dir to provide their own logging directory. User may set test_log_dir to provide their own logging directory.
When running, you may view the combined status using a command like one of the following.
The logging dir will be specified in the first second of output from the test.sh script.
```sh
tail -f <LOGGING DIR>/on*
tail -f <LOGGING DIR>/sync*
tail -f <LOGGING DIR>/build_nginx*
tail -f <LOGGING DIR>/build_otel*
tail -f <LOGGING DIR>/test_nginx*
tail -f <LOGGING DIR>/test_otel*
tail -f <LOGGING DIR>/off*
```
Additionally, virt.sh and nginx.sh can be sourced in a shell profile so that their
functions can be used at terminal prompt. For example, sourcing virt.sh will provide
the `vsh` function which turns on a VM, opens an SSH connection to it, and then turns it off again. Functions such as `vms_avail`, `vms_on`, and `vms_off` may also be of use.
In the future, routines will be added to nginx.sh that automate several local development and testing processes.
# Contributions
Any contributions which make the procedures defined in these scripts portable across
shells and platforms is welcomed. Any additional feature add for any enclosed script
is also welcomed.

66
nginx.sh
View file

@ -1,23 +1,38 @@
#!/bin/bash #!/bin/bash
# gets executed ON REMOTE HOST dirn=$(dirname "$0")
#function current_dir_is_nginx_repo() { source $dirn/common.sh
# local tok
# tok=$(basename -s .git `git config --get remote.origin.url` 2> /dev/null) if [[ ! -f $dirn/SECRET.sh ]]; then
# if [[ "$tok" == "nginx" ]]; then error "need to create SECRET.sh... see Readme"
# ret="true" exit 1
# return 0 fi
# else
# ret="false" source $dirn/SECRET.sh
# return 0
# fi # function that helps set parallelism in builds
#} # needs to strictly return 3 on VMs and strictly
# return a lot more on the hypervisor host.
function num_jobs() {
if [ ! $VM_PARALLEL ]; then
log "VM_PARALLEL not defined, defaulting to 3"
VM_PARALLEL=3
fi
if [ ! $HOST_PARALLEL ]; then
log "HOST_PARALLEL not defined, defaulting to 22"
HOST_PARALLEL=22
fi
which virsh >/dev/null && echo $HOST_PARALLEL || echo $VM_PARALLEL
}
# The following functions are all run on a VM # The following functions are all run on a VM
# Through an SSH connection. Make sure not to # Through an SSH connection. Make sure not to
# use any external functions in them. # use any external functions in them.
function build_nginx_remote() { function build_nginx() {
auto/configure \ auto/configure \
--with-threads \ --with-threads \
--with-http_ssl_module \ --with-http_ssl_module \
@ -38,31 +53,30 @@ function build_nginx_remote() {
--with-stream_realip_module \ --with-stream_realip_module \
--with-stream_ssl_preread_module \ --with-stream_ssl_preread_module \
--with-debug && \ --with-debug && \
make -j3 make -j$(num_jobs)
return $?
} }
function test_nginx_remote() { function test_nginx() {
TEST_NGINX_VERBOSE=1 TEST_NGINX_CATLOG=1 prove -vw -j 3 . TEST_NGINX_VERBOSE=1 \
return $? TEST_NGINX_CATLOG=1 \
prove -vw -j$(num_jobs) .
} }
function clean_nginx_remote() { function clean_nginx() {
make clean make clean
return $?
} }
function build_otel_remote() { function build_otel() {
mkdir -p build && cd build && \ mkdir -p build && \
cd build && \
cmake -DNGX_OTEL_NGINX_BUILD_DIR=../../nginx/objs .. && \ cmake -DNGX_OTEL_NGINX_BUILD_DIR=../../nginx/objs .. && \
make -j3 make -j$(num_jobs)
return $?
} }
function test_otel_remote() { function test_otel() {
echo "UNIMPLEMENTED!" echo "UNIMPLEMENTED!"
} }
function clean_otel_remote() { function clean_otel() {
rm -rf build rm -rf build
} }

182
test.sh
View file

@ -5,9 +5,15 @@ source $dirn/common.sh
source $dirn/virt.sh source $dirn/virt.sh
source $dirn/nginx.sh source $dirn/nginx.sh
if [[ ! $(vms_avail) ]]; then
log "no VMs available!"
exit 1
fi
nginx_dir="" nginx_dir=""
otel_dir="" otel_dir=""
tests_dir="" tests_dir=""
vm_list=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case $1 in case $1 in
-h | --help) -h | --help)
@ -16,6 +22,9 @@ while [ $# -gt 0 ]; do
log " -n <dir>, --nginx <dir>: specify an nginx directory" log " -n <dir>, --nginx <dir>: specify an nginx directory"
log " -o <dir>, --otel <dir>: specify an nginx-otel directory" log " -o <dir>, --otel <dir>: specify an nginx-otel directory"
log " -t <dir>, --tests <dir>: specify an nginx-tests directory" log " -t <dir>, --tests <dir>: specify an nginx-tests directory"
log " --vm <name>: (optional) specify a VM to operate on"
log " may be specified multiple times, or none"
log " defaults to all available VMs"
exit 0 exit 0
;; ;;
@ -43,6 +52,17 @@ while [ $# -gt 0 ]; do
tests_dir=$2 tests_dir=$2
;; ;;
--vm)
[ $2 ] || ( \
log "VM must be specified" && \
exit 1 )
[[ $vms_available == *"$2"* ]] || ( \
log "VM not available" && \
exit 1 )
[ $vm_list ] && vm_list+="\n"
vm_list+=$2
;;
*) *)
log "unknown argument: $1" log "unknown argument: $1"
exit 1 exit 1
@ -52,9 +72,13 @@ while [ $# -gt 0 ]; do
shift shift
done done
vm_nginx_dir=$(basename $nginx_dir) [ $nginx_dir ] && vm_nginx_dir=$(basename $nginx_dir)
vm_otel_dir=$(basename $otel_dir) [ $otel_dir ] && vm_otel_dir=$(basename $otel_dir)
vm_tests_dir=$(basename $tests_dir) [ $tests_dir ] && vm_tests_dir=$(basename $tests_dir)
if [ ! $vm_list ]; then
vm_list=$(vms_avail)
fi
section "script init..." section "script init..."
if [[ ! -d $test_log_dir ]]; then if [[ ! -d $test_log_dir ]]; then
@ -70,76 +94,93 @@ log "nginx code dir: $nginx_dir"
log "nginx test dir: $tests_dir" log "nginx test dir: $tests_dir"
log "otel code dir: $otel_dir" log "otel code dir: $otel_dir"
function syncs() { function sync_all_to_remote() {
sync_dir_to_vm $1 $nginx_dir # test inverse because function can only return false
sync_dir_to_vm $1 $tests_dir # when a command fails
sync_dir_to_vm $1 $otel_dir
([[ ! $nginx_dir ]] || sync_dir_to_vm $1 $nginx_dir) && \
([[ ! $tests_dir ]] || sync_dir_to_vm $1 $tests_dir) && \
([[ ! $otel_dir ]] || sync_dir_to_vm $1 $otel_dir )
} }
function build_nginx() { function build_nginx_remote() {
vm_shell $1 \ vm_cmd $1 \
"echo 'BEGIN BUILD'; set -ex; \ "set -ex;
$(typeset -f build_nginx_remote); \ $(typeset WHT)
cd $vm_nginx_dir; \ $(typeset END)
build_nginx_remote;" $(typeset -f log)
return $? $(typeset -f num_jobs)
$(typeset -f build_nginx)
cd $vm_nginx_dir;
build_nginx"
} }
function build_otel() { function build_otel_remote() {
vm_shell $1 \ vm_cmd $1 \
"echo 'BEGIN BUILD'; set -ex; \ "set -ex;
$(typeset -f build_otel_remote); \ $(typeset WHT)
cd $vm_otel_dir; \ $(typeset END)
build_otel_remote;" $(typeset -f log)
return $? $(typeset -f num_jobs)
$(typeset -f build_otel)
cd $vm_otel_dir;
build_otel"
} }
function test_nginx() { function test_nginx_remote() {
vm_shell $1 \ vm_cmd $1 \
"echo 'BEGIN TESTS'; set -ex; \ "set -ex;
$(typeset -f test_nginx_remote); \ $(typeset WHT)
cd $vm_tests_dir; \ $(typeset END)
test_nginx_remote;" $(typeset -f log)
return $? $(typeset -f num_jobs)
$(typeset -f test_nginx)
cd $vm_tests_dir;
test_nginx"
} }
function test_otel() { function test_otel_remote() {
vm_shell $1 \ vm_cmd $1 \
"echo 'BEGIN TESTS'; set -ex; \ "set -ex;
$(typeset -f test_otel_remote); \ $(typeset WHT)
cd $vm_otel_dir; \ $(typeset END)
test_otel_remote;" $(typeset -f log)
return $? $(typeset -f num_jobs)
$(typeset -f test_otel)
cd $vm_otel_dir;
test_otel"
} }
function clean_nginx() { function clean_nginx_remote() {
vm_shell $1 \ vm_cmd $1 \
"set -ex; \ "set -ex;
$(typeset -f clean_nginx_remote); \ $(typeset -f clean_nginx)
cd $vm_nginx_dir; \ cd $vm_nginx_dir;
clean_nginx_remote;" clean_nginx"
return $?
} }
function clean_otel() { function clean_otel_remote() {
vm_shell $1 \ vm_cmd $1 \
"set -ex; \ "set -ex;
$(typeset -f clean_otel_remote); \ $(typeset -f clean_otel)
cd $vm_otel_dir; \ cd $vm_otel_dir;
clean_otel_remote;" clean_otel"
return $?
} }
function cleanup() { function cleanup() {
section "cleanup!" section "cleanup!"
log "cleaning build directories" log "cleaning build directories"
if ! parallel_invoke_and_wait \ if ! parallel_invoke_and_wait \
clean_nginx "$vm_list" "$test_log_dir/clean_nginx"; then clean_nginx_remote \
"$vm_list" \
"$test_log_dir/clean_nginx_"; then
error "Failed to clean NGINX build directory" error "Failed to clean NGINX build directory"
fi fi
if ! parallel_invoke_and_wait \ if ! parallel_invoke_and_wait \
clean_otel "$vm_list" "$test_log_dir/clean_otel"; then clean_otel_remote \
"$vm_list" \
"$test_log_dir/clean_otel_"; then
error "Failed to clean otel build directory" error "Failed to clean otel build directory"
fi fi
@ -151,14 +192,6 @@ function cleanup() {
} }
section "launching VMs" section "launching VMs"
vms_avail
if [[ "$ret" == "" ]]; then
log "no VMs available!"
exit 1
fi
vm_list=$ret
ret=""
if ! parallel_invoke_and_wait \ if ! parallel_invoke_and_wait \
turn_on_vm_and_wait \ turn_on_vm_and_wait \
"$vm_list" \ "$vm_list" \
@ -170,52 +203,63 @@ fi
section "syncing code to VMs" section "syncing code to VMs"
if ! parallel_invoke_and_wait \ if ! parallel_invoke_and_wait \
syncs "$vm_list" "$test_log_dir/sync_"; then sync_all_to_remote \
"$vm_list" \
"$test_log_dir/sync_"; then
error "Failed to sync files to VMs" error "Failed to sync files to VMs"
cleanup cleanup
exit 1 exit 1
fi fi
if [ $nginx_dir ]; then if [ "$nginx_dir" ]; then
section "building NGINX" section "building NGINX"
if ! parallel_invoke_and_wait \ if ! parallel_invoke_and_wait \
build_nginx "$vm_list" "$test_log_dir/build_nginx_"; then build_nginx_remote \
"$vm_list" \
"$test_log_dir/build_nginx_"; then
error "NGINX build failures detected" error "NGINX build failures detected"
cleanup cleanup
exit 1 exit 1
fi fi
fi fi
if [ $otel_dir ]; then if [ "$otel_dir" ]; then
section "building NGINX Otel module" section "building NGINX Otel module"
if ! parallel_invoke_and_wait \ if ! parallel_invoke_and_wait \
build_otel "$vm_list" "$test_log_dir/build_otel_"; then build_otel_remote \
"$vm_list" \
"$test_log_dir/build_otel_"; then
error "Otel build failures detected" error "Otel build failures detected"
cleanup cleanup
exit 1 exit 1
fi fi
fi fi
if [ $tests_dir ]; then if [ "$tests_dir" ]; then
if [ $nginx_dir ]; then if [ "$nginx_dir" ]; then
section "testing NGINX" section "testing NGINX"
if ! parallel_invoke_and_wait \ if ! parallel_invoke_and_wait \
test_nginx "$vm_list" "$test_log_dir/test_nginx_"; then test_nginx_remote \
"$vm_list" \
"$test_log_dir/test_nginx_"; then
error "NGINX test failures detected" error "NGINX test failures detected"
cleanup cleanup
exit 1 exit 1
fi fi
fi fi
if [ $otel_dir ]; then if [ "$otel_dir" ]; then
section "testing NGINX Otel module" section "testing NGINX Otel module"
if ! parallel_invoke_and_wait \ if ! parallel_invoke_and_wait \
test_otel "$vm_list" "$test_log_dir/test_otel_"; then test_otel_remote \
"$vm_list" \
"$test_log_dir/test_otel_"; then
error "Otel test failures detected" error "Otel test failures detected"
cleanup cleanup
exit 1 exit 1
fi fi
fi fi
fi
# ------ # ------
cleanup cleanup

36
util.sh Normal file
View file

@ -0,0 +1,36 @@
#!/bin/bash
dirn=$(dirname "$0")
source $dirn/common.sh
source $dirn/virt.sh
source $dirn/nginx.sh
# this file exists to provide logic that requires functionality from one or
# more modules in the rest of the code. It can be sourced directly.
function vsh() {
if ! turn_on_vm_and_wait $1; then
exit 1
fi
sshpass -p $PASSP \
ssh -o PreferredAuthentications=password \
-o StrictHostKeyChecking=no \
$USERN@$(get_vm_ip $1) \
"cat > /tmp/u.sh <<EOF
$(typeset WHT)
$(typeset END)
$(typeset -f log)
$(typeset -f num_jobs)
$(typeset -f build_nginx)
$(typeset -f build_otel)
$(typeset -f test_nginx)
$(typeset -f test_otel)
$(typeset -f clean_nginx)
$(typeset -f clean_otel)
EOF
bash"
turn_off_vm $1
}

132
virt.sh
View file

@ -23,53 +23,44 @@ if [[ ! $PASSP ]]; then
fi fi
function vms_avail() { function vms_avail() {
ret=$(sudo virsh list --all --name | sort) sudo virsh list --all --name | sort
} }
function vms_on() { function vms_on() {
ret=$(sudo virsh list --name | sort) sudo virsh list --name | sort
} }
function vms_off() { function vms_off() {
ret=$(sudo virsh list --inactive --name | sort) sudo virsh list --inactive --name | sort
} }
function get_vm_ip() { function get_vm_ip() {
vms_avail if ! [[ $(vms_avail) =~ $1 ]]; then
if ! [[ $ret =~ $1 ]]; then
log "VM $1 doesnt exist" log "VM $1 doesnt exist"
ret=""
return 1 return 1
fi fi
vms_on if ! [[ $(vms_on) =~ $1 ]]; then
if ! [[ $ret =~ $1 ]]; then log "VM $1 is off"
log "VM $1 already off"
ret=""
return 1 return 1
fi fi
ret=$(sudo virsh net-dhcp-leases default | grep $1 | awk '{print $5}' | rev | cut -c 4- | rev) sudo virsh net-dhcp-leases default | grep $1 | awk '{print $5}' | rev | cut -c 4- | rev
} }
function turn_on_vm_and_wait() { function turn_on_vm_and_wait() {
if [[ ! $1 ]]; then if [[ ! $1 ]]; then
log "no VM specified" log "no VM specified"
ret=""
return 1 return 1
fi fi
vms_avail if ! [[ $(vms_avail) =~ $1 ]]; then
if ! [[ $ret =~ $1 ]]; then
log "VM $1 doesnt exist" log "VM $1 doesnt exist"
ret=""
return 1 return 1
fi fi
vms_off if ! [[ $(vms_off) =~ $1 ]]; then
if ! [[ $ret =~ $1 ]]; then
log "VM $1 already on" log "VM $1 already on"
ret=""
return 0 return 0
fi fi
@ -78,9 +69,7 @@ function turn_on_vm_and_wait() {
# wait for an IP # wait for an IP
ret="" while ! [[ $(get_vm_ip $1) ]]; do
while [[ "$ret" == "" ]]; do
get_vm_ip $1
sleep 0.5 sleep 0.5
done done
log "Got IP for VM $1" log "Got IP for VM $1"
@ -89,121 +78,82 @@ function turn_on_vm_and_wait() {
# ret set by get_vm_ip above # ret set by get_vm_ip above
while ! sshpass -p $PASSP \ while ! sshpass -p $PASSP \
ssh -o PreferredAuthentications=password \ ssh -o PreferredAuthentications=password \
-o StrictHostKeyChecking=no $USERN@$ret \ -o StrictHostKeyChecking=no $USERN@$(get_vm_ip $1) \
exit; do exit; do
sleep 0.1 sleep 0.1
done done
log "Got SSH on VM $1" log "Got SSH on VM $1"
} }
function vm_shell() {
vms_avail
if ! [[ $ret =~ $1 ]]; then
log "VM $1 doesnt exist"
ret=""
return 1
fi
vms_on
if ! [[ $ret =~ $1 ]]; then
log "VM $1 is off"
ret=""
return 1
fi
if [[ "$2" == "" ]]; then
log "wont execute empty command"
ret=""
return 1
fi
# wait for an IP
ret=""
while [[ "$ret" == "" ]]; do
get_vm_ip $1
sleep 0.5
done
# wait for successful ssh
# ret set by get_vm_ip above
sshpass -p $PASSP \
ssh -o PreferredAuthentications=password \
-o StrictHostKeyChecking=no \
$USERN@$ret "$2"
return $?
}
function turn_off_vm() { function turn_off_vm() {
if [[ ! $1 ]]; then if [[ ! $1 ]]; then
log "no VM specified" log "no VM specified"
ret=""
return 1 return 1
fi fi
vms_avail if ! [[ $(vms_avail) =~ $1 ]]; then
if ! [[ $ret =~ $1 ]]; then
log "VM $1 doesnt exist" log "VM $1 doesnt exist"
ret=""
return 1 return 1
fi fi
vms_on if ! [[ $(vms_on) =~ $1 ]]; then
if ! [[ $ret =~ $1 ]]; then
log "VM $1 already off" log "VM $1 already off"
ret="" return 0
return 1
fi fi
ret=$(sudo virsh shutdown $1) sudo virsh shutdown $1
} }
function sync_dir_to_vm(){ function sync_dir_to_vm(){
if [ ! -d $2 ]; then if [[ ! -d $2 ]]; then
log "directory $2 does not exist." log "directory $2 does not exist."
ret=""
return 1 return 1
fi fi
vms_avail if ! [[ $(vms_avail) =~ $1 ]]; then
if ! [[ $ret =~ $1 ]]; then
log "VM $1 doesnt exist" log "VM $1 doesnt exist"
ret=""
return 1 return 1
fi fi
vms_on if ! [[ $(vms_on) =~ $1 ]]; then
if ! [[ $ret =~ $1 ]]; then
log "VM $1 is off" log "VM $1 is off"
ret=""
return 1 return 1
fi fi
# wait for an IP # wait for an IP
ret="" while [[ ! $(get_vm_ip $1) ]]; do
while [[ "$ret" == "" ]]; do
get_vm_ip $1
log "waiting for ip"
sleep 0.5 sleep 0.5
done done
sshpass -p $PASSP \ sshpass -p $PASSP \
rsync -avze "ssh -o PreferredAuthentications=password -o StrictHostKeyChecking=no" \ rsync -avze "ssh -o PreferredAuthentications=password -o StrictHostKeyChecking=no" \
$2 $USERN@$ret: $2 $USERN@$(get_vm_ip $1):
return $?
} }
# boots vm, enters SSH, shuts down VM function vm_cmd() {
function vsh() { if ! [[ $(vms_avail) =~ $1 ]]; then
# takes care of error cases log "VM $1 doesnt exist"
if ! turn_on_vm_and_wait $1; then return 1
exit 1
fi fi
get_vm_ip $1 if ! [[ $(vms_on) =~ $1 ]]; then
log "VM $1 is off"
return 1
fi
if [[ ! $2 ]]; then
log "wont execute empty command"
return 1
fi
# wait for an IP
while [[ ! $(get_vm_ip $1) ]]; do
sleep 0.5
done
# wait for successful ssh
sshpass -p $PASSP \ sshpass -p $PASSP \
ssh -o PreferredAuthentications=password \ ssh -o PreferredAuthentications=password \
-o StrictHostKeyChecking=no \ -o StrictHostKeyChecking=no \
$USERN@$ret $USERN@$(get_vm_ip $1) "$2"
turn_off_vm $1
} }