vm flag and wip vsh rework

Signed-off-by: Ava Hahn <a.hahn@f5.com>
This commit is contained in:
Ava Hahn 2025-01-21 14:25:05 -08:00
parent 8c91778fe1
commit b47476bd69
5 changed files with 266 additions and 195 deletions

View file

@ -8,8 +8,12 @@ This is a set of scripts that roughly does the following:
## Prerequisites
- SECRET.sh must contain the following
- USERN=<username on your VMs>
- PASSP=<password for said user>
```sh
USERN= # username on your VMs
PASSP= # password for said user
VM_PARALLEL= # number of CPU cores a VM has
HOST_PARALLEL= # number of CPU cores to build with locally on the hypervisor
```
- sshpass, virsh, libvirt, etc
- cloned repos of nginx, nginx-tests, and nginx-otel
@ -53,3 +57,26 @@ Invoke `test.sh` with some or all of the following flags:
Logs are in logging directory shown. They are split out into files per VM per phase.
User may set test_log_dir to provide their own logging directory.
When running, you may view the combined status using a command like one of the following.
The logging dir will be specified in the first second of output from the test.sh script.
```sh
tail -f <LOGGING DIR>/on*
tail -f <LOGGING DIR>/sync*
tail -f <LOGGING DIR>/build_nginx*
tail -f <LOGGING DIR>/build_otel*
tail -f <LOGGING DIR>/test_nginx*
tail -f <LOGGING DIR>/test_otel*
tail -f <LOGGING DIR>/off*
```
Additionally, virt.sh and nginx.sh can be sourced in a shell profile so that their
functions can be used at terminal prompt. For example, sourcing virt.sh will provide
the `vsh` function which turns on a VM, opens an SSH connection to it, and then turns it off again. Functions such as `vms_avail`, `vms_on`, and `vms_off` may also be of use.
In the future, routines will be added to nginx.sh that automate several local development and testing processes.
# Contributions
Any contributions which make the procedures defined in these scripts portable across
shells and platforms is welcomed. Any additional feature add for any enclosed script
is also welcomed.

View file

@ -1,23 +1,38 @@
#!/bin/bash
# gets executed ON REMOTE HOST
#function current_dir_is_nginx_repo() {
# local tok
# tok=$(basename -s .git `git config --get remote.origin.url` 2> /dev/null)
# if [[ "$tok" == "nginx" ]]; then
# ret="true"
# return 0
# else
# ret="false"
# return 0
# fi
#}
dirn=$(dirname "$0")
source $dirn/common.sh
if [[ ! -f $dirn/SECRET.sh ]]; then
error "need to create SECRET.sh... see Readme"
exit 1
fi
source $dirn/SECRET.sh
# function that helps set parallelism in builds
# needs to strictly return 3 on VMs and strictly
# return a lot more on the hypervisor host.
function num_jobs() {
if [ ! $VM_PARALLEL ]; then
log "VM_PARALLEL not defined, defaulting to 3"
VM_PARALLEL=3
fi
if [ ! $HOST_PARALLEL ]; then
log "HOST_PARALLEL not defined, defaulting to 22"
HOST_PARALLEL=22
fi
which virsh >/dev/null && echo $HOST_PARALLEL || echo $VM_PARALLEL
}
# The following functions are all run on a VM
# Through an SSH connection. Make sure not to
# use any external functions in them.
function build_nginx_remote() {
function build_nginx() {
auto/configure \
--with-threads \
--with-http_ssl_module \
@ -38,31 +53,30 @@ function build_nginx_remote() {
--with-stream_realip_module \
--with-stream_ssl_preread_module \
--with-debug && \
make -j3
return $?
make -j$(num_jobs)
}
function test_nginx_remote() {
TEST_NGINX_VERBOSE=1 TEST_NGINX_CATLOG=1 prove -vw -j 3 .
return $?
function test_nginx() {
TEST_NGINX_VERBOSE=1 \
TEST_NGINX_CATLOG=1 \
prove -vw -j$(num_jobs) .
}
function clean_nginx_remote() {
function clean_nginx() {
make clean
return $?
}
function build_otel_remote() {
mkdir -p build && cd build && \
cmake -DNGX_OTEL_NGINX_BUILD_DIR=../../nginx/objs .. && \
make -j3
return $?
function build_otel() {
mkdir -p build && \
cd build && \
cmake -DNGX_OTEL_NGINX_BUILD_DIR=../../nginx/objs .. && \
make -j$(num_jobs)
}
function test_otel_remote() {
function test_otel() {
echo "UNIMPLEMENTED!"
}
function clean_otel_remote() {
function clean_otel() {
rm -rf build
}

188
test.sh
View file

@ -5,17 +5,26 @@ source $dirn/common.sh
source $dirn/virt.sh
source $dirn/nginx.sh
if [[ ! $(vms_avail) ]]; then
log "no VMs available!"
exit 1
fi
nginx_dir=""
otel_dir=""
tests_dir=""
vm_list=""
while [ $# -gt 0 ]; do
case $1 in
-h | --help)
log "test.sh: build and test code on many libvirt VMs at once"
log " -h, --help: show this help text"
log " -n <dir>, --nginx <dir>: specify an nginx directory"
log " -o <dir>, --otel <dir>: specify an nginx-otel directory"
log " -t <dir>, --tests <dir>: specify an nginx-tests directory"
log " -n <dir>, --nginx <dir>: specify an nginx directory"
log " -o <dir>, --otel <dir>: specify an nginx-otel directory"
log " -t <dir>, --tests <dir>: specify an nginx-tests directory"
log " --vm <name>: (optional) specify a VM to operate on"
log " may be specified multiple times, or none"
log " defaults to all available VMs"
exit 0
;;
@ -43,6 +52,17 @@ while [ $# -gt 0 ]; do
tests_dir=$2
;;
--vm)
[ $2 ] || ( \
log "VM must be specified" && \
exit 1 )
[[ $vms_available == *"$2"* ]] || ( \
log "VM not available" && \
exit 1 )
[ $vm_list ] && vm_list+="\n"
vm_list+=$2
;;
*)
log "unknown argument: $1"
exit 1
@ -52,9 +72,13 @@ while [ $# -gt 0 ]; do
shift
done
vm_nginx_dir=$(basename $nginx_dir)
vm_otel_dir=$(basename $otel_dir)
vm_tests_dir=$(basename $tests_dir)
[ $nginx_dir ] && vm_nginx_dir=$(basename $nginx_dir)
[ $otel_dir ] && vm_otel_dir=$(basename $otel_dir)
[ $tests_dir ] && vm_tests_dir=$(basename $tests_dir)
if [ ! $vm_list ]; then
vm_list=$(vms_avail)
fi
section "script init..."
if [[ ! -d $test_log_dir ]]; then
@ -70,76 +94,93 @@ log "nginx code dir: $nginx_dir"
log "nginx test dir: $tests_dir"
log "otel code dir: $otel_dir"
function syncs() {
sync_dir_to_vm $1 $nginx_dir
sync_dir_to_vm $1 $tests_dir
sync_dir_to_vm $1 $otel_dir
function sync_all_to_remote() {
# test inverse because function can only return false
# when a command fails
([[ ! $nginx_dir ]] || sync_dir_to_vm $1 $nginx_dir) && \
([[ ! $tests_dir ]] || sync_dir_to_vm $1 $tests_dir) && \
([[ ! $otel_dir ]] || sync_dir_to_vm $1 $otel_dir )
}
function build_nginx() {
vm_shell $1 \
"echo 'BEGIN BUILD'; set -ex; \
$(typeset -f build_nginx_remote); \
cd $vm_nginx_dir; \
build_nginx_remote;"
return $?
function build_nginx_remote() {
vm_cmd $1 \
"set -ex;
$(typeset WHT)
$(typeset END)
$(typeset -f log)
$(typeset -f num_jobs)
$(typeset -f build_nginx)
cd $vm_nginx_dir;
build_nginx"
}
function build_otel() {
vm_shell $1 \
"echo 'BEGIN BUILD'; set -ex; \
$(typeset -f build_otel_remote); \
cd $vm_otel_dir; \
build_otel_remote;"
return $?
function build_otel_remote() {
vm_cmd $1 \
"set -ex;
$(typeset WHT)
$(typeset END)
$(typeset -f log)
$(typeset -f num_jobs)
$(typeset -f build_otel)
cd $vm_otel_dir;
build_otel"
}
function test_nginx() {
vm_shell $1 \
"echo 'BEGIN TESTS'; set -ex; \
$(typeset -f test_nginx_remote); \
cd $vm_tests_dir; \
test_nginx_remote;"
return $?
function test_nginx_remote() {
vm_cmd $1 \
"set -ex;
$(typeset WHT)
$(typeset END)
$(typeset -f log)
$(typeset -f num_jobs)
$(typeset -f test_nginx)
cd $vm_tests_dir;
test_nginx"
}
function test_otel() {
vm_shell $1 \
"echo 'BEGIN TESTS'; set -ex; \
$(typeset -f test_otel_remote); \
cd $vm_otel_dir; \
test_otel_remote;"
return $?
function test_otel_remote() {
vm_cmd $1 \
"set -ex;
$(typeset WHT)
$(typeset END)
$(typeset -f log)
$(typeset -f num_jobs)
$(typeset -f test_otel)
cd $vm_otel_dir;
test_otel"
}
function clean_nginx() {
vm_shell $1 \
"set -ex; \
$(typeset -f clean_nginx_remote); \
cd $vm_nginx_dir; \
clean_nginx_remote;"
return $?
function clean_nginx_remote() {
vm_cmd $1 \
"set -ex;
$(typeset -f clean_nginx)
cd $vm_nginx_dir;
clean_nginx"
}
function clean_otel() {
vm_shell $1 \
"set -ex; \
$(typeset -f clean_otel_remote); \
cd $vm_otel_dir; \
clean_otel_remote;"
return $?
function clean_otel_remote() {
vm_cmd $1 \
"set -ex;
$(typeset -f clean_otel)
cd $vm_otel_dir;
clean_otel"
}
function cleanup() {
section "cleanup!"
log "cleaning build directories"
if ! parallel_invoke_and_wait \
clean_nginx "$vm_list" "$test_log_dir/clean_nginx"; then
clean_nginx_remote \
"$vm_list" \
"$test_log_dir/clean_nginx_"; then
error "Failed to clean NGINX build directory"
fi
if ! parallel_invoke_and_wait \
clean_otel "$vm_list" "$test_log_dir/clean_otel"; then
clean_otel_remote \
"$vm_list" \
"$test_log_dir/clean_otel_"; then
error "Failed to clean otel build directory"
fi
@ -151,14 +192,6 @@ function cleanup() {
}
section "launching VMs"
vms_avail
if [[ "$ret" == "" ]]; then
log "no VMs available!"
exit 1
fi
vm_list=$ret
ret=""
if ! parallel_invoke_and_wait \
turn_on_vm_and_wait \
"$vm_list" \
@ -170,50 +203,61 @@ fi
section "syncing code to VMs"
if ! parallel_invoke_and_wait \
syncs "$vm_list" "$test_log_dir/sync_"; then
sync_all_to_remote \
"$vm_list" \
"$test_log_dir/sync_"; then
error "Failed to sync files to VMs"
cleanup
exit 1
fi
if [ $nginx_dir ]; then
if [ "$nginx_dir" ]; then
section "building NGINX"
if ! parallel_invoke_and_wait \
build_nginx "$vm_list" "$test_log_dir/build_nginx_"; then
build_nginx_remote \
"$vm_list" \
"$test_log_dir/build_nginx_"; then
error "NGINX build failures detected"
cleanup
exit 1
fi
fi
if [ $otel_dir ]; then
if [ "$otel_dir" ]; then
section "building NGINX Otel module"
if ! parallel_invoke_and_wait \
build_otel "$vm_list" "$test_log_dir/build_otel_"; then
build_otel_remote \
"$vm_list" \
"$test_log_dir/build_otel_"; then
error "Otel build failures detected"
cleanup
exit 1
fi
fi
if [ $tests_dir ]; then
if [ $nginx_dir ]; then
if [ "$tests_dir" ]; then
if [ "$nginx_dir" ]; then
section "testing NGINX"
if ! parallel_invoke_and_wait \
test_nginx "$vm_list" "$test_log_dir/test_nginx_"; then
test_nginx_remote \
"$vm_list" \
"$test_log_dir/test_nginx_"; then
error "NGINX test failures detected"
cleanup
exit 1
fi
fi
if [ $otel_dir ]; then
if [ "$otel_dir" ]; then
section "testing NGINX Otel module"
if ! parallel_invoke_and_wait \
test_otel "$vm_list" "$test_log_dir/test_otel_"; then
test_otel_remote \
"$vm_list" \
"$test_log_dir/test_otel_"; then
error "Otel test failures detected"
cleanup
exit 1
fi
fi
fi

36
util.sh Normal file
View file

@ -0,0 +1,36 @@
#!/bin/bash
dirn=$(dirname "$0")
source $dirn/common.sh
source $dirn/virt.sh
source $dirn/nginx.sh
# this file exists to provide logic that requires functionality from one or
# more modules in the rest of the code. It can be sourced directly.
function vsh() {
if ! turn_on_vm_and_wait $1; then
exit 1
fi
sshpass -p $PASSP \
ssh -o PreferredAuthentications=password \
-o StrictHostKeyChecking=no \
$USERN@$(get_vm_ip $1) \
"cat > /tmp/u.sh <<EOF
$(typeset WHT)
$(typeset END)
$(typeset -f log)
$(typeset -f num_jobs)
$(typeset -f build_nginx)
$(typeset -f build_otel)
$(typeset -f test_nginx)
$(typeset -f test_otel)
$(typeset -f clean_nginx)
$(typeset -f clean_otel)
EOF
bash"
turn_off_vm $1
}

138
virt.sh
View file

@ -23,53 +23,44 @@ if [[ ! $PASSP ]]; then
fi
function vms_avail() {
ret=$(sudo virsh list --all --name | sort)
sudo virsh list --all --name | sort
}
function vms_on() {
ret=$(sudo virsh list --name | sort)
sudo virsh list --name | sort
}
function vms_off() {
ret=$(sudo virsh list --inactive --name | sort)
sudo virsh list --inactive --name | sort
}
function get_vm_ip() {
vms_avail
if ! [[ $ret =~ $1 ]]; then
if ! [[ $(vms_avail) =~ $1 ]]; then
log "VM $1 doesnt exist"
ret=""
return 1
fi
vms_on
if ! [[ $ret =~ $1 ]]; then
log "VM $1 already off"
ret=""
if ! [[ $(vms_on) =~ $1 ]]; then
log "VM $1 is off"
return 1
fi
ret=$(sudo virsh net-dhcp-leases default | grep $1 | awk '{print $5}' | rev | cut -c 4- | rev)
sudo virsh net-dhcp-leases default | grep $1 | awk '{print $5}' | rev | cut -c 4- | rev
}
function turn_on_vm_and_wait() {
if [[ ! $1 ]]; then
log "no VM specified"
ret=""
return 1
fi
vms_avail
if ! [[ $ret =~ $1 ]]; then
if ! [[ $(vms_avail) =~ $1 ]]; then
log "VM $1 doesnt exist"
ret=""
return 1
fi
vms_off
if ! [[ $ret =~ $1 ]]; then
if ! [[ $(vms_off) =~ $1 ]]; then
log "VM $1 already on"
ret=""
return 0
fi
@ -78,9 +69,7 @@ function turn_on_vm_and_wait() {
# wait for an IP
ret=""
while [[ "$ret" == "" ]]; do
get_vm_ip $1
while ! [[ $(get_vm_ip $1) ]]; do
sleep 0.5
done
log "Got IP for VM $1"
@ -89,121 +78,82 @@ function turn_on_vm_and_wait() {
# ret set by get_vm_ip above
while ! sshpass -p $PASSP \
ssh -o PreferredAuthentications=password \
-o StrictHostKeyChecking=no $USERN@$ret \
exit; do
sleep 0.1
done
log "Got SSH on VM $1"
}
function vm_shell() {
vms_avail
if ! [[ $ret =~ $1 ]]; then
log "VM $1 doesnt exist"
ret=""
return 1
fi
vms_on
if ! [[ $ret =~ $1 ]]; then
log "VM $1 is off"
ret=""
return 1
fi
if [[ "$2" == "" ]]; then
log "wont execute empty command"
ret=""
return 1
fi
# wait for an IP
ret=""
while [[ "$ret" == "" ]]; do
get_vm_ip $1
sleep 0.5
-o StrictHostKeyChecking=no $USERN@$(get_vm_ip $1) \
exit; do
sleep 0.1
done
# wait for successful ssh
# ret set by get_vm_ip above
sshpass -p $PASSP \
ssh -o PreferredAuthentications=password \
-o StrictHostKeyChecking=no \
$USERN@$ret "$2"
return $?
log "Got SSH on VM $1"
}
function turn_off_vm() {
if [[ ! $1 ]]; then
log "no VM specified"
ret=""
return 1
fi
vms_avail
if ! [[ $ret =~ $1 ]]; then
if ! [[ $(vms_avail) =~ $1 ]]; then
log "VM $1 doesnt exist"
ret=""
return 1
fi
vms_on
if ! [[ $ret =~ $1 ]]; then
if ! [[ $(vms_on) =~ $1 ]]; then
log "VM $1 already off"
ret=""
return 1
return 0
fi
ret=$(sudo virsh shutdown $1)
sudo virsh shutdown $1
}
function sync_dir_to_vm(){
if [ ! -d $2 ]; then
if [[ ! -d $2 ]]; then
log "directory $2 does not exist."
ret=""
return 1
fi
vms_avail
if ! [[ $ret =~ $1 ]]; then
if ! [[ $(vms_avail) =~ $1 ]]; then
log "VM $1 doesnt exist"
ret=""
return 1
fi
vms_on
if ! [[ $ret =~ $1 ]]; then
if ! [[ $(vms_on) =~ $1 ]]; then
log "VM $1 is off"
ret=""
return 1
fi
# wait for an IP
ret=""
while [[ "$ret" == "" ]]; do
get_vm_ip $1
log "waiting for ip"
while [[ ! $(get_vm_ip $1) ]]; do
sleep 0.5
done
sshpass -p $PASSP \
rsync -avze "ssh -o PreferredAuthentications=password -o StrictHostKeyChecking=no" \
$2 $USERN@$ret:
return $?
$2 $USERN@$(get_vm_ip $1):
}
# boots vm, enters SSH, shuts down VM
function vsh() {
# takes care of error cases
if ! turn_on_vm_and_wait $1; then
exit 1
function vm_cmd() {
if ! [[ $(vms_avail) =~ $1 ]]; then
log "VM $1 doesnt exist"
return 1
fi
get_vm_ip $1
if ! [[ $(vms_on) =~ $1 ]]; then
log "VM $1 is off"
return 1
fi
if [[ ! $2 ]]; then
log "wont execute empty command"
return 1
fi
# wait for an IP
while [[ ! $(get_vm_ip $1) ]]; do
sleep 0.5
done
# wait for successful ssh
sshpass -p $PASSP \
ssh -o PreferredAuthentications=password \
-o StrictHostKeyChecking=no \
$USERN@$ret
turn_off_vm $1
$USERN@$(get_vm_ip $1) "$2"
}