import to github
Signed-off-by: Ava Affine <ava@sunnypup.io>
This commit is contained in:
Ava Hahn
commit
8c91778fe1
7 changed files with 638 additions and 0 deletions
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
SECRET.sh
|
||||
12
LICENSE
Normal file
12
LICENSE
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
Copyright 2025 Ava Hahn
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
55
README.md
Normal file
55
README.md
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
# Testing Framework
|
||||
This is a set of scripts that roughly does the following:
|
||||
- launches available libvirt VMs on host
|
||||
- synchronizes code and test code to each VM
|
||||
- runs a build script on each VM
|
||||
- runs a test script on each VM
|
||||
- collects job status and logs and stores them locally
|
||||
|
||||
## Prerequisites
|
||||
- SECRET.sh must contain the following
|
||||
- USERN=<username on your VMs>
|
||||
- PASSP=<password for said user>
|
||||
- sshpass, virsh, libvirt, etc
|
||||
- cloned repos of nginx, nginx-tests, and nginx-otel
|
||||
|
||||
## About those VMs....
|
||||
- hostname and libvirt domain name need to be same for each
|
||||
- username and password should be the same on all of them
|
||||
- whatever your test functions need (see `nginx.sh`)
|
||||
- git
|
||||
- make
|
||||
- gcc
|
||||
- zlib
|
||||
- pcre
|
||||
- openssl
|
||||
- rsyncz
|
||||
- perl and perl-utils (for prove)
|
||||
|
||||
For Otel module build and tests:
|
||||
- cmake
|
||||
- c-ares
|
||||
- linux-headers
|
||||
- g++ / clang++ / etc
|
||||
|
||||
### FreeBSD
|
||||
- need to install bash
|
||||
- need to set login shell to bash
|
||||
|
||||
### Fedora
|
||||
- install zlib-ng-compat-devel and zlib-ng-compat-static for zlib, not zlibrary-devel or zlib-ng-devel.
|
||||
- fedora also seems to need the openssl-devel-engine package.
|
||||
|
||||
### Alpine
|
||||
- need to install clang instead of gcc
|
||||
|
||||
## Usage
|
||||
Invoke `test.sh` with some or all of the following flags:
|
||||
- `--nginx <nginx>` takes an nginx code directory and builds it on remote hosts
|
||||
- `--otel <otel>` takes an nginx-otel directory and builds it on remote hosts
|
||||
- `--tests <tests>` takes an nginx-tests directory and runs tests on remote hosts
|
||||
requires that `--nginx=...` also be supplied. If otel was supplied this also
|
||||
triggers testing in otel directory.
|
||||
|
||||
Logs are in logging directory shown. They are split out into files per VM per phase.
|
||||
User may set test_log_dir to provide their own logging directory.
|
||||
71
common.sh
Normal file
71
common.sh
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
#!/bin/bash
|
||||
|
||||
RED='\033[1;31m'
|
||||
GRN='\033[1;32m'
|
||||
YEL='\033[1;33m'
|
||||
BLU='\033[1;34m'
|
||||
WHT='\033[1;37m'
|
||||
MGT='\033[1;95m'
|
||||
CYA='\033[1;96m'
|
||||
END='\033[0m'
|
||||
|
||||
function section() {
|
||||
echo ""
|
||||
log "***** Section: ${MGT}$1${END} *****";
|
||||
echo ""
|
||||
}
|
||||
|
||||
function log() { >&2 printf "${WHT}#${END} $1\n"; }
|
||||
|
||||
function error() { >&2 printf "${WHT}#${END} ${RED}$1${END}\n"; }
|
||||
|
||||
# takes a function and many inputs, runs function on each input in parallel
|
||||
# inputs should be stored deliniated by newlines in $2
|
||||
# if $3 exists it will be a stub for logging filename
|
||||
function parallel_invoke_and_wait() {
|
||||
local procedure=$1
|
||||
local pids=()
|
||||
local rets=()
|
||||
|
||||
if [[ ! $1 ]]; then
|
||||
log "failed to invoke null procedure"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ ! $2 ]]; then
|
||||
log "failed to invoke procedure on 0 inputs"
|
||||
return 1
|
||||
fi
|
||||
|
||||
IFS=$'\n'
|
||||
for input in $2; do
|
||||
log "invoking procedure with input $input"
|
||||
if [[ $3 ]]; then
|
||||
$procedure $input &>${3}${input}.log &
|
||||
else
|
||||
$procedure $input &
|
||||
fi
|
||||
pids+=("$input/$!")
|
||||
done
|
||||
|
||||
cf="true"
|
||||
for pid in ${pids[*]}; do
|
||||
local p=$(basename $pid)
|
||||
local input=$(dirname $pid)
|
||||
wait $p
|
||||
local code=$?
|
||||
log "procedure with input $input returned $code"
|
||||
if [[ $3 && ! $code == 0 ]]; then # needs to catch code==2, etc
|
||||
log "tail of related logs..."
|
||||
tail ${3}${input}.log
|
||||
log "see more in ${3}${input}.log"
|
||||
cf="false"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $cf == "false" ]]; then
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
68
nginx.sh
Normal file
68
nginx.sh
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
#!/bin/bash
|
||||
|
||||
# gets executed ON REMOTE HOST
|
||||
#function current_dir_is_nginx_repo() {
|
||||
# local tok
|
||||
# tok=$(basename -s .git `git config --get remote.origin.url` 2> /dev/null)
|
||||
# if [[ "$tok" == "nginx" ]]; then
|
||||
# ret="true"
|
||||
# return 0
|
||||
# else
|
||||
# ret="false"
|
||||
# return 0
|
||||
# fi
|
||||
#}
|
||||
|
||||
# The following functions are all run on a VM
|
||||
# Through an SSH connection. Make sure not to
|
||||
# use any external functions in them.
|
||||
|
||||
function build_nginx_remote() {
|
||||
auto/configure \
|
||||
--with-threads \
|
||||
--with-http_ssl_module \
|
||||
--with-http_v2_module \
|
||||
--with-http_v3_module \
|
||||
--with-http_realip_module \
|
||||
--with-http_addition_module \
|
||||
--with-http_sub_module \
|
||||
--with-http_dav_module \
|
||||
--with-http_flv_module \
|
||||
--with-http_gzip_static_module \
|
||||
--with-http_auth_request_module \
|
||||
--with-http_random_index_module \
|
||||
--with-http_secure_link_module \
|
||||
--with-http_slice_module \
|
||||
--with-http_stub_status_module \
|
||||
--with-stream_ssl_module \
|
||||
--with-stream_realip_module \
|
||||
--with-stream_ssl_preread_module \
|
||||
--with-debug && \
|
||||
make -j3
|
||||
return $?
|
||||
}
|
||||
|
||||
function test_nginx_remote() {
|
||||
TEST_NGINX_VERBOSE=1 TEST_NGINX_CATLOG=1 prove -vw -j 3 .
|
||||
return $?
|
||||
}
|
||||
|
||||
function clean_nginx_remote() {
|
||||
make clean
|
||||
return $?
|
||||
}
|
||||
|
||||
function build_otel_remote() {
|
||||
mkdir -p build && cd build && \
|
||||
cmake -DNGX_OTEL_NGINX_BUILD_DIR=../../nginx/objs .. && \
|
||||
make -j3
|
||||
return $?
|
||||
}
|
||||
|
||||
function test_otel_remote() {
|
||||
echo "UNIMPLEMENTED!"
|
||||
}
|
||||
|
||||
function clean_otel_remote() {
|
||||
rm -rf build
|
||||
}
|
||||
222
test.sh
Executable file
222
test.sh
Executable file
|
|
@ -0,0 +1,222 @@
|
|||
#!/bin/bash
|
||||
|
||||
dirn=$(dirname "$0")
|
||||
source $dirn/common.sh
|
||||
source $dirn/virt.sh
|
||||
source $dirn/nginx.sh
|
||||
|
||||
nginx_dir=""
|
||||
otel_dir=""
|
||||
tests_dir=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-h | --help)
|
||||
log "test.sh: build and test code on many libvirt VMs at once"
|
||||
log " -h, --help: show this help text"
|
||||
log " -n <dir>, --nginx <dir>: specify an nginx directory"
|
||||
log " -o <dir>, --otel <dir>: specify an nginx-otel directory"
|
||||
log " -t <dir>, --tests <dir>: specify an nginx-tests directory"
|
||||
exit 0
|
||||
;;
|
||||
|
||||
-n | --nginx)
|
||||
[ -d $2 ] || ( \
|
||||
log "nginx flag requires valid dir" && \
|
||||
exit 1 )
|
||||
nginx_dir=$2
|
||||
;;
|
||||
|
||||
-o | --otel)
|
||||
[ -d $2 ] || ( \
|
||||
log "otel flag requires valid dir" && \
|
||||
exit 1 )
|
||||
otel_dir=$2
|
||||
;;
|
||||
|
||||
-t | --tests)
|
||||
[ -d $2 ] || ( \
|
||||
log "tests flag requires valid dir" && \
|
||||
exit 1 )
|
||||
[ $nginx_dir ] || [ $otel_dir ] || ( \
|
||||
log "must set nginx flag before tests flag" && \
|
||||
exit 1 )
|
||||
tests_dir=$2
|
||||
;;
|
||||
|
||||
*)
|
||||
log "unknown argument: $1"
|
||||
exit 1
|
||||
esac
|
||||
|
||||
shift
|
||||
shift
|
||||
done
|
||||
|
||||
vm_nginx_dir=$(basename $nginx_dir)
|
||||
vm_otel_dir=$(basename $otel_dir)
|
||||
vm_tests_dir=$(basename $tests_dir)
|
||||
|
||||
section "script init..."
|
||||
if [[ ! -d $test_log_dir ]]; then
|
||||
log "prepping new test log dir"
|
||||
ran=$((1+$RANDOM % 1000))
|
||||
test_log_dir=/tmp/nginx_autotest_fmk_$ran
|
||||
rm -rf $test_log_dir
|
||||
mkdir $test_log_dir
|
||||
fi
|
||||
|
||||
log "tests logs dir: $test_log_dir"
|
||||
log "nginx code dir: $nginx_dir"
|
||||
log "nginx test dir: $tests_dir"
|
||||
log "otel code dir: $otel_dir"
|
||||
|
||||
function syncs() {
|
||||
sync_dir_to_vm $1 $nginx_dir
|
||||
sync_dir_to_vm $1 $tests_dir
|
||||
sync_dir_to_vm $1 $otel_dir
|
||||
}
|
||||
|
||||
function build_nginx() {
|
||||
vm_shell $1 \
|
||||
"echo 'BEGIN BUILD'; set -ex; \
|
||||
$(typeset -f build_nginx_remote); \
|
||||
cd $vm_nginx_dir; \
|
||||
build_nginx_remote;"
|
||||
return $?
|
||||
}
|
||||
|
||||
function build_otel() {
|
||||
vm_shell $1 \
|
||||
"echo 'BEGIN BUILD'; set -ex; \
|
||||
$(typeset -f build_otel_remote); \
|
||||
cd $vm_otel_dir; \
|
||||
build_otel_remote;"
|
||||
return $?
|
||||
}
|
||||
|
||||
function test_nginx() {
|
||||
vm_shell $1 \
|
||||
"echo 'BEGIN TESTS'; set -ex; \
|
||||
$(typeset -f test_nginx_remote); \
|
||||
cd $vm_tests_dir; \
|
||||
test_nginx_remote;"
|
||||
return $?
|
||||
}
|
||||
|
||||
function test_otel() {
|
||||
vm_shell $1 \
|
||||
"echo 'BEGIN TESTS'; set -ex; \
|
||||
$(typeset -f test_otel_remote); \
|
||||
cd $vm_otel_dir; \
|
||||
test_otel_remote;"
|
||||
return $?
|
||||
}
|
||||
|
||||
function clean_nginx() {
|
||||
vm_shell $1 \
|
||||
"set -ex; \
|
||||
$(typeset -f clean_nginx_remote); \
|
||||
cd $vm_nginx_dir; \
|
||||
clean_nginx_remote;"
|
||||
return $?
|
||||
}
|
||||
|
||||
function clean_otel() {
|
||||
vm_shell $1 \
|
||||
"set -ex; \
|
||||
$(typeset -f clean_otel_remote); \
|
||||
cd $vm_otel_dir; \
|
||||
clean_otel_remote;"
|
||||
return $?
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
section "cleanup!"
|
||||
log "cleaning build directories"
|
||||
if ! parallel_invoke_and_wait \
|
||||
clean_nginx "$vm_list" "$test_log_dir/clean_nginx"; then
|
||||
error "Failed to clean NGINX build directory"
|
||||
fi
|
||||
|
||||
if ! parallel_invoke_and_wait \
|
||||
clean_otel "$vm_list" "$test_log_dir/clean_otel"; then
|
||||
error "Failed to clean otel build directory"
|
||||
fi
|
||||
|
||||
log "turning off VMs"
|
||||
parallel_invoke_and_wait \
|
||||
turn_off_vm \
|
||||
"$vm_list" \
|
||||
"$test_log_dir/off_"
|
||||
}
|
||||
|
||||
section "launching VMs"
|
||||
vms_avail
|
||||
if [[ "$ret" == "" ]]; then
|
||||
log "no VMs available!"
|
||||
exit 1
|
||||
fi
|
||||
vm_list=$ret
|
||||
ret=""
|
||||
|
||||
if ! parallel_invoke_and_wait \
|
||||
turn_on_vm_and_wait \
|
||||
"$vm_list" \
|
||||
"$test_log_dir/on_"; then
|
||||
error "Failed to turn on all VMs"
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
|
||||
section "syncing code to VMs"
|
||||
if ! parallel_invoke_and_wait \
|
||||
syncs "$vm_list" "$test_log_dir/sync_"; then
|
||||
error "Failed to sync files to VMs"
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $nginx_dir ]; then
|
||||
section "building NGINX"
|
||||
if ! parallel_invoke_and_wait \
|
||||
build_nginx "$vm_list" "$test_log_dir/build_nginx_"; then
|
||||
error "NGINX build failures detected"
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $otel_dir ]; then
|
||||
section "building NGINX Otel module"
|
||||
if ! parallel_invoke_and_wait \
|
||||
build_otel "$vm_list" "$test_log_dir/build_otel_"; then
|
||||
error "Otel build failures detected"
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $tests_dir ]; then
|
||||
if [ $nginx_dir ]; then
|
||||
section "testing NGINX"
|
||||
if ! parallel_invoke_and_wait \
|
||||
test_nginx "$vm_list" "$test_log_dir/test_nginx_"; then
|
||||
error "NGINX test failures detected"
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $otel_dir ]; then
|
||||
section "testing NGINX Otel module"
|
||||
if ! parallel_invoke_and_wait \
|
||||
test_otel "$vm_list" "$test_log_dir/test_otel_"; then
|
||||
error "Otel test failures detected"
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ------
|
||||
cleanup
|
||||
log "Finished :)"
|
||||
209
virt.sh
Normal file
209
virt.sh
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
#!/bin/bash
|
||||
|
||||
dirn=$(dirname "$0")
|
||||
source $dirn/common.sh
|
||||
|
||||
if [[ ! -f $dirn/SECRET.sh ]]; then
|
||||
error "need to create SECRET.sh... see Readme"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source $dirn/SECRET.sh
|
||||
|
||||
# set in SECRET.sh
|
||||
if [[ ! $USERN ]]; then
|
||||
error "\$USERN not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# set in SECRET.sh
|
||||
if [[ ! $PASSP ]]; then
|
||||
error "\$PASSP not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
function vms_avail() {
|
||||
ret=$(sudo virsh list --all --name | sort)
|
||||
}
|
||||
|
||||
function vms_on() {
|
||||
ret=$(sudo virsh list --name | sort)
|
||||
}
|
||||
|
||||
function vms_off() {
|
||||
ret=$(sudo virsh list --inactive --name | sort)
|
||||
}
|
||||
|
||||
function get_vm_ip() {
|
||||
vms_avail
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 doesnt exist"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
vms_on
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 already off"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
ret=$(sudo virsh net-dhcp-leases default | grep $1 | awk '{print $5}' | rev | cut -c 4- | rev)
|
||||
}
|
||||
|
||||
function turn_on_vm_and_wait() {
|
||||
if [[ ! $1 ]]; then
|
||||
log "no VM specified"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
vms_avail
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 doesnt exist"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
vms_off
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 already on"
|
||||
ret=""
|
||||
return 0
|
||||
fi
|
||||
|
||||
sudo virsh start $1 >/dev/null
|
||||
log "Started VM $1. Please standby"
|
||||
|
||||
|
||||
# wait for an IP
|
||||
ret=""
|
||||
while [[ "$ret" == "" ]]; do
|
||||
get_vm_ip $1
|
||||
sleep 0.5
|
||||
done
|
||||
log "Got IP for VM $1"
|
||||
|
||||
# wait for successful ssh
|
||||
# ret set by get_vm_ip above
|
||||
while ! sshpass -p $PASSP \
|
||||
ssh -o PreferredAuthentications=password \
|
||||
-o StrictHostKeyChecking=no $USERN@$ret \
|
||||
exit; do
|
||||
sleep 0.1
|
||||
done
|
||||
log "Got SSH on VM $1"
|
||||
}
|
||||
|
||||
function vm_shell() {
|
||||
vms_avail
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 doesnt exist"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
vms_on
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 is off"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$2" == "" ]]; then
|
||||
log "wont execute empty command"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
# wait for an IP
|
||||
ret=""
|
||||
while [[ "$ret" == "" ]]; do
|
||||
get_vm_ip $1
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
# wait for successful ssh
|
||||
# ret set by get_vm_ip above
|
||||
sshpass -p $PASSP \
|
||||
ssh -o PreferredAuthentications=password \
|
||||
-o StrictHostKeyChecking=no \
|
||||
$USERN@$ret "$2"
|
||||
return $?
|
||||
}
|
||||
|
||||
function turn_off_vm() {
|
||||
if [[ ! $1 ]]; then
|
||||
log "no VM specified"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
vms_avail
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 doesnt exist"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
vms_on
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 already off"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
ret=$(sudo virsh shutdown $1)
|
||||
}
|
||||
|
||||
function sync_dir_to_vm(){
|
||||
if [ ! -d $2 ]; then
|
||||
log "directory $2 does not exist."
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
vms_avail
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 doesnt exist"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
vms_on
|
||||
if ! [[ $ret =~ $1 ]]; then
|
||||
log "VM $1 is off"
|
||||
ret=""
|
||||
return 1
|
||||
fi
|
||||
|
||||
# wait for an IP
|
||||
ret=""
|
||||
while [[ "$ret" == "" ]]; do
|
||||
get_vm_ip $1
|
||||
log "waiting for ip"
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
sshpass -p $PASSP \
|
||||
rsync -avze "ssh -o PreferredAuthentications=password -o StrictHostKeyChecking=no" \
|
||||
$2 $USERN@$ret:
|
||||
return $?
|
||||
}
|
||||
|
||||
# boots vm, enters SSH, shuts down VM
|
||||
function vsh() {
|
||||
# takes care of error cases
|
||||
if ! turn_on_vm_and_wait $1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
get_vm_ip $1
|
||||
sshpass -p $PASSP \
|
||||
ssh -o PreferredAuthentications=password \
|
||||
-o StrictHostKeyChecking=no \
|
||||
$USERN@$ret
|
||||
|
||||
turn_off_vm $1
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue