testing: Collect test results with an on-guest script in parallel
In particular the swanctl calls all take a while and this allows doing them in parallel if multiple hosts are involved. This reduces the runtime of each test by 1-3 seconds.
This commit is contained in:
Tobias Brunner
+70
-102
@@ -41,6 +41,7 @@ TESTRESULTSHTML=$TODAYDIR/all.html
|
|||||||
INDEX=$TODAYDIR/index.html
|
INDEX=$TODAYDIR/index.html
|
||||||
DEFAULTTESTSDIR=$TESTDIR/testing/tests
|
DEFAULTTESTSDIR=$TESTDIR/testing/tests
|
||||||
|
|
||||||
|
GUEST_OUTPUT_DIR=/var/cache/testing-results
|
||||||
SOURCEIP_ROUTING_TABLE=220
|
SOURCEIP_ROUTING_TABLE=220
|
||||||
|
|
||||||
export LEAK_DETECTIVE_LOG=/var/log/leak-detective.log
|
export LEAK_DETECTIVE_LOG=/var/log/leak-detective.log
|
||||||
@@ -474,9 +475,74 @@ do
|
|||||||
|
|
||||||
|
|
||||||
##########################################################################
|
##########################################################################
|
||||||
# log statusall and listall output
|
# collect test results from hosts
|
||||||
# get copies of ipsec.conf, ipsec.secrets
|
#
|
||||||
|
|
||||||
|
COLLECT_OPTIONS_BASE="-t $SOURCEIP_ROUTING_TABLE"
|
||||||
|
|
||||||
|
if [ $name = "net2net-ip4-in-ip6-ikev2" -o $name = "net2net-ip6-in-ip4-ikev2" ]
|
||||||
|
then
|
||||||
|
COLLECT_OPTIONS_BASE="$COLLECT_OPTIONS_BASE -4 -6"
|
||||||
|
IPROUTE_DSP="ip (-6) route list table $SOURCEIP_ROUTING_TABLE"
|
||||||
|
IPTABLES_DSP="iptables -L ; ip6tables -L"
|
||||||
|
IPTABLES_SAVE_DSP="iptables-save ; ip6tables-save"
|
||||||
|
elif [ -n "$IPV6" ]
|
||||||
|
then
|
||||||
|
COLLECT_OPTIONS_BASE="$COLLECT_OPTIONS_BASE -6"
|
||||||
|
IPROUTE_DSP="ip -6 route list table $SOURCEIP_ROUTING_TABLE"
|
||||||
|
IPTABLES_DSP="ip6tables -L"
|
||||||
|
IPTABLES_SAVE_DSP="ip6tables-save"
|
||||||
|
else
|
||||||
|
COLLECT_OPTIONS_BASE="$COLLECT_OPTIONS_BASE -4"
|
||||||
|
IPROUTE_DSP="ip route list table $SOURCEIP_ROUTING_TABLE"
|
||||||
|
IPTABLES_DSP="iptables -L"
|
||||||
|
IPTABLES_SAVE_DSP="iptables-save"
|
||||||
|
fi
|
||||||
|
|
||||||
|
declare -A COLLECT_OPTIONS=()
|
||||||
|
|
||||||
|
for host in $IPSECHOSTS
|
||||||
|
do
|
||||||
|
COLLECT_OPTIONS[${host}]="${COLLECT_OPTIONS[${host}]} -i"
|
||||||
|
if [ -z "$SWANCTL" ]
|
||||||
|
then
|
||||||
|
COLLECT_OPTIONS[${host}]="${COLLECT_OPTIONS[${host}]} -l"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
for host in $DBHOSTS
|
||||||
|
do
|
||||||
|
COLLECT_OPTIONS[${host}]="${COLLECT_OPTIONS[${host}]} -d"
|
||||||
|
done
|
||||||
|
|
||||||
|
for host in $RADIUSHOSTS
|
||||||
|
do
|
||||||
|
COLLECT_OPTIONS[${host}]="${COLLECT_OPTIONS[${host}]} -r"
|
||||||
|
done
|
||||||
|
|
||||||
|
# collect test results in parallel, copy them afterwards
|
||||||
|
WAIT_FOR=()
|
||||||
|
|
||||||
|
for host in ${!COLLECT_OPTIONS[@]}
|
||||||
|
do
|
||||||
|
eval HOSTLOGIN=root@\$ipv4_${host}
|
||||||
|
ssh $SSHCONF $HOSTLOGIN /usr/local/bin/collect-results \
|
||||||
|
$COLLECT_OPTIONS_BASE ${COLLECT_OPTIONS[${host}]} ${GUEST_OUTPUT_DIR} &
|
||||||
|
WAIT_FOR+=($!)
|
||||||
|
done
|
||||||
|
|
||||||
|
test ${#WAIT_FOR[@]} -gt 0 && wait ${WAIT_FOR[@]}
|
||||||
|
|
||||||
|
for host in ${!COLLECT_OPTIONS[@]}
|
||||||
|
do
|
||||||
|
eval HOSTLOGIN=root@\$ipv4_${host}
|
||||||
|
scp $SSHCONF -q $HOSTLOGIN:${GUEST_OUTPUT_DIR}/* $TESTRESULTDIR/
|
||||||
|
done
|
||||||
|
chmod a+r $TESTRESULTDIR/*
|
||||||
|
|
||||||
|
##########################################################################
|
||||||
# create index.html for the given test case
|
# create index.html for the given test case
|
||||||
|
#
|
||||||
|
|
||||||
cat > $TESTRESULTDIR/index.html <<@EOF
|
cat > $TESTRESULTDIR/index.html <<@EOF
|
||||||
<html>
|
<html>
|
||||||
@@ -501,102 +567,16 @@ do
|
|||||||
<img src="../../images/$DIAGRAM" alt="$VIRTHOSTS">
|
<img src="../../images/$DIAGRAM" alt="$VIRTHOSTS">
|
||||||
@EOF
|
@EOF
|
||||||
|
|
||||||
IPTABLES_CMD_V4="echo -e '=== filter table ==='; iptables -v -n -L; echo -e '\n=== nat table ==='; iptables -v -n -t nat -L; echo -e '\n=== mangle table ==='; iptables -v -n -t mangle -L"
|
|
||||||
IPTABLES_CMD_V6="echo -e '=== filter table ==='; ip6tables -v -n -L; echo -e '\n=== nat table ==='; ip6tables -v -n -t nat -L; echo -e '\n=== mangle table ==='; ip6tables -v -n -t mangle -L"
|
|
||||||
|
|
||||||
if [ -n "$IPV6" ]
|
|
||||||
then
|
|
||||||
IPROUTE_CMD="ip -6 route list table $SOURCEIP_ROUTING_TABLE"
|
|
||||||
IPROUTE_DSP=$IPROUTE_CMD
|
|
||||||
IPTABLES_CMD="$IPTABLES_CMD_V6"
|
|
||||||
IPTABLES_DSP="ip6tables -L"
|
|
||||||
IPTABLES_SAVE_CMD="ip6tables-save"
|
|
||||||
IPTABLES_SAVE_DSP="ip6tables-save"
|
|
||||||
else
|
|
||||||
IPROUTE_CMD="ip route list table $SOURCEIP_ROUTING_TABLE"
|
|
||||||
IPROUTE_DSP=$IPROUTE_CMD
|
|
||||||
IPTABLES_CMD="$IPTABLES_CMD_V4"
|
|
||||||
IPTABLES_DSP="iptables -L"
|
|
||||||
IPTABLES_SAVE_CMD="iptables-save"
|
|
||||||
IPTABLES_SAVE_DSP="iptables-save"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ $name = "net2net-ip4-in-ip6-ikev2" -o $name = "net2net-ip6-in-ip4-ikev2" ]
|
|
||||||
then
|
|
||||||
IPROUTE_CMD="ip route list table $SOURCEIP_ROUTING_TABLE; echo; ip -6 route list table $SOURCEIP_ROUTING_TABLE"
|
|
||||||
IPROUTE_DSP="ip (-6) route list table $SOURCEIP_ROUTING_TABLE"
|
|
||||||
IPTABLES_CMD="$IPTABLES_CMD_V4; echo; $IPTABLES_CMD_V6"
|
|
||||||
IPTABLES_DSP="iptables -L ; ip6tables -L"
|
|
||||||
IPTABLES_SAVE_CMD="iptables-save; echo; ip6tables-save"
|
|
||||||
IPTABLES_SAVE_DSP="iptables-save ; ip6tables-save"
|
|
||||||
fi
|
|
||||||
|
|
||||||
for host in $DBHOSTS
|
|
||||||
do
|
|
||||||
eval HOSTLOGIN=root@\$ipv4_${host}
|
|
||||||
|
|
||||||
scp $SSHCONF $HOSTLOGIN:/etc/db.d/ipsec.sql \
|
|
||||||
$TESTRESULTDIR/${host}.ipsec.sql > /dev/null 2>&1
|
|
||||||
done
|
|
||||||
|
|
||||||
for host in $IPSECHOSTS
|
for host in $IPSECHOSTS
|
||||||
do
|
do
|
||||||
eval HOSTLOGIN=root@\$ipv4_${host}
|
if [ -n "$SWANCTL" ]
|
||||||
|
|
||||||
scp $SSHCONF $HOSTLOGIN:/etc/strongswan.conf \
|
|
||||||
$TESTRESULTDIR/${host}.strongswan.conf > /dev/null 2>&1
|
|
||||||
if [ -n "$SWANCTL" ]
|
|
||||||
then
|
then
|
||||||
scp $SSHCONF $HOSTLOGIN:/etc/swanctl/swanctl.conf \
|
echo >> $TESTRESULTDIR/${host}.swanctl.sas
|
||||||
$TESTRESULTDIR/${host}.swanctl.conf > /dev/null 2>&1
|
|
||||||
|
|
||||||
for subsys in conns algs certs pools authorities sas pols
|
|
||||||
do
|
|
||||||
ssh $SSHCONF $HOSTLOGIN swanctl --list-$subsys \
|
|
||||||
> $TESTRESULTDIR/${host}.swanctl.$subsys 2>/dev/null
|
|
||||||
done
|
|
||||||
|
|
||||||
ssh $SSHCONF $HOSTLOGIN swanctl --stats \
|
|
||||||
> $TESTRESULTDIR/${host}.swanctl.stats 2>/dev/null
|
|
||||||
|
|
||||||
echo "" >> $TESTRESULTDIR/${host}.swanctl.sas
|
|
||||||
cat $TESTRESULTDIR/${host}.swanctl.pols >> \
|
cat $TESTRESULTDIR/${host}.swanctl.pols >> \
|
||||||
$TESTRESULTDIR/${host}.swanctl.sas
|
$TESTRESULTDIR/${host}.swanctl.sas
|
||||||
cat $TESTRESULTDIR/${host}.swanctl.algs >> \
|
cat $TESTRESULTDIR/${host}.swanctl.algs >> \
|
||||||
$TESTRESULTDIR/${host}.swanctl.stats
|
$TESTRESULTDIR/${host}.swanctl.stats
|
||||||
else
|
|
||||||
for file in ipsec.conf ipsec.secrets
|
|
||||||
do
|
|
||||||
scp $SSHCONF $HOSTLOGIN:/etc/$file \
|
|
||||||
$TESTRESULTDIR/${host}.$file > /dev/null 2>&1
|
|
||||||
done
|
|
||||||
|
|
||||||
for command in statusall listall
|
|
||||||
do
|
|
||||||
ssh $SSHCONF $HOSTLOGIN ipsec $command \
|
|
||||||
> $TESTRESULTDIR/${host}.$command 2>/dev/null
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
if (! [ -f $TESTRESULTDIR/${host}.ipsec.sql ] ) then
|
|
||||||
scp $SSHCONF $HOSTLOGIN:/etc/ipsec.d/ipsec.sql \
|
|
||||||
$TESTRESULTDIR/${host}.ipsec.sql > /dev/null 2>&1
|
|
||||||
fi
|
|
||||||
|
|
||||||
ssh $SSHCONF $HOSTLOGIN ip -s xfrm policy \
|
|
||||||
> $TESTRESULTDIR/${host}.ip.policy 2>/dev/null
|
|
||||||
ssh $SSHCONF $HOSTLOGIN ip -s xfrm state \
|
|
||||||
> $TESTRESULTDIR/${host}.ip.state 2>/dev/null
|
|
||||||
ssh $SSHCONF $HOSTLOGIN $IPROUTE_CMD \
|
|
||||||
> $TESTRESULTDIR/${host}.ip.route 2>/dev/null
|
|
||||||
ssh $SSHCONF $HOSTLOGIN $IPTABLES_CMD \
|
|
||||||
> $TESTRESULTDIR/${host}.iptables 2>/dev/null
|
|
||||||
ssh $SSHCONF $HOSTLOGIN $IPTABLES_SAVE_CMD \
|
|
||||||
> $TESTRESULTDIR/${host}.iptables-save 2>/dev/null
|
|
||||||
chmod a+r $TESTRESULTDIR/*
|
|
||||||
|
|
||||||
if [ -n "$SWANCTL" ]
|
|
||||||
then
|
|
||||||
cat >> $TESTRESULTDIR/index.html <<@EOF
|
cat >> $TESTRESULTDIR/index.html <<@EOF
|
||||||
<h3>$host</h3>
|
<h3>$host</h3>
|
||||||
<table border="0" cellspacing="0" width="600">
|
<table border="0" cellspacing="0" width="600">
|
||||||
@@ -672,18 +652,6 @@ do
|
|||||||
|
|
||||||
for host in $RADIUSHOSTS
|
for host in $RADIUSHOSTS
|
||||||
do
|
do
|
||||||
eval HOSTLOGIN=root@\$ipv4_${host}
|
|
||||||
|
|
||||||
for file in clients.conf radiusd.conf proxy.conf users sites-enabled/default sites-enabled/inner-tunnel mods-enabled/eap
|
|
||||||
do
|
|
||||||
scp $SSHCONF $HOSTLOGIN:/etc/freeradius/3.0/$file \
|
|
||||||
$TESTRESULTDIR/${host}.$(basename $file) > /dev/null 2>&1
|
|
||||||
done
|
|
||||||
|
|
||||||
scp $SSHCONF $HOSTLOGIN:/var/log/freeradius/radius.log \
|
|
||||||
$TESTRESULTDIR/${host}.radius.log > /dev/null 2>&1
|
|
||||||
|
|
||||||
chmod a+r $TESTRESULTDIR/*
|
|
||||||
cat >> $TESTRESULTDIR/index.html <<@EOF
|
cat >> $TESTRESULTDIR/index.html <<@EOF
|
||||||
<h3>$host</h3>
|
<h3>$host</h3>
|
||||||
<table border="0" cellspacing="0" width="600">
|
<table border="0" cellspacing="0" width="600">
|
||||||
|
|||||||
+122
@@ -0,0 +1,122 @@
|
|||||||
|
#! /bin/bash
|
||||||
|
|
||||||
|
while getopts "46idt:lr" opt
|
||||||
|
do
|
||||||
|
case "$opt" in
|
||||||
|
4)
|
||||||
|
IPV4=YES
|
||||||
|
;;
|
||||||
|
6)
|
||||||
|
IPV6=YES
|
||||||
|
;;
|
||||||
|
i)
|
||||||
|
IPSEC=YES
|
||||||
|
;;
|
||||||
|
d)
|
||||||
|
DB=YES
|
||||||
|
;;
|
||||||
|
t)
|
||||||
|
ROUTING_TABLE=${OPTARG}
|
||||||
|
;;
|
||||||
|
l)
|
||||||
|
LEGACY=YES
|
||||||
|
;;
|
||||||
|
r)
|
||||||
|
RADIUS=YES
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
shift $((OPTIND-1))
|
||||||
|
|
||||||
|
NAME=$(hostname)
|
||||||
|
OUTPUT_DIR=$1
|
||||||
|
|
||||||
|
if [ -z "$OUTPUT_DIR" ]
|
||||||
|
then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# create/clear output dir
|
||||||
|
mkdir -p $OUTPUT_DIR
|
||||||
|
rm -f $OUTPUT_DIR/*
|
||||||
|
|
||||||
|
# collect networking output
|
||||||
|
if [ -n "$IPV4" ]
|
||||||
|
then
|
||||||
|
{ ip route list table $ROUTING_TABLE; echo; } >> ${OUTPUT_DIR}/${NAME}.ip.route
|
||||||
|
{ iptables-save; echo; } >> ${OUTPUT_DIR}/${NAME}.iptables-save
|
||||||
|
{
|
||||||
|
echo -e '=== filter table ==='
|
||||||
|
iptables -v -n -L
|
||||||
|
echo -e '\n=== nat table ==='
|
||||||
|
iptables -v -n -t nat -L
|
||||||
|
echo -e '\n=== mangle table ==='
|
||||||
|
iptables -v -n -t mangle -L
|
||||||
|
echo
|
||||||
|
} >> ${OUTPUT_DIR}/${NAME}.iptables
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "$IPV6" ]
|
||||||
|
then
|
||||||
|
ip -6 route list table $ROUTING_TABLE >> ${OUTPUT_DIR}/${NAME}.ip.route
|
||||||
|
ip6tables-save >> ${OUTPUT_DIR}/${NAME}.iptables-save
|
||||||
|
{
|
||||||
|
echo -e '=== filter table ==='
|
||||||
|
ip6tables -v -n -L
|
||||||
|
echo -e '\n=== nat table ==='
|
||||||
|
ip6tables -v -n -t nat -L
|
||||||
|
echo -e '\n=== mangle table ==='
|
||||||
|
ip6tables -v -n -t mangle -L
|
||||||
|
} >> ${OUTPUT_DIR}/${NAME}.iptables
|
||||||
|
fi
|
||||||
|
|
||||||
|
# collect DB scheme/data
|
||||||
|
if [ -n "$DB" ]
|
||||||
|
then
|
||||||
|
cp /etc/db.d/ipsec.sql ${OUTPUT_DIR}/${NAME}.ipsec.sql >/dev/null 2>&1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# collect XFRM and strongSwan output
|
||||||
|
if [ -n "$IPSEC" ]
|
||||||
|
then
|
||||||
|
ip -s xfrm policy > ${OUTPUT_DIR}/${NAME}.ip.policy
|
||||||
|
ip -s xfrm state > ${OUTPUT_DIR}/${NAME}.ip.state
|
||||||
|
|
||||||
|
cp /etc/strongswan.conf ${OUTPUT_DIR}/${NAME}.strongswan.conf
|
||||||
|
if [ -n "$LEGACY" ]
|
||||||
|
then
|
||||||
|
for file in ipsec.conf ipsec.secrets
|
||||||
|
do
|
||||||
|
cp /etc/$file ${OUTPUT_DIR}/${NAME}.$file
|
||||||
|
done
|
||||||
|
|
||||||
|
for command in statusall listall
|
||||||
|
do
|
||||||
|
ipsec $command > ${OUTPUT_DIR}/${NAME}.$command
|
||||||
|
done
|
||||||
|
else
|
||||||
|
cp /etc/swanctl/swanctl.conf ${OUTPUT_DIR}/${NAME}.swanctl.conf
|
||||||
|
|
||||||
|
swanctl --stats > ${OUTPUT_DIR}/${NAME}.swanctl.stats 2>&1
|
||||||
|
|
||||||
|
for subsys in conns algs certs pools authorities sas pols
|
||||||
|
do
|
||||||
|
swanctl --list-$subsys > ${OUTPUT_DIR}/${NAME}.swanctl.$subsys 2>&1
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
if [ ! -f ${OUTPUT_DIR}/${NAME}.ipsec.sql ]
|
||||||
|
then
|
||||||
|
cp /etc/ipsec.d/ipsec.sql ${OUTPUT_DIR}/${NAME}.ipsec.sql >/dev/null 2>&1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# collect RADIUS output
|
||||||
|
if [ -n "$RADIUS" ]
|
||||||
|
then
|
||||||
|
for file in clients.conf radiusd.conf proxy.conf users sites-enabled/default sites-enabled/inner-tunnel mods-enabled/eap
|
||||||
|
do
|
||||||
|
cp /etc/freeradius/3.0/$file ${OUTPUT_DIR}/${NAME}.$(basename $file) >/dev/null 2>&1
|
||||||
|
done
|
||||||
|
|
||||||
|
cp /var/log/freeradius/radius.log ${OUTPUT_DIR}/${NAME}.radius.log
|
||||||
|
fi
|
||||||
Reference in New Issue
Block a user