#! /bin/sh
# $OpenLDAP$
## This work is part of OpenLDAP Software .
##
## Copyright 1998-2019 The OpenLDAP Foundation.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## .
echo "running defines.sh"
. $SRCDIR/scripts/defines.sh
if test $SYNCPROV = syncprovno; then
echo "Syncrepl provider overlay not available, test skipped"
exit 0
fi
MMR=${MMR-4}
if [ $MMR -gt 9 ]; then
MMR=9
fi
XDIR=$TESTDIR/srv
TMP=$TESTDIR/tmp
mkdir -p $TESTDIR
$SLAPPASSWD -g -n >$CONFIGPWF
if test x"$SYNCMODE" = x ; then
SYNCMODE=rp
fi
case "$SYNCMODE" in
ro)
SYNCTYPE="type=refreshOnly interval=00:00:00:03"
;;
rp)
SYNCTYPE="type=refreshAndPersist"
;;
*)
echo "unknown sync mode $SYNCMODE"
exit 1;
;;
esac
#
# Test replication of dynamic config:
# - start servers
# - configure over ldap
# - populate over ldap
# - configure syncrepl over ldap
# - retrieve database over ldap and compare against expected results
#
echo "Initializing server configurations..."
n=1
while [ $n -le $MMR ]; do
DBDIR=${XDIR}$n/db
CFDIR=${XDIR}$n/slapd.d
mkdir -p ${XDIR}$n $DBDIR $CFDIR
$SLAPADD -F $CFDIR -n 0 < $LOG1 2>&1 &
PID=$!
if test $WAIT != 0 ; then
echo PID $PID
read foo
fi
KILLPIDS="$PID"
cd $TESTWD
sleep 1
echo "Using ldapsearch to check that server 1 is running..."
for i in 0 1 2 3 4 5; do
$LDAPSEARCH -s base -b "" -H $URI1 \
'objectclass=*' > /dev/null 2>&1
RC=$?
if test $RC = 0 ; then
break
fi
echo "Waiting 5 seconds for slapd to start..."
sleep 5
done
if test $RC != 0 ; then
echo "ldapsearch failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Inserting syncprov overlay on server 1..."
echo "" > $TMP
if [ "$SYNCPROV" = syncprovmod ]; then
cat <> $TMP
dn: cn=module,cn=config
changetype: add
objectClass: olcModuleList
cn: module
olcModulePath: $TESTWD/../servers/slapd/overlays
olcModuleLoad: syncprov.la
EOF
fi
#
# Note that we configure a timeout here; it's possible for both
# servers to attempt to bind to each other while a modify to
# cn=config is in progress. When the modify pauses the thread pool
# neither server will progress. The timeout will drop the syncrepl
# attempt and allow the modifies to complete.
#
read CONFIGPW < $CONFIGPWF
echo "dn: cn=config" >> $TMP
echo "changetype: modify" >> $TMP
echo "replace: olcServerID" >> $TMP
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "olcServerID: $n $URI" >> $TMP
n=`expr $n + 1`
done
cat <> $TMP
dn: olcOverlay=syncprov,olcDatabase={0}config,cn=config
changetype: add
objectClass: olcOverlayConfig
objectClass: olcSyncProvConfig
olcOverlay: syncprov
dn: olcDatabase={0}config,cn=config
changetype: modify
add: olcSyncRepl
EOF
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "olcSyncRepl: rid=00$n provider=$URI binddn=\"cn=config\" bindmethod=simple" >> $TMP
echo " credentials=$CONFIGPW searchbase=\"cn=config\" type=refreshAndPersist" >> $TMP
echo " retry=\"3 10 300 5\" timeout=3" >> $TMP
n=`expr $n + 1`
done
echo "-" >> $TMP
echo "add: olcMirrorMode" >> $TMP
echo "olcMirrorMode: TRUE" >> $TMP
$LDAPMODIFY -D cn=config -H $URI1 -y $CONFIGPWF < $TMP >> $TESTOUT 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapmodify failed for syncrepl config ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
n=2
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
LOG=$TESTDIR/slapd.$n.log
echo "Starting server $n on TCP/IP port $PORT..."
cd ${XDIR}$n
$SLAPD -F ./slapd.d -h $URI -d $LVL $TIMING > $LOG 2>&1 &
SLAVEPID=$!
if test $WAIT != 0 ; then
echo SLAVEPID $SLAVEPID
read foo
fi
KILLPIDS="$KILLPIDS $SLAVEPID"
cd $TESTWD
sleep 1
echo "Using ldapsearch to check that server $n is running..."
for i in 0 1 2 3 4 5; do
$LDAPSEARCH -s base -b "" -H $URI \
'objectclass=*' > /dev/null 2>&1
RC=$?
if test $RC = 0 ; then
break
fi
echo "Waiting 5 seconds for slapd to start..."
sleep 5
done
if test $RC != 0 ; then
echo "ldapsearch failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Configuring syncrepl on server $n..."
cat < $TMP
dn: olcDatabase={0}config,cn=config
changetype: modify
add: olcSyncRepl
EOF
j=1
while [ $j -le $MMR ]; do
P2=`expr $BASEPORT + $j`
U2="ldap://${LOCALHOST}:$P2/"
echo "olcSyncRepl: rid=00$j provider=$U2 binddn=\"cn=config\" bindmethod=simple" >> $TMP
echo " credentials=$CONFIGPW searchbase=\"cn=config\" type=refreshAndPersist" >> $TMP
echo " retry=\"3 10 300 5\" timeout=3" >> $TMP
j=`expr $j + 1`
done
cat <> $TMP
-
add: olcMirrorMode
olcMirrorMode: TRUE
EOF
$LDAPMODIFY -D cn=config -H $URI -y $CONFIGPWF < $TMP >>$TESTOUT 2>&1
n=`expr $n + 1`
done
echo "Adding schema and databases on server 1..."
$LDAPADD -D cn=config -H $URI1 -y $CONFIGPWF <>$TESTOUT 2>&1
include: file://$ABS_SCHEMADIR/core.ldif
include: file://$ABS_SCHEMADIR/cosine.ldif
include: file://$ABS_SCHEMADIR/inetorgperson.ldif
include: file://$ABS_SCHEMADIR/openldap.ldif
include: file://$ABS_SCHEMADIR/nis.ldif
EOF
RC=$?
if test $RC != 0 ; then
echo "ldapadd failed for schema config ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
nullExclude=""
test $BACKEND = null && nullExclude="# "
echo "" > $TMP
if [ "$BACKENDTYPE" = mod ]; then
cat <> $TMP
dn: cn=module,cn=config
objectClass: olcModuleList
cn: module
olcModulePath: $TESTWD/../servers/slapd/back-$BACKEND
olcModuleLoad: back_$BACKEND.la
EOF
fi
cat <> $TMP
dn: olcDatabase={1}$BACKEND,cn=config
objectClass: olcDatabaseConfig
${nullExclude}objectClass: olc${BACKEND}Config
olcDatabase: {1}$BACKEND
olcSuffix: $BASEDN
${nullExclude}olcDbDirectory: ./db
olcRootDN: $MANAGERDN
olcRootPW: $PASSWD
EOF
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "olcSyncRepl: rid=01$n provider=$URI binddn=\"$MANAGERDN\" bindmethod=simple" >> $TMP
echo " credentials=$PASSWD searchbase=\"$BASEDN\" $SYNCTYPE" >> $TMP
echo " retry=\"3 10 300 5\" timeout=3" >> $TMP
n=`expr $n + 1`
done
cat <> $TMP
olcMirrorMode: TRUE
dn: olcOverlay=syncprov,olcDatabase={1}${BACKEND},cn=config
objectClass: olcOverlayConfig
objectClass: olcSyncProvConfig
olcOverlay: syncprov
EOF
$LDAPADD -D cn=config -H $URI1 -y $CONFIGPWF < $TMP >>$TESTOUT 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapadd failed for database config ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
if test $INDEXDB = indexdb ; then
$LDAPMODIFY -D cn=config -H $URI1 -y $CONFIGPWF <>$TESTOUT 2>&1
dn: olcDatabase={1}$BACKEND,cn=config
changetype: modify
add: olcDbIndex
olcDbIndex: objectClass,entryUUID,entryCSN eq
olcDbIndex: cn,uid pres,eq,sub
EOF
RC=$?
if test $RC != 0 ; then
echo "ldapadd modify for database config ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
fi
echo "Using ldapadd to populate server 1..."
$LDAPADD -D "$MANAGERDN" -H $URI1 -w $PASSWD -f $LDIFORDERED \
>> $TESTOUT 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapadd failed for server 1 database ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Waiting $SLEEP2 seconds for syncrepl to receive changes..."
sleep $SLEEP2
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "Using ldapsearch to read config from server $n..."
$LDAPSEARCH -b cn=config -D cn=config -H $URI -y $CONFIGPWF \
'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at server $n ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done
n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved configs from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT
if test $? != 0 ; then
echo "test failed - server 1 and server $n configs differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
n=`expr $n + 1`
done
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD \
'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at server $n ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done
n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT
if test $? != 0 ; then
echo "test failed - server 1 and server $n databases differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
n=`expr $n + 1`
done
echo "Using ldapadd to populate server 2..."
$LDAPADD -D "$MANAGERDN" -H $URI2 -w $PASSWD -f $LDIFADD1 \
>> $TESTOUT 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapadd failed for server 2 database ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD \
'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at server $n ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done
n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT
if test $? != 0 ; then
echo "test failed - server 1 and server $n databases differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
n=`expr $n + 1`
done
echo "Using ldapadd to populate server 3..."
$LDAPADD -D "$MANAGERDN" -H $URI3 -w $PASSWD \
<< EOMODS >> $TESTOUT 2>&1
dn: cn=Server 3 Test,dc=example,dc=com
changetype: add
objectClass: device
cn: Server 3 Test
EOMODS
RC=$?
if test $RC != 0 ; then
echo "ldapadd failed for server 3 database ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD \
'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at server $n ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done
n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT
if test $? != 0 ; then
echo "test failed - server 1 and server $n databases differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
n=`expr $n + 1`
done
echo "Using ldapmodify to add to server 1 entries that will be deleted..."
$LDAPMODIFY -D "$MANAGERDN" -H $URI1 -w $PASSWD \
>> $TESTOUT 2>&1 << EOADDS
dn: cn=To be deleted by server 1,dc=example,dc=com
changetype: add
objectClass: device
# no distinguished values, will be added by DSA
dn: cn=To be deleted by server 2,dc=example,dc=com
changetype: add
objectClass: device
# no distinguished values, will be added by DSA
dn: cn=To be deleted by server 3,dc=example,dc=com
changetype: add
objectClass: device
# no distinguished values, will be added by DSA
dn: cn=To be deleted by server 1,dc=example,dc=com
changetype: delete
EOADDS
RC=$?
if test $RC != 0 ; then
echo "ldapmodify failed for server 1 database ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD \
'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at server $n ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done
n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT
if test $? != 0 ; then
echo "test failed - server 1 and server $n databases differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
n=`expr $n + 1`
done
echo "Using ldapmodify to delete entries from server 2..."
$LDAPMODIFY -D "$MANAGERDN" -H $URI2 -w $PASSWD \
>> $TESTOUT 2>&1 << EOADDS
dn: cn=To be deleted by server 2,dc=example,dc=com
changetype: delete
EOADDS
RC=$?
if test $RC != 0 ; then
echo "ldapmodify failed for server 2 database ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1
echo "Using ldapmodify to delete entries from server 3..."
$LDAPMODIFY -D "$MANAGERDN" -H $URI3 -w $PASSWD \
>> $TESTOUT 2>&1 << EOADDS
dn: cn=To be deleted by server 3,dc=example,dc=com
changetype: delete
EOADDS
RC=$?
if test $RC != 0 ; then
echo "ldapmodify failed for server 3 database ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
sleep $SLEEP1
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD \
'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at server $n ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done
n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT
if test $? != 0 ; then
echo "test failed - server 1 and server $n databases differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
n=`expr $n + 1`
done
# kill!
# test $KILLSERVERS != no && kill -HUP $KILLPIDS
kill -HUP $KILLPIDS
# kill!
# test $KILLSERVERS != no && wait
wait
echo "Restarting servers..."
KILLPIDS=""
echo "Starting server 1 on TCP/IP port $PORT1..."
echo "======================= RESTART =======================" >> $LOG1
cd ${XDIR}1
$SLAPD -F slapd.d -h $URI1 -d $LVL $TIMING >> $LOG1 2>&1 &
PID=$!
if test $WAIT != 0 ; then
echo PID $PID
read foo
fi
KILLPIDS="$PID"
cd $TESTWD
sleep 1
echo "Using ldapsearch to check that server 1 is running..."
for i in 0 1 2 3 4 5; do
$LDAPSEARCH -s base -b "" -H $URI1 \
'objectclass=*' > /dev/null 2>&1
RC=$?
if test $RC = 0 ; then
break
fi
echo "Waiting 5 seconds for slapd to start..."
sleep 5
done
if test $RC != 0 ; then
echo "ldapsearch failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
n=2
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
LOG=$TESTDIR/slapd.$n.log
echo "Starting server $n on TCP/IP port $PORT..."
cd ${XDIR}$n
echo "======================= RESTART =======================" >> $LOG
$SLAPD -F ./slapd.d -h $URI -d $LVL $TIMING >> $LOG 2>&1 &
PID=$!
if test $WAIT != 0 ; then
echo PID $PID
read foo
fi
KILLPIDS="$KILLPIDS $PID"
cd $TESTWD
n=`expr $n + 1`
done
n=2
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "Using ldapsearch to check that server $n is running..."
for i in 0 1 2 3 4 5; do
$LDAPSEARCH -s base -b "" -H $URI \
'objectclass=*' > /dev/null 2>&1
RC=$?
if test $RC = 0 ; then
break
fi
echo "Waiting 5 seconds for slapd to start..."
sleep 5
done
if test $RC != 0 ; then
echo "ldapsearch failed ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
n=`expr $n + 1`
done
# Insert modifications and more tests here.
echo "Waiting $SLEEP1 seconds for servers to resync..."
sleep $SLEEP1
echo "Using ldapmodify to add/modify/delete entries from server 1..."
for i in 1 2 3 4 5 6 7 8 9 10; do
echo " iteration $i"
$LDAPMODIFY -D "$MANAGERDN" -H $URI1 -w $PASSWD \
>> $TESTOUT 2>&1 << EOMODS
dn: cn=Add-Mod-Del,dc=example,dc=com
changetype: add
cn: Add-Mod-Del
objectclass: organizationalRole
dn: cn=Add-Mod-Del,dc=example,dc=com
changetype: modify
replace: description
description: guinea pig
-
dn: cn=Add-Mod-Del,dc=example,dc=com
changetype: delete
EOMODS
RC=$?
if test $RC != 0 ; then
echo "ldapmodify failed for server 1 database ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
done
echo "Waiting $SLEEP1 seconds for servers to resync..."
sleep $SLEEP1
n=1
while [ $n -le $MMR ]; do
PORT=`expr $BASEPORT + $n`
URI="ldap://${LOCALHOST}:$PORT/"
echo "Using ldapsearch to read all the entries from server $n..."
$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD \
'objectclass=*' > $TESTDIR/server$n.out 2>&1
RC=$?
if test $RC != 0 ; then
echo "ldapsearch failed at server $n ($RC)!"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit $RC
fi
$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt
n=`expr $n + 1`
done
n=2
while [ $n -le $MMR ]; do
echo "Comparing retrieved entries from server 1 and server $n..."
$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT
if test $? != 0 ; then
echo "test failed - server 1 and server $n databases differ"
test $KILLSERVERS != no && kill -HUP $KILLPIDS
exit 1
fi
n=`expr $n + 1`
done
test $KILLSERVERS != no && kill -HUP $KILLPIDS
echo ">>>>> Test succeeded"
test $KILLSERVERS != no && wait
exit 0