#!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # Script to automate testing of an svnsync master/slave # configuration. Commits to the slave should write through # to the master, and the master's post-commit hook svnsync's # to the slave. The test should be able to throw all kinds # of svn operations at one or the other, and master/slave # verified as identical in the end. # # Master / slave setup is achieved in a single httpd process # using virtual hosts bound to different addresses on the # loopback network (127.0.0.1, 127.0.0.2) for slave and # master, respectively. # # The set of changes sent through the system is currently # just the test case for issue 2939, using svnmucc # https://issues.apache.org/jira/browse/SVN-2939 # But of course, any svn traffic liable to break over # mirroring would be a good addition. # # Most of the httpd setup was lifted from davautocheck.sh. # The common boilerplate snippets to setup/start/stop httpd # between the two could be factored out and shared. # SCRIPTDIR=$(dirname $0) SCRIPT=$(basename $0) trap stop_httpd_and_die SIGHUP SIGTERM SIGINT # Ensure the server uses a known locale. LC_ALL=C export LC_ALL function stop_httpd_and_die() { [ -e "$HTTPD_PID" ] && kill $(cat "$HTTPD_PID") exit 1 } function say() { echo "$SCRIPT: $*" } function fail() { say "FAIL: " $* stop_httpd_and_die } function get_loadmodule_config() { local SO="$($APXS -q LIBEXECDIR)/$1.so" # shared object module? if [ -r "$SO" ]; then local NM=$(echo "$1" | sed 's|mod_\(.*\)|\1_module|') echo "LoadModule $NM \"$SO\"" && return fi # maybe it's built-in? "$HTTPD" -l | grep -q "$1\\.c" && return return 1 } # Check apxs's SBINDIR and BINDIR for given program names function get_prog_name() { for prog in $* do for dir in $($APXS -q SBINDIR) $($APXS -q BINDIR) do if [ -e "$dir/$prog" ]; then echo "$dir/$prog" && return fi done done return 1 } # splat out httpd config function setup_config() { say "setting up config: " $1 cat > "$1" <<__EOF__ $LOAD_MOD_MPM $LOAD_MOD_LOG_CONFIG $LOAD_MOD_MIME $LOAD_MOD_UNIXD $LOAD_MOD_DAV LoadModule dav_svn_module "$MOD_DAV_SVN" $LOAD_MOD_AUTH $LOAD_MOD_AUTHN_CORE $LOAD_MOD_AUTHN_FILE $LOAD_MOD_PROXY $LOAD_MOD_PROXY_HTTP $LOAD_MOD_AUTHZ_CORE $LOAD_MOD_AUTHZ_USER $LOAD_MOD_AUTHZ_HOST __EOF__ if "$HTTPD" -v | grep '/2\.[012]' >/dev/null; then cat >> "$1" <<__EOF__ LockFile lock User $(id -un) Group $(id -gn) __EOF__ else HTTPD_LOCK="$HTTPD_ROOT/lock" mkdir "$HTTPD_LOCK" \ || fail "couldn't create lock directory '$HTTPD_LOCK'" cat >> "$1" <<__EOF__ # worker and prefork MUST have a mpm-accept lockfile in 2.3.0+ Mutex "file:$HTTPD_LOCK" mpm-accept Mutex "file:$HTTPD_LOCK" mpm-accept __EOF__ fi cat >> "$1" <<__EOF__ Listen ${TEST_PORT} ServerName localhost PidFile "${HTTPD_ROOT}/pid" LogFormat "%h %l %u %t \"%r\" %>s %b" common CustomLog "${HTTPD_ROOT}/access_log" common ErrorLog "${HTTPD_ROOT}/error_log" LogLevel Debug ServerRoot "${HTTPD_ROOT}" DocumentRoot "${HTTPD_ROOT}" CoreDumpDirectory "${HTTPD_ROOT}" TypesConfig "${HTTPD_ROOT}/mime.types" StartServers 4 MaxRequestsPerChild 0 ThreadsPerChild 8 ThreadsPerChild 8 MaxClients 16 HostNameLookups Off LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" format CustomLog "${HTTPD_ROOT}/req" format CustomLog "${HTTPD_ROOT}/ops" "%t %u %{SVN-REPOS-NAME}e %{SVN-ACTION}e" env=SVN-ACTION AllowOverride none AllowOverride none #Require all granted # slave ServerName ${SLAVE_HOST} CustomLog "${HTTPD_ROOT}/slave_access_log" common ErrorLog "${HTTPD_ROOT}/slave_error_log" # slave 'normal' location DAV svn SVNPath "${SLAVE_REPOS}" SVNMasterURI "${MASTER_URL}" AuthType Basic AuthName "Subversion Repository" AuthUserFile ${HTTPD_ROOT}/users Require valid-user # slave 'sync' location DAV svn SVNPath "${SLAVE_REPOS}" AuthType Basic AuthName "Slave Sync Repository" AuthUserFile ${HTTPD_ROOT}/users Require valid-user # master ServerName ${MASTER_HOST}> CustomLog "${HTTPD_ROOT}/master_access_log" common ErrorLog "${HTTPD_ROOT}/master_error_log" DAV svn SVNPath "${MASTER_REPOS}" AuthType Basic AuthName "Subversion Repository" AuthUserFile ${HTTPD_ROOT}/users Require valid-user __EOF__ } function usage() { echo "usage: $SCRIPT " 1>&2 echo " e.g. \"$SCRIPT /tmp/test-work\"" 1>&2 echo echo " " '' must not exist, \ I will not clobber it for you 1>&2 exit 1 } ### Start execution here ### SCRIPT=$(basename $0) NO_TESTS= if [ "x$1" = 'x--no-tests' ]; then NO_TESTS=1 shift fi if [ $# -ne 1 ] ; then usage fi # httpd ServerRoot, all test and runtime artifacts below here # verify that this doesn't already exist - don't clobber HTTPD_ROOT=$1 if [ -e "$HTTPD_ROOT" ] ; then say "ERROR: test work directory $HTTPD_ROOT already exists, please remove" 1>&2 usage fi #set -e # Don't assume sbin is in the PATH. PATH="$PATH:/usr/sbin:/usr/local/sbin" # Pick up value from environment or PATH (also try apxs2 - for Debian) [ ${APXS:+set} ] \ || APXS=$(which apxs) \ || APXS=$(which apxs2) \ || fail "neither apxs or apxs2 found - required to run $SCRIPT" [ -x $APXS ] || fail "Can't execute apxs executable $APXS" say APXS: $APXS if [ -x subversion/svn/svn ]; then ABS_BUILDDIR=$(pwd) elif [ -x $SCRIPTDIR/../../svn/svn ]; then pushd $SCRIPTDIR/../../../ >/dev/null ABS_BUILDDIR=$(pwd) popd >/dev/null else fail "Run this script from the root of Subversion's build tree!" fi # find all our needed executables, in WC or via apxs httpd="$($APXS -q PROGNAME)" HTTPD=$(get_prog_name $httpd) || fail "HTTPD not found" HTPASSWD=$(get_prog_name htpasswd htpasswd2) \ || fail "Could not find htpasswd or htpasswd2" SVN=$ABS_BUILDDIR/subversion/svn/svn SVNADMIN=$ABS_BUILDDIR/subversion/svnadmin/svnadmin SVNSYNC=$ABS_BUILDDIR/subversion/svnsync/svnsync SVNMUCC=$ABS_BUILDDIR/subversion/svnmucc/svnmucc SVNLOOK=$ABS_BUILDDIR/subversion/svnlook/svnlook [ -x $HTTPD ] || fail "HTTPD '$HTTPD' not executable" [ -x $HTPASSWD ] \ || fail "HTPASSWD '$HTPASSWD' not executable" [ -x $SVN ] || fail "SVN $SVN not built" [ -x $SVNADMIN ] || fail "SVNADMIN $SVNADMIN not built" [ -x $SVNSYNC ] || fail "SVNSYNC $SVNSYNC not built" [ -x $SVNLOOK ] || fail "SVNLOOK $SVNLOOK not built" [ -x $SVNMUCC ] || fail "SVNMUCC $SVNMUCC not built" say HTTPD: $HTTPD say SVN: $SVN say SVNADMIN: $SVNADMIN say SVNSYNC: $SVNSYNC say SVNLOOK: $SVNLOOK say SVNMUCC: $SVNMUCC LOAD_MOD_DAV=$(get_loadmodule_config mod_dav) \ || fail "DAV module not found" LOAD_MOD_LOG_CONFIG=$(get_loadmodule_config mod_log_config) \ || fail "log_config module not found" # proxy needed for svnsync mirroring LOAD_MOD_PROXY=$(get_loadmodule_config mod_proxy) \ || fail "proxy module not found" LOAD_MOD_PROXY_HTTP=$(get_loadmodule_config mod_proxy_http) \ || fail "proxy_http module not found" # needed for TypesConfig LOAD_MOD_MIME=$(get_loadmodule_config mod_mime) \ || fail "MIME module not found" # needed for Auth*, Require, etc. directives LOAD_MOD_AUTH=$(get_loadmodule_config mod_auth) \ || { say "Monolithic Auth module not found. Assuming we run against Apache 2.1+" LOAD_MOD_AUTH="$(get_loadmodule_config mod_auth_basic)" \ || fail "Auth_Basic module not found." LOAD_MOD_ACCESS_COMPAT="$(get_loadmodule_config mod_access_compat)" \ && { say "Found modules for Apache 2.3.0+" LOAD_MOD_AUTHN_CORE="$(get_loadmodule_config mod_authn_core)" \ || fail "Authn_Core module not found." LOAD_MOD_AUTHZ_CORE="$(get_loadmodule_config mod_authz_core)" \ || fail "Authz_Core module not found." LOAD_MOD_AUTHZ_HOST="$(get_loadmodule_config mod_authz_host)" \ || fail "Authz_Host module not found." LOAD_MOD_UNIXD=$(get_loadmodule_config mod_unixd) \ || fail "UnixD module not found" } LOAD_MOD_AUTHN_FILE="$(get_loadmodule_config mod_authn_file)" \ || fail "Authn_File module not found." LOAD_MOD_AUTHZ_USER="$(get_loadmodule_config mod_authz_user)" \ || fail "Authz_User module not found." } if [ ${APACHE_MPM:+set} ]; then LOAD_MOD_MPM=$(get_loadmodule_config mod_mpm_$APACHE_MPM) \ || fail "MPM module not found" fi if [ ${MODULE_PATH:+set} ]; then MOD_DAV_SVN="$MODULE_PATH/mod_dav_svn.so" MOD_AUTHZ_SVN="$MODULE_PATH/mod_authz_svn.so" else MOD_DAV_SVN="$ABS_BUILDDIR/subversion/mod_dav_svn/.libs/mod_dav_svn.so" MOD_AUTHZ_SVN="$ABS_BUILDDIR/subversion/mod_authz_svn/.libs/mod_authz_svn.so" fi [ -r "$MOD_DAV_SVN" ] \ || fail "dav_svn_module not found, please use '--enable-shared --enable-dso --with-apxs' with your 'configure' script" [ -r "$MOD_AUTHZ_SVN" ] \ || fail "authz_svn_module not found, please use '--enable-shared --enable-dso --with-apxs' with your 'configure' script" export LD_LIBRARY_PATH="$ABS_BUILDDIR/subversion/libsvn_ra_neon/.libs:$ABS_BUILDDIR/subversion/libsvn_ra_local/.libs:$ABS_BUILDDIR/subversion/libsvn_ra_svn/.libs:$LD_LIBRARY_PATH" MASTER_REPOS="${MASTER_REPOS:-"$HTTPD_ROOT/master_repos"}" SLAVE_REPOS="${SLAVE_REPOS:-"$HTTPD_ROOT/slave_repos"}" MASTER_HOST=127.0.0.2 SLAVE_HOST=127.0.0.1 #TEST_PORT=11111 TEST_PORT=$(($RANDOM+1024)) # location directive elements for master,slave,sync # tests currently work if master==slave,fail if different # ** Should different locations for each work? #MASTER_LOCATION="master" #SLAVE_LOCATION="slave" MASTER_LOCATION="repo" SLAVE_LOCATION="repo" SYNC_LOCATION="sync" MASTER_URL="http://${MASTER_HOST}:${TEST_PORT}/${MASTER_LOCATION}" SLAVE_URL="http://${SLAVE_HOST}:${TEST_PORT}/${SLAVE_LOCATION}" SYNC_URL="http://${SLAVE_HOST}:${TEST_PORT}/${SYNC_LOCATION}" BASE_URL="$SLAVE_URL" # setup server and repositories say "setting up in ${HTTPD_ROOT}:" mkdir -p $HTTPD_ROOT || fail "cannot mkdir $HTTPD_ROOT" HTTPD_CONFIG=$HTTPD_ROOT/cfg setup_config $HTTPD_CONFIG touch $HTTPD_ROOT/mime.types HTTPD_USERS="$HTTPD_ROOT/users" $HTPASSWD -bc $HTTPD_USERS jrandom rayjandom $HTPASSWD -b $HTTPD_USERS jconstant rayjandom $HTPASSWD -b $HTTPD_USERS scm scm $HTPASSWD -b $HTTPD_USERS svnsync svnsync $SVNADMIN create "$MASTER_REPOS" || fail "create master repos failed" $SVNADMIN create "$SLAVE_REPOS" || fail "create slave repos failed" # dup them $SVNADMIN dump "$MASTER_REPOS" | $SVNADMIN load "$SLAVE_REPOS" \ || fail "duplicate repositories failed" # make sure uuid's match read MASTER_UUID < "$MASTER_REPOS/db/uuid" read SLAVE_UUID < "$SLAVE_REPOS/db/uuid" [ "$SLAVE_UUID" = "$MASTER_UUID" ] \ || fail "master/slave uuid mismatch" # setup hooks: # slave allows revprop changes # master syncs changes to slave echo "#!/bin/sh" > "$SLAVE_REPOS/hooks/pre-revprop-change" echo "#!/bin/sh" > "$MASTER_REPOS/hooks/post-revprop-change" echo "#!/bin/sh" > "$MASTER_REPOS/hooks/post-commit" echo "$SVNSYNC --non-interactive sync '$SYNC_URL' --username=svnsync --password=svnsync" \ >> "$MASTER_REPOS/hooks/post-revprop-change" echo "$SVNSYNC --non-interactive sync '$SYNC_URL' --username=svnsync --password=svnsync" \ >> "$MASTER_REPOS/hooks/post-commit" chmod 0755 "$SLAVE_REPOS/hooks/pre-revprop-change" chmod 0755 "$MASTER_REPOS/hooks/post-revprop-change" chmod 0755 "$MASTER_REPOS/hooks/post-commit" say "created master and slave repositories" # test config $HTTPD -f $HTTPD_CONFIG -t || fail "httpd config failure in $HTTPD_CONFIG" # start httpd echo -n "${SCRIPT}: starting httpd: " $HTTPD -f $HTTPD_CONFIG -k start || fail "httpd start failed" echo "." say initializing svnsync to $SYNC_URL HTTPD_PID=$HTTPD_ROOT/pid $SVNSYNC initialize --non-interactive "$SYNC_URL" "$MASTER_URL" \ --username=svnsync --password=svnsync \ || fail "svnsync initialize failed" if [ $NO_TESTS ]; then echo "MASTER_URL=$MASTER_URL" echo "SLAVE_URL=$SLAVE_URL" exit fi # OK, let's start testing! Commit changes to slave, expect # them to proxy through to the master, and then # svnsync back to the slave # # reproducible test case from: # https://issues.apache.org/jira/browse/SVN-2939 # BASE_URL="$SLAVE_URL" say running svnmucc test to $BASE_URL svnmucc="$SVNMUCC --non-interactive --username jrandom --password rayjandom -mm" $svnmucc mkdir "$BASE_URL/trunk" mkdir "$BASE_URL/trunk/dir1" mkdir "$BASE_URL/trunk/dir1/dir2" $svnmucc rm "$BASE_URL/trunk/dir1/dir2" $svnmucc cp 2 "$BASE_URL/trunk" "$BASE_URL/branch" put /dev/null "$BASE_URL/branch/dir1/dir2" $svnmucc rm "$BASE_URL/branch" cp 2 "$BASE_URL/trunk" "$BASE_URL/branch" put /dev/null "$BASE_URL/branch/dir1/dir2" say "svn log on $BASE_URL : " $SVN --username jrandom --password rayjandom log -vq "$BASE_URL" # verify result: should be at rev 4 in both repos # FIXME: do more rigorous verification here MASTER_HEAD=`$SVNLOOK youngest "$MASTER_REPOS"` SLAVE_HEAD=`$SVNLOOK youngest "$SLAVE_REPOS"` say checking consistency of master, slave repositories: if [ "$MASTER_HEAD" != "4" ] || [ "$SLAVE_HEAD" != "4" ] ; then say FAIL: master, slave are at rev $MASTER_HEAD, $SLAVE_HEAD, not 4 say server may be started/stopped manually with: say " $HTTPD -f $HTTPD_CONFIG -k start|stop" fail charred remains in $HTTPD_ROOT for your perusal fi say "PASS: master, slave are both at r4, as expected" # The following test case is for the regression issue triggered by r917523. # The revision r917523 do some url encodings to the paths and uris which are # not url-encoded. But there is one additional url-encoding of an uri which is # already encoded. With this extra encoding, committing a path to slave which # has space in it fails. Please see this thread # http://svn.haxx.se/dev/archive-2011-03/0641.shtml for more info. say "Test case for regression issue triggered by r917523" $svnmucc cp 2 "$BASE_URL/trunk" "$BASE_URL/branch new" $svnmucc put /dev/null "$BASE_URL/branch new/file" \ --config-option servers:global:http-library=neon RETVAL=$? if [ $RETVAL -eq 0 ] ; then say "PASS: committing a path which has space in it passes" else say "FAIL: committing a path which has space in it fails as there are extra url-encodings happening in server side" fi # Test case for commit to out-dated(though target path is up to date) slave. # See issue #3860 for details. say "Test case for out-dated slave commit" svn="$SVN --non-interactive --username=jrandom --password=rayjandom" # Make a working copy of the slave. $svn checkout $SLAVE_URL $HTTPD_ROOT/wc cd $HTTPD_ROOT/wc # Add a new file named newfile and commit it. touch branch/newfile $svn add branch/newfile $svn commit -mm say "De-activating post-commit hook on $MASTER_REPOS to make $SLAVE_REPOS go out of sync" mv "$MASTER_REPOS/hooks/post-commit" "$MASTER_REPOS/hooks/post-commit_" echo "Change made to file in branch" > $HTTPD_ROOT/wc/branch/newfile $svn ci -m "Commit from slave" MASTER_HEAD=`$SVNLOOK youngest "$MASTER_REPOS"` SLAVE_HEAD=`$SVNLOOK youngest "$SLAVE_REPOS"` say "Now the slave is at r$SLAVE_HEAD and master is at r$MASTER_HEAD." # Now any other commit operation will fail with an out-of-date error $svn cp -m "Creating a branch" ^/trunk ^/branch/newbranch --config-option "servers:global:http-library=neon" RETVAL=$? if [ $RETVAL -eq 0 ]; then say "PASS: Commits succeed even with an out-of-date slave" else say "FAIL: Commits fail with an out-of-date slave" fi say "Some house-keeping..." say "Re-activating the post-commit hook on the master repo: $MASTER_REPOS." mv "$MASTER_REPOS/hooks/post-commit_" "$MASTER_REPOS/hooks/post-commit" say "Syncing slave with master." $SVNSYNC --non-interactive sync "$SYNC_URL" --username=svnsync --password=svnsync # shut it down echo -n "${SCRIPT}: stopping httpd: " $HTTPD -f $HTTPD_CONFIG -k stop echo "." exit 0