You can get the error indicates then connection broken and return, when first portsnap server goes down. If the first server goes down, portsnap will automatically switch to the next server for failover. Fix: for example, please recheck. % diff -u /usr/sbin/portsnap /tmp/portsnap --- /usr/sbin/portsnap +++ /tmp/portsnap @@ -45,6 +45,8 @@ -k KEY -- Trust an RSA key with SHA256 hash of KEY -p portsdir -- Location of uncompressed ports tree (default: /usr/ports/) + -r -- Do not reorder servernames. + (default: true) -s server -- Server from which to fetch updates. (default: portsnap.FreeBSD.org) path -- Extract only parts of the tree starting with the given @@ -81,6 +83,8 @@ DDSTATS="" INDEXONLY="" SERVERNAME="" + SERVERLIST="" + REORDER_SERVERLIST="YES" } # Parse the command line @@ -132,6 +136,9 @@ if [ ! -z "${SERVERNAME}" ]; then usage; fi shift; SERVERNAME="$1" ;; + -r) + REORDER_SERVERLIST="" + ;; cron | extract | fetch | update) COMMANDS="${COMMANDS} $1" ;; @@ -286,6 +293,16 @@ } +reorder_serverlist() { + [ "$REORDER_SERVERLIST" != "YES" ] && return + while [ $# -gt 0 ]; do + _item="$1"; shift + SERVERLIST=`echo $SERVERLIST | sed "s,$_item,,"` + SERVERLIST="$SERVERLIST $_item" + SERVERLIST="${SERVERLIST# }" + done +} + #### Core functionality -- the actual work gets done here # Use an SRV query to pick a server. If the SRV query doesn't provide @@ -350,16 +367,17 @@ SRV_W=`echo $X | cut -f 2 -d ' '` SRV_W=$(($SRV_W + $SRV_W_ADD)) if [ $SRV_RND -le $SRV_W ]; then - SERVERNAME=`echo $X | cut -f 3 -d ' '` - break + SERVERLIST="`echo ${X%.} | cut -f 3 -d ' '` ${SERVERLIST}" else SRV_RND=$(($SRV_RND - $SRV_W)) + SERVERLIST="${SERVERLIST} `echo $X | cut -f 3 -d ' '`" fi ;; esac done < serverlist - echo " using ${SERVERNAME}" + SERVERLIST="${SERVERLIST# }" + echo " using ${SERVERLIST}" } # Check that we have a public key with an appropriate hash, or @@ -369,12 +387,16 @@ return fi - echo -n "Fetching public key... " - rm -f pub.ssl - fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ - 2>${QUIETREDIR} || true - if ! [ -r pub.ssl ]; then + for X in ${SERVERLIST}; do + rm -f pub.ssl + echo -n "Fetching public key from ${X} ... " + fetch ${QUIETFLAG} http://${X}/pub.ssl \ + 2>${QUIETREDIR} && break echo "failed." + reorder_serverlist "${X}" + done + if ! [ -r pub.ssl ]; then + echo "Fetching public key failed." return 1 fi if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then @@ -387,13 +409,16 @@ # Fetch a snapshot tag fetch_tag() { - rm -f snapshot.ssl tag.new - - echo ${NDEBUG} "Fetching snapshot tag... " - fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl - 2>${QUIETREDIR} || true - if ! [ -r $1.ssl ]; then + for X in ${SERVERLIST}; do + rm -f snapshot.ssl tag.new + echo ${NDEBUG} "Fetching snapshot tag from ${X} ... " + fetch ${QUIETFLAG} http://${X}/$1.ssl \ + 2>${QUIETREDIR} && break echo "failed." + reorder_serverlist "${X}" + done + if ! [ -r $1.ssl ]; then + echo "Fetching snapshot tag failed." return 1 fi @@ -466,11 +491,19 @@ # Fetch snapshot metadata file fetch_metadata() { - rm -f ${SNAPSHOTHASH} tINDEX.new - echo ${NDEBUG} "Fetching snapshot metadata... " - fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} - 2>${QUIETREDIR} || return + for X in ${SERVERLIST}; do + rm -f ${SNAPSHOTHASH} tINDEX.new + echo ${NDEBUG} "Fetching snapshot metadata from ${X} ... " + fetch ${QUIETFLAG} http://${X}/t/${SNAPSHOTHASH} \ + 2>${QUIETREDIR} && break + echo "failed." + reorder_serverlist "${X}" + done + if ! [ -r ${SNAPSHOTHASH} ]; then + echo ${NDEBUG} "Fetching snapshot metadata failed" + return + fi if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then echo "snapshot metadata corrupt." return 1 @@ -554,14 +587,23 @@ fetch_metadata || return 1 fetch_metadata_sanity || return 1 - rm -f ${SNAPSHOTHASH}.tgz - rm -rf snap/ - # Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will # probably take a while, so the progrees reports that fetch(1) generates # will be useful for keeping the users' attention from drifting. - echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" - fetch http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 + for X in ${SERVERLIST}; do + rm -f ${SNAPSHOTHASH}.tgz + rm -rf snap/ + + echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}` from ${X}:" + fetch http://${X}/s/${SNAPSHOTHASH}.tgz && break + echo "failed." + reorder_serverlist "${X}" + done + + if ! [ -r ${SNAPSHOTHASH}.tgz ]; then + echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}` failed." + return 1 + fi echo -n "Extracting snapshot... " tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1 @@ -612,13 +654,30 @@ fetch_make_patchlist > patchlist # Attempt to fetch metadata patches - echo -n "Fetching `wc -l < patchlist | tr -d ' '` " - echo ${NDEBUG} "metadata patches.${DDSTATS}" - tr '|' '-' < patchlist | - lam -s "tp/" - -s ".gz" | - xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ - 2>${STATSREDIR} | fetch_progress - echo "done." + patchlist_l=`wc -l < patchlist | tr -d ' '` + [ ${patchlist_l} -ne 0 ] && { + for X in ${SERVERLIST}; do + echo -n "Fetching ${patchlist_l} " + echo ${NDEBUG} "metadata patches${DDSTATS} from ${X}" + tr '|' '-' < patchlist | + lam -s "tp/" - -s ".gz" | + xargs ${XARGST} ${PHTTPGET} ${X} \ + 2>${STATSREDIR} | fetch_progress + + br_flag=no + while read LINE; do + A=`echo ${LINE} | cut -f 1 -d '|'` + B=`echo ${LINE} | cut -f 2 -d '|'` + [ -f "${A}-${B}.gz" ] && continue + br_flag="YES" + echo "failed." + reorder_serverlist "${X}" + break + done < patchlist 2>${QUIETREDIR} + [ "$br_flag" = "YES" ] && continue + echo "done." + break + done # Attempt to apply metadata patches echo -n "Applying metadata patches... " @@ -638,6 +697,7 @@ rm -f diff OLD NEW ${X}-${Y}.gz ptmp done < patchlist 2>${QUIETREDIR} echo "done." + } # Update metadata without patches join -t '|' -v 2 tINDEX tINDEX.new | @@ -647,11 +707,18 @@ echo ${Y}; fi done > filelist - echo -n "Fetching `wc -l < filelist | tr -d ' '` " - echo ${NDEBUG} "metadata files... " - lam -s "f/" - -s ".gz" < filelist | - xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ - 2>${QUIETREDIR} + + filelist_l=`wc -l < filelist | tr -d ' '` + [ ${filelist_l} -ne 0 ] && { + for X in ${SERVERLIST}; do + echo -n "Fetching ${filelist_l} " + echo ${NDEBUG} "metadata files from ${X} ... " + lam -s "f/" - -s ".gz" < filelist | + xargs ${XARGST} ${PHTTPGET} ${X} \ + 2>${QUIETREDIR} && break + echo "failed." + reorder_serverlist "${X}" + done while read Y; do if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then @@ -662,6 +729,7 @@ fi done < filelist echo "done." + } # Extract the index gunzip -c files/`look INDEX tINDEX.new | @@ -673,12 +741,29 @@ fetch_make_patchlist > patchlist # Attempt to fetch ports patches - echo -n "Fetching `wc -l < patchlist | tr -d ' '` " - echo ${NDEBUG} "patches.${DDSTATS}" - tr '|' '-' < patchlist | lam -s "bp/" - | - xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ - 2>${STATSREDIR} | fetch_progress - echo "done." + patchlist_l=`wc -l < patchlist | tr -d ' '` + [ ${patchlist_l} -ne 0 ] && { + for X in ${SERVERLIST}; do + echo -n "Fetching ${patchlist_l} " + echo ${NDEBUG} "patches${DDSTATS} from ${X}" + tr '|' '-' < patchlist | lam -s "bp/" - | + xargs ${XARGST} ${PHTTPGET} ${X} \ + 2>${STATSREDIR} | fetch_progress + + br_flag=no + while read LINE; do + A=`echo ${LINE} | cut -f 1 -d '|'` + B=`echo ${LINE} | cut -f 2 -d '|'` + [ -f "${A}-${B}" ] && continue + br_flag="YES" + echo "failed." + reorder_serverlist "${X}" + break + done < patchlist 2>${QUIETREDIR} + [ "$br_flag" = "YES" ] && continue + echo "done." + break + done # Attempt to apply ports patches echo -n "Applying patches... " @@ -695,6 +780,7 @@ rm -f diff OLD NEW ${X}-${Y} done < patchlist 2>${QUIETREDIR} echo "done." + } # Update ports without patches join -t '|' -v 2 INDEX INDEX.new | @@ -704,11 +790,18 @@ echo ${Y}; fi done > filelist - echo -n "Fetching `wc -l < filelist | tr -d ' '` " - echo ${NDEBUG} "new ports or files... " - lam -s "f/" - -s ".gz" < filelist | - xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ - 2>${QUIETREDIR} + + filelist_l=`wc -l < filelist | tr -d ' '` + [ ${filelist_l} -ne 0 ] && { + for X in ${SERVERLIST}; do + echo -n "Fetching ${filelist_l} " + echo ${NDEBUG} "new ports or files from ${X} ... " + lam -s "f/" - -s ".gz" < filelist | + xargs ${XARGST} ${PHTTPGET} ${X} \ + 2>${QUIETREDIR} && break + echo "failed." + reorder_serverlist "${X}" + done while read Y; do if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then @@ -719,6 +812,7 @@ fi done < filelist echo "done." + } # Remove files which are no longer needed cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles How-To-Repeat: Example with broken portsnap2.freebsd.org lines: # /usr/sbin/portsnap fetch Looking up portsnap.FreeBSD.org mirrors... using portsnap2.FreeBSD.org. Fetching snapshot tag... fetch: http://portsnap2.FreeBSD.org./latest.ssl: Operation timed out failed. # # /tmp/portsnap fetch Looking up portsnap.FreeBSD.org mirrors... using portsnap2.FreeBSD.org portsnap1.FreeBSD.org Fetching snapshot tag from portsnap2.FreeBSD.org ... failed. Fetching snapshot tag from portsnap1.FreeBSD.org ... done. Fetching snapshot metadata from portsnap1.FreeBSD.org ... done. Updating from ÐÏÎÅÄÅÌØÎÉË, 24 ÁÐÒÅÌÑ 2006 Ç. 06:14:30 (EEST) to ÐÏÎÅÄÅÌØÎÉË, 24 ÁÐÒÅÌÑ 2006 Ç. 22:27:53 (EEST). Fetching 4 metadata patches from portsnap1.FreeBSD.org.. done. Applying metadata patches... done. Fetching 69 patches from portsnap1.FreeBSD.org....10....20....30....40....50....60.... done. Applying patches... done. Fetching 1 new ports or files from portsnap1.FreeBSD.org ... done. # [restore previous db/portsnap] # /tmp/portsnap -r fetch Looking up portsnap.FreeBSD.org mirrors... using portsnap2.FreeBSD.org portsnap1.FreeBSD.org Fetching snapshot tag from portsnap2.FreeBSD.org ... failed. Fetching snapshot tag from portsnap1.FreeBSD.org ... done. Fetching snapshot metadata from portsnap2.FreeBSD.org ... failed. Fetching snapshot metadata from portsnap1.FreeBSD.org ... done. Updating from ÐÏÎÅÄÅÌØÎÉË, 24 ÁÐÒÅÌÑ 2006 Ç. 06:14:30 (EEST) to ÐÏÎÅÄÅÌØÎÉË, 24 ÁÐÒÅÌÑ 2006 Ç. 22:27:53 (EEST). Fetching 4 metadata patches from portsnap2.FreeBSD.org failed. Fetching 4 metadata patches from portsnap1.FreeBSD.org.. done. Applying metadata patches... done. Fetching 69 patches from portsnap2.FreeBSD.org failed. Fetching 69 patches from portsnap1.FreeBSD.org....10....20....30....40....50....60.... done. Applying patches... done. Fetching 1 new ports or files from portsnap2.FreeBSD.org ... failed. Fetching 1 new ports or files from portsnap1.FreeBSD.org ... done. #
Responsible Changed From-To: freebsd-bugs->cperciva Over to portsnap author/maintainer.
Hi since you probably check this you could check make -DPORTSNAP_UPDATE update (on ports/Makefile) as well which actualy fetch and tries updating but do not run `portsnap=20 extract` between and fails thanks Jo=E3o A mensagem foi scaneada pelo sistema de e-mail e pode ser considerada segura. Service fornecido pelo Datacenter Matik https://datacenter.matik.com.br
Adding failover functionality to portsnap is on my TODO list for this summer; but since a server failing in the middle of a portsnap update is far less likely than a server failing before the update starts, I'll probably just do the failover once at the point where the snapshot tag is downloaded. Colin Percival
State Changed From-To: open->closed Fixed in HEAD, RELENG_6, and RELENG_5 in the src tree, and in version 1.1 of portsnap in the ports tree.