#!/bin/sh
# ~jhs/bin/.sh/distfiles_fetch	by jhs@
# http://berklix.com/~jhs/bin/.sh/distfiles_fetch

echo "Consider first running /usr/local/bin/ctms to see if ports/ is recent"
#	/pub/FreeBSD/branches/-current/ports
#		is maintained via ~/public_html/src/bsd/jhs/bin/local/mail
#	/pri/FreeBSD/branches/-current/ports
#		is examined by ~jhs/public_html/src/bsd/jhs/bin/local/ctms
echo "Warning: This may takes a week to run."
echo "Warning: You may want to setenv IS_INTERACTIVE or BATCH."
#	See /usr/ports/Mk/* re IS_INTERACTIVE & BATCH
echo "Warning: This will need maybe 50G extra free space while running."
#	If a backup of distfiles is running at the same time,
#	it will waste CPU & may overflow disk,
cd /pub/FreeBSD/dists || exit 1
echo "DU `pwd`"
echo "Checking ports/ & ports/distfiles."
# PORTSDIRS:
#	Releases for which we have complete ports/
#	directory structures, & will do make fetch.
#	Older ones commented out, now that I run this script fairly often,
#	as it consumes too much CPU, keep refetching cp & cmpd old stuff.
PORTSDIRS="${PORTSDIRS} current"
#	current relies on a link created by: ln -s ../branches/-current current
PORTSDIRS="${PORTSDIRS} 9.0-RELEASE"
PORTSDIRS="${PORTSDIRS} 8.2-RELEASE"
# JJLATER comment these below out again,
#	After Ive done a run with my newly fixed FTP proxy.
PORTSDIRS="${PORTSDIRS} 7.4-RELEASE"
PORTSDIRS="${PORTSDIRS} 6.4-RELEASE"	# Last release with isdn support. ?
PORTSDIRS="${PORTSDIRS} 4.11-RELEASE"

# Add a symbolic link so next for loop can treat current the same as releases.
if test -e /pri/FreeBSD/releases/current ; then	#{
	cd /pri/FreeBSD/releases/current || exit 1
	rm -f ports
	ln -s ../../branches/`uname -m`/-current/ports ports
	if test ! -L /pri/FreeBSD/branches/i386/-current/ports ; then	#{
	ln -s ../../branches/i386/-current/686/-current/ports \
		/pri/FreeBSD/branches/i386/-current/ports
	fi								#}
fi							#}

# Check all directories exist else exit.
for i in ${PORTSDIRS}
	do	#{
	echo "$0 Loop Check Exist Start $i {"
	echo "Ensuring existence of /pri/FreeBSD/releases/$i/ports"
	cd /pri/FreeBSD/releases/$i/ports || exit 1
	echo "Ensuring existence of /pri/FreeBSD/releases/$i/ports/distfiles"
	#	Putting this in backets	(cd distfiles || exit 1)
	#	fails to catch error code on non existences,
	#	so as distfiles is a symbolic link elsewhere, add a cd after.
	cd distfiles || exit 1
	cd /pri/FreeBSD/releases/$i/ports || exit 1
	echo "Ensuring /pri/FreeBSD/releases/$i/ports/.customise"
	#	else if not customised, fetch may lock up.
	#	however if it Is customised then ctm update will 
	#	periodically fail. Quandary !
	if test -f .customise ; then	#{
		echo OK
	else							#}{
		echo -n "Not customised, Run: "
		echo "customise /pri/FreeBSD/releases/$i/ports $i"
		echo "	(If $i=current, specify latest release,"
		echo "	 eg maybe `uname -r` or newer)."
		exit 1
	fi							#}
	echo "$0 Loop Check Exist End $i }"
	done	#}

echo "Fetching non interactive for all releases"
# Fetch each release.
#	No need to add code to fetch in reverse numeric order of releases,
#	(using something like	echo ${PORTSDIRS} | sed -e 's/ /\n/' | sort -r )
#	for 2 reasons:
#	- Reverse order comes from top of this script.
#	- Release numeric order is not chronological order,
#	 (as we often have 2 major releases interleaving the minor numbers)
for i in ${PORTSDIRS}
	do	#{
	echo "$0 Loop Fetch Start $i {"
	cd /pri/FreeBSD/releases/$i/ports || exit 1
	/bin/pwd
	PORTSDIR=`pwd` ; export PORTSDIR
	echo "$0 Loop Fetch Batch Start $i {"
	nice nice make -k -j 2 BATCH=YES checksum
	# -j 2 	otherwise single slow sites will delay the whole fetch.
	# -j 3 	is too much load
	# "checksum" does more than "fetch", detecting occasional bad
	# distfiles (from eg when make fetch is run in
	# background & DSL link drops periodicaly).
	echo "$0 Loop Fetch Batch End $i }"
	echo "$0 Loop Strip Start $i {"
	nice nice distfiles_cmpd
	# The line above can be un-necessarily heavy,
	# comparing & deleting over 30G of largely similar files between
	# latest releas & current. Its best if fetch order here matches strip
	# order in distfiles_cmpd.
	echo "$0 Loop Strip End $i }"
	echo "$0 Loop Fetch End $i }"
	done	#}

echo "This only fetched distfiles needed by BATCH, not for IS_INTERACTIVE"
echo "so some distfiles needed for interactive current may be languishing in"
echo "/pub/freebsd/dists/current.old , so do not just delete them !"

#echo "Fetching interactive ports, Warning: May hang on input."
#for i in ${PORTSDIRS}
#	do	#{
#	echo "$0 Loop Fetch Start $i {"
#	cd /pri/FreeBSD/releases/$i/ports || exit 1
#	/bin/pwd
#	PORTSDIR=`pwd` ; export PORTSDIR
#	echo "$0 Loop Fetch Interactive Start $i {"
#	echo "Interactive" # More complete, but slower.
#	nice nice make -k INTERACTIVE=YES checksum
#		# Above, no -j as it would get confusing.
#	echo "$0 Loop Fetch Interactive End $i }"
#	echo "$0 Loop Strip Start $i {"
#	nice nice distfiles_cmpd
#	echo "$0 Loop Strip End $i }"
#	echo "$0 Loop Fetch End $i }"
#	done	#}

echo "Suggestion: Run nice nice distfiles_cmpd |& grep Different"