Voici les 3 scripts appelés depuis la crontab :
sync-libre-office.sh :
#!/bin/dash
fatal() {
echo "$1"
exit 1
}
warn() {
echo "$1"
}
# http://download.documentfoundation.org/mirroring.html
RSYNCSOURCE=rsync://rsync.documentfoundation.org/tdf-pub
# Define where you want the mirror-data to be on your mirror
BASEDIR=/home/libre-office/
if [ ! -d ${BASEDIR} ]; then
warn "${BASEDIR} does not exist yet, trying to create it..."
mkdir -p ${BASEDIR} || fatal "Creation of ${BASEDIR} failed."
fi
# Première synchro avec débit limité à 41 Mb/s :
# rajouter --bwlimit=5000 --progress (+démarrer dans un screen)
rsync --verbose --recursive --times --links --hard-links --perms \
--stats --delete-after \
${RSYNCSOURCE} ${BASEDIR} || fatal "!!!!!!!!!!!!!!!!!!! ERREUR : RSYNC EN ECHEC !!!!!!!!!!!!!!!!!!!"
#date -u > ${BASEDIR}/.trace/$(hostname -f)
echo "######################## SYNCHRONISATION OK ########################"
date
sync-ubuntu-cd.sh :
#!/bin/dash
fatal() {
echo "$1"
exit 1
}
warn() {
echo "$1"
}
# Find a source mirror near you which supports rsync on
# https://launchpad.net/ubuntu/+cdmirrors
# rsync://<iso-country-code>.rsync.releases.ubuntu.com/releases should always work
RSYNCSOURCE=rsync://fr.rsync.releases.ubuntu.com/releases
# Define where you want the mirror-data to be on your mirror
BASEDIR=/home/ubuntu-cd/
if [ ! -d ${BASEDIR} ]; then
warn "${BASEDIR} does not exist yet, trying to create it..."
mkdir -p ${BASEDIR} || fatal "Creation of ${BASEDIR} failed."
fi
# Première synchro avec débit limité à 41 Mb/s :
# rajouter --bwlimit=5000 --progress (+démarrer dans un screen)
rsync --verbose --recursive --times --links --hard-links --perms \
--stats --delete-after \
${RSYNCSOURCE} ${BASEDIR} || fatal "!!!!!!!!!!!!!!!!!!! ERREUR : RSYNC EN ECHEC !!!!!!!!!!!!!!!!!!!"
date -u > ${BASEDIR}/.trace/$(hostname -f)
echo "######################## SYNCHRONISATION OK ########################"
date
sync-ubuntu-archive.sh :
Il y a une particularité à ce script : il doit déjà synchroniser tous les fichiers avant de mettre à jour les indexes.
Si les indexes sont mis à jour avant les fichiers, on risque d'avoir un utilisateur qui tente de télécharger un fichier qui n'est pas encore sur le miroir (une mise-à-jour de sécurité par exemple).
C'est pour cela que pour ce type de miroir, un script avec un Rsync en deux étapes est nécessaire.
Je propose un double script : le premier script appelle le second, ce qui permet d'avoir des logs de ce qui a été réalisé.
Script sync-ubuntu-archive.sh
#!/bin/dash
fatal() {
echo "$1"
exit 1
}
warn() {
echo "$1"
}
LOGDIR=/home/ubuntu-archive/log # Emplacement des log
NB_INCR=10 # Nombre de sauvegardes
if [ ! -d ${LOGDIR} ]; then
warn "${LOGDIR} does not exist yet, trying to create it..."
mkdir -p ${LOGDIR} || fatal "Creation of ${LOGDIR} failed."
fi
#### Decale les autres fichiers de log / err
i=${NB_INCR}
while [ $i -gt 0 ]
do
j=`expr $i - 1`
if [ -f ${LOGDIR}/sync-ubuntu.log.$j ]; then
mv ${LOGDIR}/sync-ubuntu.log.$j ${LOGDIR}/sync-ubuntu.log.$i
fi
if [ -f ${LOGDIR}/sync-ubuntu.err.$j ]; then
mv ${LOGDIR}/sync-ubuntu.err.$j ${LOGDIR}/sync-ubuntu.err.$i
fi
i=`expr $i - 1`
done
echo "Ubuntu archive mirror: two stage sync"
# lancement du script principal
/home/scripts/sync-ubuntu-archive-2.sh >${LOGDIR}/sync-ubuntu.log.0 2>${LOGDIR}/sync-ubuntu.err.0 &
Script sync-ubuntu-archive-2.sh
#!/bin/dash
fatal() {
echo "$1"
exit 1
}
warn() {
echo "$1"
}
# Find a source mirror near you which supports rsync on
# https://launchpad.net/ubuntu/+archivemirrors
# rsync://<iso-country-code>.rsync.archive.ubuntu.com/ubuntu should always work
RSYNCSOURCE=rsync://fr.archive.ubuntu.com/ubuntu
# Define where you want the mirror-data to be on your mirror
BASEDIR=/home/ubuntu-archive/ubuntu/
if [ ! -d ${BASEDIR} ]; then
warn "${BASEDIR} does not exist yet, trying to create it..."
mkdir -p ${BASEDIR} || fatal "Creation of ${BASEDIR} failed."
fi
attempt="1"
# First sync with bit rate limited to 41 Mb/s: add
# --bwlimit=5000 --progress \
# Step 1 of 2
date -R
echo "#################### SYNC STEP 1 OF 2 ####################"
echo "exclude: Packages*, Sources*, Release*, InRelease; .~tmp~"
rsync --verbose --out-format="%o %b bytes for sync %n (%l bytes)" \
--recursive --times --links --safe-links --hard-links --perms \
--stats \
--exclude "/dists" \
--exclude "Packages*" --exclude "Sources*" \
--exclude "Release*" --exclude "InRelease" \
--exclude ".~tmp~" \
--exclude ".trace" \
${RSYNCSOURCE} ${BASEDIR} || attempt="2"
if [ "$attempt" = "2" ]; then
echo "!!!!!!!!!!!!!!!!!!! ERROR : SYNC STEP 1 OF 2 HAS FAILED (1st attempt) !!!!!!!!!!!!!!!!!!!"
echo "ERROR : SYNC STEP 1 OF 2 HAS FAILED (1st attempt)" >&2
echo "-------------------------------------------------" >&2
date -R
echo "Second attempt after 4 minutes (240 seconds) of pause"
sleep 240
date -R
echo "#################### SYNC STEP 1 OF 2 (2nd attempt) ####################"
echo "exclude: Packages*, Sources*, Release*, InRelease; .~tmp~"
rsync --verbose --out-format="%o %b bytes for sync %n (%l bytes)" \
--recursive --times --links --safe-links --hard-links --perms \
--stats \
--exclude "/dists" \
--exclude "Packages*" --exclude "Sources*" \
--exclude "Release*" --exclude "InRelease" \
--exclude ".~tmp~" \
--exclude ".trace" \
${RSYNCSOURCE} ${BASEDIR} || attempt="3"
fi
if [ "$attempt" = "3" ]; then
echo "!!!!!!!!!!!!!!!!!!! ERROR : SYNC STEP 1 OF 2 HAS FAILED (2nd attempt) !!!!!!!!!!!!!!!!!!!"
echo "ERROR : SYNC STEP 1 OF 2 HAS FAILED (2nd attempt)" >&2
echo "-------------------------------------------------" >&2
date -R
echo "Third attempt after 10 minutes (600 seconds) of pause"
sleep 600
date -R
echo "#################### SYNC STEP 1 OF 2 (3rd attempt) ####################"
echo "exclude: Packages*, Sources*, Release*, InRelease; .~tmp~"
rsync --verbose --out-format="%o %b bytes for sync %n (%l bytes)" \
--recursive --times --links --safe-links --hard-links --perms \
--stats \
--exclude "/dists" \
--exclude "Packages*" --exclude "Sources*" \
--exclude "Release*" --exclude "InRelease" \
--exclude ".~tmp~" \
--exclude ".trace" \
${RSYNCSOURCE} ${BASEDIR} || fatal "!!!!!!!!!!!!!!!!!!! ERROR : SYNC STEP 1 OF 2 HAS FAILED (3rd attempt) !!!!!!!!!!!!!!!!!!!"
fi
echo "#################### SYNC STEP 1 OF 2 IS OK WITH $attempt attempt ####################"
date -R
# Step 2 of 2
attempt="1"
echo "#################### SYNC STEP 2 OF 2 ####################"
echo "SYNC Packages* + Sources* + Release* + InRelease"
echo "exclude: .~tmp~"
rsync --verbose --out-format="%o %b bytes for sync %n (%l bytes)" \
--recursive --times --links --safe-links --hard-links --perms \
--stats --delete --delete-after \
--exclude ".~tmp~" \
${RSYNCSOURCE} ${BASEDIR} || attempt="2"
if [ "$attempt" = "2" ]; then
echo "!!!!!!!!!!!!!!!!!!! ERROR : SYNC STEP 2 OF 2 HAS FAILED (1st attempt) !!!!!!!!!!!!!!!!!!!"
echo "ERROR : SYNC STEP 2 OF 2 HAS FAILED (1st attempt)" >&2
echo "-------------------------------------------------" >&2
date -R
echo "Second attempt after 4 minutes (240 seconds) of pause"
sleep 240
date -R
echo "#################### SYNC STEP 2 OF 2 (2nd attempt) ####################"
echo "SYNC Packages* + Sources* + Release* + InRelease"
echo "exclude: .~tmp~"
rsync --verbose --out-format="%o %b bytes for sync %n (%l bytes)" \
--recursive --times --links --safe-links --hard-links --perms \
--stats --delete --delete-after \
--exclude ".~tmp~" \
${RSYNCSOURCE} ${BASEDIR} || attempt="3"
fi
if [ "$attempt" = "3" ]; then
echo "!!!!!!!!!!!!!!!!!!! ERROR : SYNC STEP 2 OF 2 HAS FAILED (2nd attempt !!!!!!!!!!!!!!!!!!!"
echo "ERROR : SYNC STEP 2 OF 2 HAS FAILED (2nd attempt)" >&2
echo "-------------------------------------------------" >&2
date -R
echo "Third attempt after 10 minutes (600 seconds) of pause"
sleep 600
date -R
echo "#################### SYNC STEP 2 OF 2 (3rd attempt) ####################"
echo "SYNC Packages* + Sources* + Release* + InRelease"
echo "exclude: .~tmp~"
rsync --verbose --out-format="%o %b bytes for sync %n (%l bytes)" \
--recursive --times --links --safe-links --hard-links --perms \
--stats --delete --delete-after \
--exclude ".~tmp~" \
${RSYNCSOURCE} ${BASEDIR} || fatal "!!!!!!!!!!!!!!!!!!! ERROR : SYNC STEP 2 OF 2 HAS FAILED (3rd attempt) !!!!!!!!!!!!!!!!!!!"
fi
echo "#################### SYNC STEP 2 OF 2 IS OK WITH $attempt attempt ####################"
date -R > ${BASEDIR}/project/trace/fr.archive.ubuntu.com
date -R
Pour la première exécution, quand le miroir que vous venez de monter est vise, il est fortement conseillé de limiter le débit avec la directive --bwlimit=5000 qui limite le débit à 5000 Kio/s afin de ne pas trop charger le serveur qui vous fourni les fichiers. Vous pouvez également laisser cette directive après pour ne pas trop dégrader votre connexion lors des mises à jour.