Fichiers de départ

bin/oldies/bin_incremental_backup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+export TARGET=/opt/backups/X11952
+export SOURCE=/opt/bin
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+export LEJOUR=`date +%d`
+export TODELETE=`date +%Y%m%d -d "3 months ago"`
+
+SCRIPT=`basename $0 .sh`
+FLAGFILE=$TARGET/flag_$SCRIPT.$LADATE
+
+if [ -f $FLAGFILE ]
+then
+  echo $FLAGFILE found ...
+  exit 0
+fi
+
+HEURE=`date +%H:%M`
+echo "--- $HEURE - BEGIN" >> $FLAGFILE
+
+/opt/bin/incremental_backup.sh
+
+HEURE=`date +%H:%M`
+echo "--- $HEURE - END" >> $FLAGFILE

bin/oldies/incremental_backup.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+if [ ! -e $SOURCE ]
+then
+  echo "$SOURCE n'existe pas !"
+  exit 1
+fi
+
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+export LEJOUR=`date +%d`
+export TODELETE=`date +%Y%m%d -d "3 months ago"`
+
+DESTINATION=`basename $SOURCE`
+if [ -d $TARGET/$LADATE/$DESTINATION ]
+then
+  echo "ALREADY DONE : $TARGET/$LADATE"
+  exit 0
+fi
+
+if [ ! -d $TARGET/$HIER/$DESTINATION ]
+then
+  # Le backup d'hier n'existe pas
+  #   PC éteint, problème backup, etc.
+  # On prend le dernier backup comme référence
+  echo "NOT FOUND : $TARGET/$HIER/$DESTINATION"
+  LAST_ONE=`ls -tr $TARGET | tail -n 1`
+  echo $LAST_ONE
+  if [ ! -d $TARGET/$LAST_ONE/$DESTINATION ]
+  then
+    echo "SECOND CHANCE, NOT FOUND : $TARGET/$LAST_ONE/$DESTINATION"
+    exit 1
+  fi
+  mv $TARGET/$LAST_ONE $TARGET/$HIER
+  ln -s $TARGET/$HIER $TARGET/$LAST_ONE
+fi
+
+if [ -d $TARGET/$TODELETE/$DESTINATION ]
+then
+  rm -rf $TARGET/$TODELETE/$DESTINATION
+fi
+
+echo $TARGET/$LADATE/$DESTINATION
+
+mkdir -p $TARGET
+
+/usr/bin/rsync -a --no-o --delete --safe-links $EXCLUDED_DIRS --link-dest=$TARGET/$HIER/ $SOURCE $TARGET/$LADATE/
+
+echo "DONE"

bin/oldies/incremental_backup.sh.20180829

@@ -0,0 +1,49 @@
+#!/bin/bash
+if [ ! -e $SOURCE ]
+then
+  echo "$SOURCE n'existe pas !"
+  exit 1
+fi
+
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+export LEJOUR=`date +%d`
+export TODELETE=`date +%Y%m%d -d "3 months ago"`
+
+DESTINATION=`basename $SOURCE`
+if [ -d $TARGET/$LADATE/$DESTINATION ]
+then
+  echo "ALREADY DONE : $TARGET/$LADATE"
+  exit 0
+fi
+
+if [ ! -d $TARGET/$HIER/$DESTINATION ]
+then
+  # Le backup d'hier n'existe pas
+  #   PC éteint, problème backup, etc.
+  # On prend le dernier backup comme référence
+  echo "NOT FOUND : $TARGET/$HIER/$DESTINATION"
+  LAST_ONE=`ls -tr $TARGET | tail -n 1`
+  echo $LAST_ONE
+  if [ ! -d $TARGET/$LAST_ONE/$DESTINATION ]
+  then
+    echo "SECOND CHANCE, NOT FOUND : $TARGET/$LAST_ONE/$DESTINATION"
+    exit 1
+  fi
+  mv $TARGET/$LAST_ONE $TARGET/$HIER
+  ln -s $TARGET/$HIER $TARGET/$LAST_ONE
+fi
+
+if [ -d $TARGET/$TODELETE/$DESTINATION ]
+then
+  rm -rf $TARGET/$TODELETE/$DESTINATION
+fi
+
+echo $TARGET/$LADATE/$DESTINATION
+
+mkdir -p $TARGET
+
+/usr/bin/rsync -a --no-o --delete --safe-links $EXCLUDED_DIRS --link-dest=$TARGET/$HIER/ $SOURCE $TARGET/$LADATE/
+
+echo "DONE"

bin/oldies/quotidien.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+#
+# Les backups du serveur topisto.net
+#
+/opt/bin/recup_topisto_backups.sh
+
+#
+# Les backups locaux
+#
+#/opt/bin/twitterbot_incremental_backup.sh
+#/opt/bin/www_incremental_backup.sh
+#/opt/bin/tibo_incremental_backup.sh
+#/opt/bin/bin_incremental_backup.sh
+/opt/bin/backup.sh
+
+#
+# Garder une trace des fichiers modifiés
+#
+#/opt/bin/trace_incremental_backup.sh

bin/oldies/recup_topisto_backups.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+ladate=`date +%Y%m%d`
+targetdir=/opt/topisto.net
+rootdir=/tmp/download_topisto
+backdir=$rootdir/$ladate
+
+param=`echo TTNsY2gxczNkM2sK | base64 -d`
+salt=`date +%Y%m%d%H`
+
+cmd="wget --quiet --user backup --password $param"
+
+if [ ! -d $targetdir ]
+then
+  mkdir -p $targetdir
+fi
+
+if [ ! -d $backdir ]
+then
+  rm -rf $rootdir/*
+  rm -rf $targetdir/*
+
+  mkdir -p $backdir
+
+  for fichier in www.tar.bz2 TOPISTO_apps.tar.bz2 
+  do
+    code=`echo -n /opt/backup/$fichier$salt | md5sum | awk '{ print $1 }' `
+    echo Récupération de $fichier dans $backdir/$fichier via $code
+    $cmd -O $backdir/$fichier https://www.topisto.net/backup/download.php?file=$code
+    cd $targetdir
+    echo Extraction de $backdir/$fichier dans $targetdir
+    tar xfj $backdir/$fichier
+  done
+else
+  echo "ALREADY DONE : Downloads from topisto.net"
+fi

bin/oldies/rsync.sh

@@ -0,0 +1,81 @@
+#! /bin/bash
+
+# Put exclude patterns into ~/.backup.exclude
+
+# Setup
+
+DEST="/opt/backups/rsync/"
+SRC="/home/tibo/public_html/"
+
+
+#############################################################################
+#
+# Copyright (c) 2009-2014, Florian Jung
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+#
+#############################################################################
+
+
+LC_COLLATE="C" # this script relies on the correct sorting order for [0-9]!
+
+#remove trailing slashes
+DEST=$(echo "$DEST" | sed -e 's|\(.*\)/$|\1|')
+SRC=$(echo "$SRC" | sed -e 's|\(.*\)/$|\1|')
+
+cd "$DEST"
+# get last and current backup names
+LAST="$(echo [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9] | sed -e 's|.* \([0-9]*\)$|\1|')"
+NOW=$(date +%Y%m%d)
+
+if echo $LAST | grep '\['; then
+  echo "no previous backup found. creating initial backup..."
+  LINKDEST=""
+else
+  echo "last backup was: $LAST"
+  LINKDEST="--link-dest=../$LAST/"
+fi
+
+echo "today's backup is: $NOW"
+
+if [ $LAST = $NOW ]; then
+  echo "ERROR: you already did a backup today"
+else
+  FOO='';
+  if [ x$1 != x ]; then FOO='-n'; fi
+# do the backup
+#  echo rsync -a -v -x $FOO --hard-links "$LINKDEST" --exclude-from="$SRC/.backup.exclude" "$SRC/" "$DEST/$NOW/"
+#  rsync -a -v -x $FOO --hard-links "$LINKDEST" --exclude-from="$SRC/.backup.exclude" "$SRC/" "$DEST/$NOW/"
+  echo rsync -a -v -x $FOO --hard-links "$LINKDEST" --exclude-from="$SRC/.backup.exclude" "$SRC/" "$DEST/$NOW/"
+  rsync -a -v -x $FOO --hard-links "$LINKDEST" "$SRC/" "$DEST/$NOW/"
+  echo
+# Disable the next 4 lines at your option
+  echo -n "Today's backup took "
+  du -sh "$LAST" "$NOW" | tail -n 1 | sed -e "s,$NOW,,"
+  echo -n "Total disk usage by backups: "
+  du -sh . | sed -e 's,\([kMGT]\).*,\1,'
+  echo done!
+  echo
+fi

bin/oldies/test_recup_topisto_backups.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+targetdir=/opt/backups/topisto.net
+tmpdir=/tmp/topisto.net
+rootdir=/tmp/download_topisto
+ladate=`date +%Y%m%d`
+param=`echo TTNsY2gxczNkM2sK | base64 -d`
+cmd="wget --quiet --user backup --password $param"
+backdir=$rootdir/$ladate
+salt=`date +%Y%m%d%H`
+
+for fichier in Twitterbot.tar.bz2 www.tar.bz2
+do
+    code=`echo -n /opt/backup/$fichier$salt | md5sum | awk '{ print $1 }' `
+    echo Récupération de $fichier $code 
+    $cmd -O /tmp/$fichier https://www.topisto.net/backup/download.php?file=$code
+done
+
+ls -lhtr /tmp/*.tar.bz2
+rm -f /tmp/*.tar.bz2
+

bin/oldies/tibo_TOPISTO_incremental_backup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+export TARGET=/opt/backups/X11952/TOPISTO
+export SOURCE=/home/tibo/TOPISTO
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+/opt/bin/incremental_backup.sh

bin/oldies/tibo_incremental_backup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+export TARGET=/opt/backups/X11952
+export SOURCE=/home/tibo
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+export EXCLUDED_DIRS="--exclude-from /home/tibo/backups/rsync_excluded_dirs.txt"
+
+/opt/bin/incremental_backup.sh

bin/oldies/tibo_public_html_incremental_backup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+export TARGET=/opt/backups/X11952/tibo_public_html
+export SOURCE=/home/tibo/public_html
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+/opt/bin/incremental_backup.sh

bin/oldies/topisto_incremental_backup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+export TARGET=/opt/topisto.net
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+export SOURCE=/tmp/topisto.net/www
+
+/opt/bin/incremental_backup.sh
+
+export SOURCE=/tmp/topisto.net/TOPISTO
+
+/opt/bin/incremental_backup.sh

bin/oldies/trace_incremental_backup.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+export LEJOUR=`date +%d`
+export TODELETE=`date +%Y%m%d -d "3 months ago"`
+
+# Garder une trace des fichiers qui ont changé
+# c'est à dire ceux qui ne sont pas des liens
+find /opt/backups/*/$LADATE -type f -printf '%n %p\n' | awk '$1 == 1{print $2}' > /opt/backups/trace_$LADATE.log
+
+
+if [ "$LEJOUR" != "01" ]
+then
+	echo TO DELETE $TODELETE
+	rm -f /opt/backups/trace_$TODELETE.log
+	rm -rf /opt/backups/*/$TODELETE
+fi

bin/oldies/twitterbot_incremental_backup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+export TARGET=/opt/backups/Twitterbot
+export SOURCE=/opt/Twitterbot
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+/opt/bin/incremental_backup.sh

bin/oldies/www_incremental_backup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+export TARGET=/opt/backups/X11952
+export SOURCE=/var/www/html
+export LADATE=`date +%Y%m%d`
+export HIER=`date +%Y%m%d -d "1 day ago"`
+
+/opt/bin/incremental_backup.sh

bin/r_backup.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+export TARGET=/opt/backups
+export LADATE=`date +%Y%m%d`
+export TODELETE=`date +%Y%m%d -d "3 months ago"`
+
+SCRIPT=`basename $0 .sh`
+FLAGFILE=$TARGET/flags/$SCRIPT.$LADATE
+
+if [ -f $FLAGFILE ]
+then
+  exit 0
+fi
+
+if [ ! -f $TARGET/data/liste.txt ]
+then
+  echo No $TARGET/data/liste.txt found ...
+  exit 1 
+fi
+
+if [ ! -f $TARGET/data/signatures.sqlite ]
+then
+  echo No $TARGET/data/signatures.sqlite found ...
+  exit 1
+fi
+
+HEURE=`date +%H:%M`
+echo "--- $HEURE - BEGIN" >> $FLAGFILE
+
+for SOURCE in `grep -v "#" $TARGET/data/liste.txt`
+do
+  HEURE=`date +%H:%M`
+  echo "--- $HEURE - $SOURCE" >> $FLAGFILE
+  php $TARGET/scripts/backup.php $SOURCE $TARGET/data
+done
+
+HEURE=`date +%H:%M`
+echo "--- $HEURE - END" >> $FLAGFILE
+
+sqlite3 $TARGET/data/signatures.sqlite 'VACUUM'
+cp $TARGET/data/signatures.sqlite $TARGET/data/$LADATE/signatures.sqlite
+
+sqlite3 $TARGET/data/signatures.sqlite 'select substr(filename,0,9) as date, count(*) as nbre, sum(filesize) as size from signatures group by substr(filename,0,9)' >> $FLAGFILE

data/liste.txt

@@ -0,0 +1,4 @@
+/opt/bin
+/opt/topisto.net
+/var/www
+/home/tibo

scripts/backup.php

@@ -0,0 +1,177 @@
+<?php
+
+function my_echo_with_eol($msg)
+{
+  if (false) echo $msg.PHP_EOL;
+}
+
+function gzCompressFile($source, $dest, $level = 9){ 
+    $mode = 'wb' . $level; 
+    $error = false; 
+    if ($fp_out = gzopen($dest, $mode)) { 
+        if ($fp_in = fopen($source,'rb')) { 
+            while (!feof($fp_in)) 
+                gzwrite($fp_out, fread($fp_in, 1024 * 512)); 
+            fclose($fp_in); 
+        } else {
+            $error = true; 
+        }
+        gzclose($fp_out); 
+    } else {
+        $error = true; 
+    }
+    if ($error)
+        return false; 
+    else
+        return $dest; 
+} 
+
+function fileBackup($file_db, $fichier, $destination) // Fonction qui liste un dossier de façon récursive
+{
+    $ladate = date('Ymd');
+    $basename = basename($fichier);
+    $filename = str_replace('//', '/', "$ladate/$fichier");
+    $filesize = filesize($fichier);
+    $archive  = "$destination/$filename"; 
+    $flag = false;
+
+    // On sort si c'est un dossier ou un fichier caché
+    if ($basename[0] == '.') return FALSE;
+    if (is_dir($fichier))    return FALSE;
+
+    // On sort si l'archive existe
+    if (file_exists($archive))      return FALSE;
+    if (file_exists("$archive.gz")) return FALSE;
+
+    my_echo_with_eol("Compute SHA1 of $basename");
+    $sha1_fichier = sha1_file($fichier);
+
+    // Re-Hash with filesize to avoid conflict
+    $sha1_fichier = sha1($sha1_fichier.$filesize);
+
+    $insert = "INSERT INTO signatures (sha1, filename, filesize) VALUES (:sha1, :filename, :filesize)"; 
+    $stmt1  = $file_db->prepare($insert);
+
+    $update = "UPDATE signatures SET filename = :filename , filesize = :filesize WHERE sha1 = :sha1";
+    $stmt2  = $file_db->prepare($update);
+
+    $delete = "DELETE FROM signatures WHERE sha1 = :sha1";
+    $stmt3  = $file_db->prepare($delete);
+
+    my_echo_with_eol("Searching SHA1 in database for $basename");
+    $result = $file_db->query("SELECT filename, filesize FROM signatures WHERE sha1 = '$sha1_fichier'");
+    if ($result !== FALSE)
+    {
+        foreach($result as $f) 
+        {
+            my_echo_with_eol("Found ".$f['filename']);
+            if ($f['filesize'] != $filesize) my_echo_with_eol("Oups, database and file filesize differ !");
+            $in  = $destination.'/'.$f['filename'];
+            $out = $archive;
+            if (file_exists($in)) link($in, $out);
+            if (file_exists("$in.gz")) link("$in.gz", "$out.gz");
+            if (file_exists($out)||file_exists("$out.gz"))
+            {
+              $stmt2->execute(array(':sha1' => $sha1_fichier, ':filename' => $filename, ':filesize' => $filesize));
+              $flag = true;
+              my_echo_with_eol("Update database filename : $filename");
+            }
+
+            if (!$flag)
+	    {
+              my_echo_with_eol("Delete database sha1 : $sha1_fichier");
+              $stmt3->execute(array(':sha1' => $sha1_fichier));
+            }
+        }
+    }
+    if (!$flag) 
+    {
+        my_echo_with_eol("Not Found $basename");
+
+        // Les formats compressés sont juste copiés
+        $ext = pathinfo($fichier, PATHINFO_EXTENSION);
+        if ($ext == "avi") copy($fichier, $archive);
+        if ($ext == "mkv") copy($fichier, $archive);
+        if ($ext == "mp4") copy($fichier, $archive);
+        if ($ext == "mp3") copy($fichier, $archive);
+
+        if ($ext == "bz2") copy($fichier, $archive);
+        if ($ext == "gz")  copy($fichier, $archive);
+        if ($ext == "zip") copy($fichier, $archive);
+
+        if ($ext == "iso") copy($fichier, $archive);
+
+        if (!file_exists($archive)) 
+        {
+          my_echo_with_eol("Compress $basename");
+          gzCompressFile($fichier, "$archive.gz");
+        } else my_echo_with_eol("Copy $basename");
+
+        my_echo_with_eol("Insert database filename : $filename");
+	$stmt1->execute(array(':sha1' => $sha1_fichier, ':filename' => $filename, ':filesize' => $filesize));
+    }
+    my_echo_with_eol("--------- ");
+    return TRUE;
+}
+
+function listeDossier($file_db, $dossier, $destination) // Fonction qui liste un dossier de façon récursive
+{
+    $ladate = date('Ymd');
+
+    if (file_exists("$destination/$ladate/$dossier"))
+    {
+      my_echo_with_eol('BACKUP already done');
+      return true;
+    }
+
+    if (is_dir($dossier))
+    {
+        if (!file_exists("$destination/$ladate")) mkdir("$destination/$ladate");
+        $path = explode("/", $dossier);
+        $init_path = "$destination/$ladate";
+        foreach($path as $p)
+        {
+            $init_path .= '/'.$p;
+            if (!file_exists($init_path)) mkdir($init_path);
+        }
+        my_echo_with_eol("BACKUP OF $dossier");
+        if($dossierOuvert=opendir($dossier))
+        {
+            $n = 0;
+            while(($fichier=readdir($dossierOuvert))!== false)
+            {
+                // if ($n++ == 8) break;
+
+                if ($fichier[0] == '.') continue;
+
+                if(is_dir("$dossier/$fichier"))
+                    listeDossier($file_db, "$dossier/$fichier", $destination);
+                else 
+                    fileBackup($file_db, "$dossier/$fichier", $destination);
+            }
+        }
+    }
+    else
+    {
+        my_echo_with_eol("Erreur, le paramètre précisé dans la fonction n'est pas un dossier!");
+    }
+}
+
+$source = $argv[1];
+$backup = $argv[2];
+$signatures=$backup.'/signatures.sqlite';
+
+if (!file_exists($source))     die("$source NOT FOUND !");
+if (!is_dir($backup))          die("$backup NOT FOUND !");
+if (!file_exists($signatures)) die("$signatures NOT FOUND !");
+
+$connexion = new PDO('sqlite:'.$signatures);
+my_echo_with_eol("DATABASE IS OPEN");
+if(is_dir($backup))
+    listeDossier($connexion, $source, $backup);
+else 
+    fileBackup($connexion, $source, $backup);
+$connexion = null;
+my_echo_with_eol("DATABASE IS CLOSED");
+
+?>

scripts/controle.php

@@ -0,0 +1,32 @@
+<?php
+$global_flag = true;
+$backup = $argv[1];
+$signatures=$backup.'/signatures.sqlite';
+
+if (!is_dir($backup))          die("$backup NOT FOUND !");
+if (!file_exists($signatures)) die("$signatures NOT FOUND !");
+
+$connexion = new PDO('sqlite:'.$signatures);
+
+$result = $connexion->query("SELECT sha1, filename, filesize FROM signatures");
+if ($result !== FALSE)
+  foreach($result as $f)
+  {
+    $flag    = FALSE;
+    $fichier = $backup.'/'.$f['filename'];
+    
+    if (file_exists($fichier)) $flag = TRUE;
+    if (file_exists("$fichier.gz")) $flag = TRUE;
+
+    if (!$flag)
+    {
+      echo $fichier.' inexistant !'.PHP_EOL;
+      $global_flag = FALSE;
+    }
+  }
+$connexion = null;
+
+if ($global_flag) echo "Control signatures OK".PHP_EOL;
+else echo "Control signatures NOK".PHP_EOL;
+
+?>