Adds backup vm in Vagrant

Vagrantfile

@@ -24,6 +24,21 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box_version = ">= 308.0.1"
   config.vm.box_url = "http://%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json" % $update_channel
 
+  config.vm.define "backup" do |backup|
+    backup.vm.provider :virtualbox do |vb|
+      vb.memory = 512
+      vb.cpus = 1
+      vb.check_guest_additions = false
+      vb.functional_vboxsf = false
+    end
+     # plugin conflict
+    if Vagrant.has_plugin?("vagrant-vbguest") then
+      backup.vbguest.auto_update = false
+    end
+    backup.vm.hostname = "backup.dev"
+    backup.vm.network :private_network, ip: "192.168.65.100"
+  end
+
   (1..$num_instances).each do |i|
     config.vm.define "core-#{i}" do |core|
       core.vm.provider :virtualbox do |vb|
@@ -44,8 +59,16 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       core.vm.network :private_network, ip: "#{BASE_IP_ADDR}.#{i+1}"
       core.vm.synced_folder ".", "/data/indiehosters", id: "coreos-indiehosters", :nfs => true, :mount_options => ['nolock,vers=3,udp']
       core.vm.provision :file, source: "./cloud-config", destination: "/tmp/vagrantfile-user-data"
+      $install_insecure_keys = <<SCRIPT
+mkdir ~/.ssh
+wget https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub -O ~/.ssh/id_rsa.pub
+wget https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant -O ~/.ssh/id_rsa
+chmod 600 ~/.ssh/id_rsa
+SCRIPT
+      core.vm.provision :shell, inline: $install_insecure_keys
       core.vm.provision :shell, inline: "mkdir -p /data/runtime/haproxy/approved-certs; cp /data/indiehosters/scripts/unsecure-certs/*.pem /data/runtime/haproxy/approved-certs"
       core.vm.provision :shell, path: "./scripts/setup.sh", args: [HOSTNAME]
     end
+    
   end
 end

cloud-config

@@ -9,3 +9,9 @@ coreos:
   units:
     - name: etcd.service
       command: start
+write_files:
+  - path: /data/BACKUP_DESTINATION
+    permissions: 0644
+    owner: root
+    content: |
+      core@backup.dev

deploy/deploy.sh

@@ -1,14 +1,10 @@
 #!/bin/sh
-if [ $# -ge 1 ]; then
-  SERVER=$1
-else
-  echo "Usage: sh ./deploy/deploy.sh server [folder [branch [user]]]"
-  exit 1
-fi
 if [ $# -ge 2 ]; then
-  FOLDER=$2
+  SERVER=$1
+  BACKUP_DEST=$2
 else
-  FOLDER="./data/"
+  echo "Usage: sh ./deploy/deploy.sh server backup_dest [branch [user]]]"
+  exit 1
 fi
 if [ $# -ge 3 ]; then
   BRANCH=$3
@@ -21,20 +17,18 @@ else
   USER="core"
 fi
 
-if [ -e "${FOLDER}runtime/haproxy/approved-certs/${SERVER}.pem" ]; then
+echo "Server to deploy is $SERVER"
-  DEFAULTSITE=$SERVER
+echo "Backups will live under $BACKUP_DEST"
-else
+echo "IndieHosters repo branch is $BRANCH"
-  echo "Please make sure ${FOLDER}runtime/haproxy/approved-certs/${SERVER}.pem exists, then retry"
-  exit 1
-fi
-
-echo "Hoster data folder is $FOLDER"
-echo "Infrastructure branch is $BRANCH"
 echo "Remote user is $USER"
-echo "Default site is $DEFAULTSITE"
 
-scp -r $FOLDER $USER@$SERVER:/data
 scp ./deploy/onServer.sh $USER@$SERVER:
+
 ssh $USER@$SERVER sudo mkdir -p /var/lib/coreos-install/
 scp cloud-config $USER@$SERVER:/var/lib/coreos-install/user_data
-ssh $USER@$SERVER sudo sh ./onServer.sh $BRANCH $DEFAULTSITE
+ssh $USER@$SERVER sudo sh ./onServer.sh $BRANCH $SERVER
+
+# overrides BACKUP_DESTINATION from cloud-config
+echo $BACKUP_DEST > ./deploy/tmp.txt
+scp ./deploy/tmp.txt $USER@SERVER:/data/BACKUP_DESTINATION
+rm ./deploy/tmp.txt

deploy/onServer.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+#Usage from deploy/deploy.sh:
+#ssh $USER@$SERVER sudo sh ./onServer.sh $BRANCH $SERVER
+
 echo Starting etcd:
 /usr/bin/coreos-cloudinit --from-file=/var/lib/coreos-install/user_data
 

doc/deploying-a-server.md

@@ -11,18 +11,12 @@ Make sure you read [getting started](getting-started-as-a-hoster.md) first.
 * Give the new server a name (in this example, we call the server 'k3')
 * Add k3 to your /etc/hosts with the right IP address
 * If you have used this name before, run `./deploy/forget-server-fingerprint.sh k3`
-* From the root folder of this repository, run `sh ./deploy/deploy.sh k3 ./data/ master root` (where `./data/` should contain
+* ssh into your server, and run `ssh-keygen -t rsa`  (use all the default settings, empty passphrase)
-  `runtime/postfix/`
-  and `runtime/haproxy/approved-certs/k3.pem`; see the existing folder `data/` in this repo for an example of what the email forwards and
-  TLS certificate files should look like).
-* Add the default site by following the 'Adding a website to your server' instructions below with domain name k3 instead of example.com
-* The rest should be automatic!
-
-### Preparing backups
-
-* ssh into your server, and run `ssh-keygen -t rsa`
 * set up a backups server at an independent location (at least a different data center, but preferably also a different IaaS provider, the bu25 plan of https://securedragon.net/ is a good option at 3 dollars per month).
-* set up a git server with one private git repo per domain by following http://www.git-scm.com/book/en/v2/Git-on-the-Server-Setting-Up-the-Server (instead of 'project.git' you can use 'domainname.com.git')
+* set up a git server by following http://www.git-scm.com/book/en/v2/Git-on-the-Server-Setting-Up-the-Server (no need to set up any repos like 'project.git' yet).  Let's call the backup server 'bu25' (add this to /etc/hosts on k3).
+* add the ssh key from k3 to the authorized_keys for the git user (not the root user) on bu25.
+* Exit from the double ssh back to your laptop, and from the root folder of this repository, run `sh ./deploy/deploy.sh k3 master root`
+* The rest should be automatic!
 
 ### Adding a website to your server
 * For each site you want to deploy on the server, e.g. example.com, do the following:
@@ -47,9 +41,5 @@ Make sure you read [getting started](getting-started-as-a-hoster.md) first.
   * Now run `deploy/add-site.sh k3 example.com ../hoster-data/TLS/example.com.pem nginx https://github.com/someone/example.com.git root`.
     It will make sure the server is in the correct state, and git pull and scp the user data and the
     approved cert into place, start a container running the image requested, update haproxy config, and restart the haproxy container.
-  * set up a git repo for the new site on the backup server (see http://www.git-scm.com/book/en/v2/Git-on-the-Server-Setting-Up-the-Server again), and for instance if you called the backup repo example.com.git and your backup server is in /etc/hosts on k3 as 'bu25', ssh into k3 and run:
-
-    sh scripts/backup-init.sh example.com git@bu25:/opt/git/example.com.git
-
   * Test the site using your /etc/hosts. You should see the data from the git repo on both http and https.
   * Switch DNS and monitoring.

importers/backup-snapshot.sh

@@ -1,48 +0,0 @@
-#!/bin/bash -eux
-
-if [ -e /data/domains/$DOMAIN/mysql ]; then
-  echo backing up mysql databases for $DOMAIN
-  mkdir -p /data/domains/$DOMAIN/backup/mysql/
-  cp /data/domains/$DOMAIN/mysql/.env /data/domains/$DOMAIN/backup/mysql/.env
-  /usr/bin/docker run --link mysql-$DOMAIN:db\
-     --env-file /data/domains/$DOMAIN/mysql/.env \
-     indiehosters/mysql mysqldump --all-databases --events -u admin \
-     -p$(cat /data/domains/$DOMAIN/mysql/.env | cut -d'=' -f2) \
-     -h db > /data/domains/$DOMAIN/backup/mysql/dump.sql
-fi
-
-if [ -e /data/domains/$DOMAIN/wordpress ]; then
-  echo backing up www from wordpress for $DOMAIN
-  mkdir -p /data/domains/$DOMAIN/backup/www/
-  rsync -r /data/domains/$DOMAIN/wordpress /data/domains/$DOMAIN/backup/www/wordpress
-fi
-
-if [ -e /data/domains/$DOMAIN/nginx ]; then
-  echo backing up www from nginx for $DOMAIN
-  mkdir -p /data/domains/$DOMAIN/backup/www/nginx/
-  if [ -e /data/domains/$DOMAIN/nginx/data/GITURL ]; then
-    cp /data/domains/$DOMAIN/nginx/data/GITURL /data/domains/$DOMAIN/backup/www/nginx/GITURL
-  else
-    rsync -r /data/domains/$DOMAIN/nginx/data/www-content /data/domains/$DOMAIN/backup/www/nginx/www-content
-  fi
-fi
-
-echo copying TLS cert
-mkdir -p /data/domains/$DOMAIN/backup/TLS/
-cp /data/runtime/haproxy/approved-certs/$DOMAIN.pem /data/domains/$DOMAIN/backup/TLS/$DOMAIN.pem
-
-echo committing everything
-cd /data/domains/$DOMAIN/backup/
-pwd
-git add *
-git status
-
-git config --local user.email "backups@`hostname`"
-git config --local user.name "`hostname` hourly backups"
-git config --local push.default simple
-
-git commit -m"backup $DOMAIN @ `hostname` - `date`"
-if [ -e /data/domains/$DOMAIN/backup/BACKUPDEST ]; then
-  git pull --rebase
-  git push
-fi

scripts/backup-init.sh

@@ -1,36 +1,20 @@
-#!/bin/sh
+#!/bin/bash -eux
-if [ $# -ge 2 ]; then
+
-  DOMAIN=$1
+BACKUP_DESTINATION=`cat /data/BACKUP_DESTINATION`
-  BACKUPDEST=$2
+
-else
+echo "Intitializing backups with $BACKUP_DESTINATION"
-  echo "Usage: sh ./scripts/backups-init.sh domain gitrepo"
+if [ ! -d /data/domains/$DOMAIN ]; then
-  exit 1
+  ssh $BACKUP_DESTINATION " \
+    if [ ! -d $DOMAIN ]; then \
+      mkdir -p $DOMAIN; \
+      cd $DOMAIN; \
+      git init --bare; \
+    else
+      echo \"Git folder already present\"
+    fi"
+  git clone $BACKUP_DESTINATION:$DOMAIN /data/domains/$DOMAIN
+  cd /data/domains/$DOMAIN
+  git config --local user.email "backups@`hostname`"
+  git config --local user.name "`hostname` hourly backups"
+  git config --local push.default simple
 fi
-echo "Adding backup job for $DOMAIN to $BACKUPDEST"
-
-echo "First, trying to clone latest master from $BACKUPDEST"
-git clone $BACKUPDEST /data/domains/$DOMAIN/backup
-
-sudo mkdir -p /data/domains/$DOMAIN/backup
-sudo echo "$BACKUPDEST" > /data/domains/$DOMAIN/backup/BACKUPDEST
-
-echo initializing backups for $DOMAIN
-mkdir -p /data/domains/$DOMAIN/backup/mysql
-mkdir -p /data/domains/$DOMAIN/backup/www
-mkdir -p /data/domains/$DOMAIN/backup/TLS
-cd /data/domains/$DOMAIN/backup/
-git config --local user.email "backups@`hostname`"
-git config --local user.name "`hostname` hourly backups"
-git config --local push.default simple
-
-if [ -e /data/domains/$DOMAIN/backup/.git ]; then
-  git init
-fi
-echo "backups of $DOMAIN at IndieHosters server `hostname`" > README.md
-git add README.md
-git commit -m"initial commit"
-
-echo "Pushing initial commit to $BACKUPDEST master branch"
-cd /data/domains/$DOMAIN/backup/
-git remote add destination $BACKUPDEST
-git push -u destination master

scripts/setup.sh

@@ -24,10 +24,6 @@ docker pull indiehosters/wordpress
 cp /data/indiehosters/unit-files/* /etc/systemd/system
 systemctl daemon-reload
 
-# Activate default domain
-sh /data/indiehosters/scripts/activate-user.sh $HOSTNAME static
-etcdctl set /services/default '{"app":"static", "hostname":"'$HOSTNAME'"}'
-
 # Configure and start HAproxy
 mkdir -p /data/runtime/haproxy/approved-certs
 systemctl enable haproxy-confd.service

unit-files/backup@.service

@@ -1,7 +1,31 @@
 [Unit]
-Description= Back up mysql and www data to a git repo and optionally push it out
+Description=Back up domain data to a git repo and push it out
 
 [Service]
 Type=oneshot
 Environment=DOMAIN=%i
-ExecStart=/data/indiehosters/importers/backup-snapshot.sh
+ExecStartPre=/bin/bash -euxc ' \
+  if [ -e /data/domains/$DOMAIN/mysql ]; then \
+    echo "Backing up mysql databases for $DOMAIN"; \
+    /usr/bin/docker run \
+      --link mysql-$DOMAIN:db \
+      --env-file /data/domains/$DOMAIN/mysql/.env \
+      indiehosters/mysql \
+        mysqldump \
+          --all-databases \
+          --events \
+          -u admin \
+          -p$(cat /data/domains/$DOMAIN/mysql/.env | cut -d'=' -f2) \
+          -h db \
+          > /data/domains/$DOMAIN/mysql/dump.sql; \
+  fi'
+
+ExecStart=/bin/bash -euxc ' \
+  echo "Committing everything"; \
+  cd /data/domains/$DOMAIN/; \
+  git add *; \
+  git status; \
+  git commit -m"backup $DOMAIN @ `hostname` - `date`"; \
+  # be careful: hidden sync functionnality; \
+  git pull --rebase; \
+  git push'

unit-files/static-git-importer@.service

@@ -7,6 +7,8 @@ Before=static-git@%i.service
 [Service]
 Type=oneshot
 RemainAfterExit=yes
+Environment=DOMAIN=%i
+ExecStartPre=/data/indiehosters/scripts/backup-init.sh
 ExecStart=/bin/bash -euxc ' \
   domain_folder=/data/domains/%i; \
   cp $domain_folder/TLS/%i.pem /data/runtime/haproxy/approved-certs/%i.pem; \

unit-files/static-importer@.service

@@ -7,6 +7,8 @@ Before=static@%i.service
 [Service]
 Type=oneshot
 RemainAfterExit=yes
+Environment=DOMAIN=%i
+ExecStartPre=/data/indiehosters/scripts/backup-init.sh
 ExecStart=/bin/bash -euxc ' \
   domain_folder=/data/domains/%i; \
   cp $domain_folder/TLS/%i.pem /data/runtime/haproxy/approved-certs/%i.pem; \

unit-files/wordpress-importer@.service

@@ -12,6 +12,7 @@ BindsTo=wordpress@%i.service
 Type=oneshot
 RemainAfterExit=yes
 Environment=DOMAIN=%i
+ExecStartPre=/data/indiehosters/scripts/backup-init.sh
 ExecStart=/bin/bash -euxc ' \
   domain_folder=/data/domains/%i; \
   cp $domain_folder/TLS/%i.pem /data/runtime/haproxy/approved-certs/%i.pem; \