From 68d5f9d19643a51d2f25294bd566f9b10ffa5f91 Mon Sep 17 00:00:00 2001 From: Justin Thomas Date: Sat, 29 Oct 2011 23:35:02 -0700 Subject: [PATCH 1/5] backup task handles postgresql dumps --- lib/tasks/backup.rake | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/lib/tasks/backup.rake b/lib/tasks/backup.rake index 785b625ac..c4995a2bd 100644 --- a/lib/tasks/backup.rake +++ b/lib/tasks/backup.rake @@ -3,9 +3,9 @@ namespace :backup do require File.join(Rails.root, 'config', 'initializers', '_load_app_config.rb') require 'cloudfiles' - task :mysql do + task :database do NUMBER_OF_DAYS = 3 - puts("event=backup status=start type=mysql") + puts("event=backup status=start type=database") db = YAML::load(File.open(File.join(File.dirname(__FILE__), '..','..', 'config', 'database.yml'))) user = db['production']['username'] password = db['production']['password'] @@ -20,30 +20,45 @@ namespace :backup do puts "Logging into Cloud Files" cf = CloudFiles::Connection.new(:username => AppConfig[:cloudfiles_username], :api_key => AppConfig[:cloudfiles_api_key]) - mysql_container = cf.container("MySQL Backup") - puts "Dumping Mysql at #{Time.now.to_s}" - `mkdir -p /tmp/backup/mysql` - `nice mysqldump --single-transaction --quick --user=#{user} --password=#{password} #{database} > /tmp/backup/mysql/backup.txt ` + if db['production']['adapter'] == 'postgresql' + container = cf.container("PostgreSQL Backup") - puts "Gzipping dump at #{Time.now.to_s}" - tar_name = "mysql_#{Time.now.to_i}.tar" - `nice tar cfPz /tmp/backup/#{tar_name} /tmp/backup/mysql` + puts "Dumping PostgreSQL at #{Time.now.to_s}" + `mkdir -p /tmp/backup/postgres` + `PGPASSFILE=/etc/pgpass.conf nice pg_dump -h localhost -p 5432 -U #{user} #{database} > /tmp/backup/postgres/backup.txt ` - file = mysql_container.create_object(tar_name) + puts "Gzipping dump at #{Time.now.to_s}" + tar_name = "postgres_#{Time.now.to_i}.tar" + `nice tar cfPz /tmp/backup/#{tar_name} /tmp/backup/postgres` + + file = container.create_object(tar_name) + elsif db['production']['adapter'] == 'mysql2' + container = cf.container("MySQL Backup") + + puts "Dumping Mysql at #{Time.now.to_s}" + `mkdir -p /tmp/backup/mysql` + `nice mysqldump --single-transaction --quick --user=#{user} --password=#{password} #{database} > /tmp/backup/mysql/backup.txt ` + + puts "Gzipping dump at #{Time.now.to_s}" + tar_name = "mysql_#{Time.now.to_i}.tar" + `nice tar cfPz /tmp/backup/#{tar_name} /tmp/backup/mysql` + + file = container.create_object(tar_name) + end puts "Uploading archive at #{Time.now.to_s}" if file.write File.open("/tmp/backup/" + tar_name) - puts("event=backup status=success type=mysql") + puts("event=backup status=success type=database") `rm /tmp/backup/#{tar_name}` - files = mysql_container.objects + files = container.objects files.sort!.pop(NUMBER_OF_DAYS * 24) files.each do |file| - mysql_container.delete_object(file) + container.delete_object(file) end else - puts("event=backup status=failure type=mysql") + puts("event=backup status=failure type=database") end end From e50f0cfdb2d27e231e3370a0808974f550ded1f4 Mon Sep 17 00:00:00 2001 From: Justin Thomas Date: Tue, 1 Nov 2011 15:24:04 -0700 Subject: [PATCH 2/5] removed dependency on /tmp, improved retention period compliance, maintain legacy commands --- config/application.yml.example | 3 + lib/tasks/backup.rake | 103 +++++++++++++++++++++++---------- 2 files changed, 76 insertions(+), 30 deletions(-) diff --git a/config/application.yml.example b/config/application.yml.example index c2817d5ce..cd726cf43 100644 --- a/config/application.yml.example +++ b/config/application.yml.example @@ -159,10 +159,13 @@ defaults: &defaults piwik_url: # Backups + backup_retention_days: 3 # cloudfiles username and api-key - used for backups cloudfiles_username: 'example' cloudfiles_api_key: 'abc123' + cloudfiles_db_container: 'Database Backup' + cloudfiles_images_container: 'Image Backup' # Donations diff --git a/lib/tasks/backup.rake b/lib/tasks/backup.rake index c4995a2bd..415adc5a2 100644 --- a/lib/tasks/backup.rake +++ b/lib/tasks/backup.rake @@ -1,19 +1,37 @@ namespace :backup do - desc "Backup Mysql" + desc "Backup Tasks" require File.join(Rails.root, 'config', 'initializers', '_load_app_config.rb') require 'cloudfiles' + require 'fileutils' + + tmp_backup_dir = "./tmp/backup" + tmp_sql_file = "backup.sql" + tmp_db_dir = "db" + + task :all => [:database, :photos] + + task :mysql => [:database] + + task :postgresql => [:database] task :database do - NUMBER_OF_DAYS = 3 puts("event=backup status=start type=database") + tmp_pass_file = ".pgpass.conf.tmp" db = YAML::load(File.open(File.join(File.dirname(__FILE__), '..','..', 'config', 'database.yml'))) + host = db['production']['host'] + port = db['production']['port'].to_s user = db['production']['username'] password = db['production']['password'] database = db['production']['database'] - unless AppConfig[:cloudfiles_username] && AppConfig[:cloudfiles_api_key] && !user.blank? + + unless AppConfig[:cloudfiles_username] && AppConfig[:cloudfiles_api_key] && AppConfig[:cloudfiles_db_container] && + AppConfig[:backup_retention_days] && !user.blank? + puts "Cloudfiles username needed" unless AppConfig[:cloudfiles_username] puts "Cloudfiles api_key needed" unless AppConfig[:cloudfiles_api_key] - puts "DB auth data needed" if user.blank? + puts "Cloudfiles database container needed" unless AppConfig[:cloudfiles_db_container] + puts "Retention period needed" unless AppConfig[:backup_retention_days] + puts "Database auth data needed" if user.blank? Process.exit end @@ -21,41 +39,54 @@ namespace :backup do cf = CloudFiles::Connection.new(:username => AppConfig[:cloudfiles_username], :api_key => AppConfig[:cloudfiles_api_key]) + FileUtils.mkdir_p(tmp_backup_dir + "/" + tmp_db_dir) + container = cf.container(AppConfig[:cloudfiles_db_container]) + if db['production']['adapter'] == 'postgresql' - container = cf.container("PostgreSQL Backup") + file = File.new(tmp_backup_dir + "/" + tmp_pass_file, 'w') + file.chmod( 0600 ) + file.write(host + ":" + + port + ":" + + database + ":" + + user + ":" + + password + "\n") + file.close puts "Dumping PostgreSQL at #{Time.now.to_s}" - `mkdir -p /tmp/backup/postgres` - `PGPASSFILE=/etc/pgpass.conf nice pg_dump -h localhost -p 5432 -U #{user} #{database} > /tmp/backup/postgres/backup.txt ` + `PGPASSFILE=#{tmp_backup_dir}/#{tmp_pass_file} nice pg_dump -h #{host} -p #{port} -U #{user} #{database} > #{tmp_backup_dir}/#{tmp_db_dir}/#{tmp_sql_file} ` puts "Gzipping dump at #{Time.now.to_s}" - tar_name = "postgres_#{Time.now.to_i}.tar" - `nice tar cfPz /tmp/backup/#{tar_name} /tmp/backup/postgres` - - file = container.create_object(tar_name) + tar_name = "postgresql_#{Time.now.to_i}.tar" + `nice tar cfPz #{tmp_backup_dir}/#{tar_name} #{tmp_backup_dir}/#{tmp_db_dir}` elsif db['production']['adapter'] == 'mysql2' - container = cf.container("MySQL Backup") - puts "Dumping Mysql at #{Time.now.to_s}" - `mkdir -p /tmp/backup/mysql` - `nice mysqldump --single-transaction --quick --user=#{user} --password=#{password} #{database} > /tmp/backup/mysql/backup.txt ` + `nice mysqldump --single-transaction --quick --user=#{user} --password=#{password} #{database} > #{tmp_backup_dir}/#{tmp_db_dir}/#{tmp_sql_file} ` puts "Gzipping dump at #{Time.now.to_s}" tar_name = "mysql_#{Time.now.to_i}.tar" - `nice tar cfPz /tmp/backup/#{tar_name} /tmp/backup/mysql` - - file = container.create_object(tar_name) + `nice tar cfPz /tmp/backup/#{tar_name} #{tmp_backup_dir}/#{tmp_db_dir}` end - puts "Uploading archive at #{Time.now.to_s}" - if file.write File.open("/tmp/backup/" + tar_name) - puts("event=backup status=success type=database") - `rm /tmp/backup/#{tar_name}` + file = container.create_object(tar_name) + puts "Uploading archive at #{Time.now.to_s}" + if file.write File.open(tmp_backup_dir + "/" + tar_name) + puts("event=backup status=success type=database") + + File.delete(tmp_backup_dir + "/" + tar_name) + File.delete(tmp_backup_dir + "/" + tmp_pass_file) + File.delete(tmp_backup_dir + "/" + tmp_db_dir + "/" + tmp_sql_file) + Dir.delete(tmp_backup_dir + "/" + tmp_db_dir) + Dir.delete(tmp_backup_dir) + + puts("Deleting Cloud Files objects that are older than specified retention period") files = container.objects - files.sort!.pop(NUMBER_OF_DAYS * 24) files.each do |file| - container.delete_object(file) + object = container.object(file) + if object.last_modified < (Time.now - (AppConfig[:backup_retention_days] * 24 * 60 * 60)) + puts("Deleting expired Cloud Files object: " + file) + container.delete_object(file) + end end else puts("event=backup status=failure type=database") @@ -65,19 +96,20 @@ namespace :backup do task :photos do puts("event=backup status=start type=photos") - if AppConfig[:cloudfiles_username] && AppConfig[:cloudfiles_api_key] + if AppConfig[:cloudfiles_username] && AppConfig[:cloudfiles_api_key] && AppConfig[:cloudfiles_images_container] puts "Logging into Cloud Files" cf = CloudFiles::Connection.new(:username => AppConfig[:cloudfiles_username], :api_key => AppConfig[:cloudfiles_api_key]) - photo_container = cf.container("Photo Backup") + container = cf.container(AppConfig[:cloudfiles_images_container]) + FileUtils.mkdir_p(tmp_backup_dir) tar_name = "photos_#{Time.now.to_i}.tar" - `tar cfPz /dev/stdout /usr/local/app/diaspora/public/uploads/images/ | split -d -b 4831838208 - /tmp/backup/#{tar_name}` + `tar cfPz /dev/stdout /usr/local/app/diaspora/public/uploads/images/ | split -d -b 4831838208 - #{tmp_backup_dir}/#{tar_name}` (0..99).each do |n| padded_str = n.to_s.rjust(2,'0') - file = photo_container.create_object(tar_name + padded_str) - file_path = "/tmp/backup/" + tar_name + padded_str + file = container.create_object(tar_name + padded_str) + file_path = tmp_backup_dir + "/" + tar_name + padded_str if File.exists?(file_path) if file.write File.open(file_path) @@ -85,10 +117,21 @@ namespace :backup do else puts("event=backup status=failure type=photos") end - `rm #{file_path}` + File.delete(file_path) end end + Dir.delete(tmp_backup_dir) + + puts("Deleting Cloud Files objects that are older than specified retention period") + files = container.objects + files.each do |file| + object = container.object(file) + if object.last_modified < (Time.now - (AppConfig[:backup_retention_days] * 24 * 60 * 60)) + puts("Deleting expired Cloud Files object: " + file) + container.delete_object(file) + end + end else puts "Cloudfiles username and api key needed" end From 221f58378d8621b47a04c7fd159a0bcab96d2f9c Mon Sep 17 00:00:00 2001 From: Justin Thomas Date: Tue, 1 Nov 2011 22:14:14 -0700 Subject: [PATCH 3/5] removed overlooked static path --- lib/tasks/backup.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/backup.rake b/lib/tasks/backup.rake index 415adc5a2..38aac535d 100644 --- a/lib/tasks/backup.rake +++ b/lib/tasks/backup.rake @@ -64,7 +64,7 @@ namespace :backup do puts "Gzipping dump at #{Time.now.to_s}" tar_name = "mysql_#{Time.now.to_i}.tar" - `nice tar cfPz /tmp/backup/#{tar_name} #{tmp_backup_dir}/#{tmp_db_dir}` + `nice tar cfPz #{tmp_backup_dir}/#{tar_name} #{tmp_backup_dir}/#{tmp_db_dir}` end file = container.create_object(tar_name) From 1b254ec2c57b7289e27bfb6aedc03ce0d06d404f Mon Sep 17 00:00:00 2001 From: Justin Thomas Date: Wed, 2 Nov 2011 11:59:17 -0700 Subject: [PATCH 4/5] removed legacy reference to a static path --- lib/tasks/backup.rake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/tasks/backup.rake b/lib/tasks/backup.rake index 38aac535d..c95776f86 100644 --- a/lib/tasks/backup.rake +++ b/lib/tasks/backup.rake @@ -102,9 +102,10 @@ namespace :backup do cf = CloudFiles::Connection.new(:username => AppConfig[:cloudfiles_username], :api_key => AppConfig[:cloudfiles_api_key]) container = cf.container(AppConfig[:cloudfiles_images_container]) + images_dir = File.join(File.dirname(__FILE__), '..', '..', 'public/uploads/images/') FileUtils.mkdir_p(tmp_backup_dir) tar_name = "photos_#{Time.now.to_i}.tar" - `tar cfPz /dev/stdout /usr/local/app/diaspora/public/uploads/images/ | split -d -b 4831838208 - #{tmp_backup_dir}/#{tar_name}` + `tar cfPz /dev/stdout #{images_dir} | split -d -b 4831838208 - #{tmp_backup_dir}/#{tar_name}` (0..99).each do |n| padded_str = n.to_s.rjust(2,'0') From 3e1561db24efd0580e963dffd1e0471bf5d32438 Mon Sep 17 00:00:00 2001 From: Justin Thomas Date: Tue, 29 Nov 2011 11:34:50 -0800 Subject: [PATCH 5/5] moved pass_file deletion to postgres code block --- lib/tasks/backup.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/backup.rake b/lib/tasks/backup.rake index c95776f86..80ffd29fb 100644 --- a/lib/tasks/backup.rake +++ b/lib/tasks/backup.rake @@ -54,6 +54,7 @@ namespace :backup do puts "Dumping PostgreSQL at #{Time.now.to_s}" `PGPASSFILE=#{tmp_backup_dir}/#{tmp_pass_file} nice pg_dump -h #{host} -p #{port} -U #{user} #{database} > #{tmp_backup_dir}/#{tmp_db_dir}/#{tmp_sql_file} ` + File.delete(tmp_backup_dir + "/" + tmp_pass_file) puts "Gzipping dump at #{Time.now.to_s}" tar_name = "postgresql_#{Time.now.to_i}.tar" @@ -74,7 +75,6 @@ namespace :backup do puts("event=backup status=success type=database") File.delete(tmp_backup_dir + "/" + tar_name) - File.delete(tmp_backup_dir + "/" + tmp_pass_file) File.delete(tmp_backup_dir + "/" + tmp_db_dir + "/" + tmp_sql_file) Dir.delete(tmp_backup_dir + "/" + tmp_db_dir) Dir.delete(tmp_backup_dir)