Merge pull request #1976 from jtimberman/refactor-chef

Refactor Chef recipes to utilize natural features of Chef
This commit is contained in:
Maxwell Salzberg 2011-09-13 23:40:19 -07:00
commit 88688d9bb6
46 changed files with 155 additions and 507 deletions

View file

@ -1 +0,0 @@
{"recipes": ["centos::bootstrap" ] }

View file

@ -1 +0,0 @@
{"recipes": ["centos::main" ] }

View file

@ -1,37 +0,0 @@
execute "bundler deps" do
command "yum install -y cpio"
end
execute "nokogiri deps" do
command "yum install -y libxml2-devel libxslt-devel"
end
execute "eventmachine deps" do
command "yum install -y gcc-c++"
end
execute "ssl lib" do
command "yum install -y openssl-devel"
end
execute "htop" do
command "yum install -y htop psmisc screen"
end
execute "rvm deps" do
command "yum install -y bzip2"
end
def harden_ruby(ruby_string)
Dir.glob("/usr/local/rvm/wrappers/#{ruby_string}/*").each do |file|
link "/usr/local/bin/#{file.split('/').last}" do
to file
end
end
Dir.glob("/usr/local/rvm/gems/#{ruby_string}/bin/*").each do |file|
link "/usr/local/bin/#{file.split('/').last}" do
to file
end
end
end
harden_ruby("ree-1.8.7-2010.02")
include_recipe "centos::post_bootstrap"

View file

@ -1,3 +0,0 @@
execute "install ImageMagick" do
command "yum install -y ImageMagick"
end

View file

@ -1,3 +0,0 @@
execute "JAVA!! for Jammit" do
command "yum install -y java"
end

View file

@ -1,6 +0,0 @@
include_recipe "centos::image_magick"
include_recipe "centos::mysql"
include_recipe "common::main"
include_recipe "centos::nginx"
include_recipe "centos::redis"
include_recipe "centos::curl"

View file

@ -1,11 +0,0 @@
execute "install mysql" do
command "yum install -y mysql mysql-server mysql-devel"
end
execute "start mysql service to create the system tables" do
command "service mysqld start"
end
execute "stop service again" do
command "service mysqld stop"
end

View file

@ -1,4 +0,0 @@
execute "pcre dependency" do
command "yum install -y pcre-devel"
end
include_recipe "common::nginx"

View file

@ -1 +0,0 @@
include_recipe "centos::java"

View file

@ -1,20 +0,0 @@
execute "refresh yum" do
command "yum update -y"
end
execute "install redis" do
command "yum install -y redis"
end
cookbook_file "/usr/local/etc/redis.conf" do
source "redis.conf"
end
execute "change redis.conf permissions" do
command "chmod 755 /usr/local/etc/redis.conf"
end
execute "make the redis db directory" do
command "mkdir -p /usr/local/var/db/redis"
end

View file

@ -1,7 +0,0 @@
execute "Remove Any Pervious Cronjob" do
command "crontab -r || true"
end
execute "Add the current cronjob" do
command "crontab /usr/local/app/diaspora/chef/cookbooks/common/files/default/backupcron.txt"
end

View file

@ -1,7 +0,0 @@
cookbook_file "/etc/sysconfig/iptables" do
source "iptables"
end
execute "restart iptables" do #TODO only do this if the file changes
command "/etc/init.d/iptables restart"
end

View file

@ -1,4 +0,0 @@
include_recipe "common::iptables"
include_recipe "common::daemontools"
include_recipe "common::splunk"
include_recipe "common::backup"

View file

@ -1 +0,0 @@
{"recipes": ["debian::bootstrap" ] }

View file

@ -1,312 +0,0 @@
# Redis configuration file example
# Note on units: when memory size is needed, it is possible to specifiy
# it in the usual form of 1k 5GB 4M and so forth:
#
# 1k => 1000 bytes
# 1kb => 1024 bytes
# 1m => 1000000 bytes
# 1mb => 1024*1024 bytes
# 1g => 1000000000 bytes
# 1gb => 1024*1024*1024 bytes
#
# units are case insensitive so 1GB 1Gb 1gB are all the same.
# By default Redis does not run as a daemon. Use 'yes' if you need it.
# Note that Redis will write a pid file in /usr/local/var/run/redis.pid when daemonized.
daemonize no
# When running daemonized, Redis writes a pid file in /usr/local/var/run/redis.pid by
# default. You can specify a custom pid file location here.
pidfile /usr/local/var/run/redis.pid
# Accept connections on the specified port, default is 6379
port 6379
# If you want you can bind a single interface, if the bind option is not
# specified all the interfaces will listen for incoming connections.
#
# bind 127.0.0.1
# Close the connection after a client is idle for N seconds (0 to disable)
timeout 300
# Set server verbosity to 'debug'
# it can be one of:
# debug (a lot of information, useful for development/testing)
# verbose (many rarely useful info, but not a mess like the debug level)
# notice (moderately verbose, what you want in production probably)
# warning (only very important / critical messages are logged)
loglevel verbose
# Specify the log file name. Also 'stdout' can be used to force
# Redis to log on the standard output. Note that if you use standard
# output for logging but daemonize, logs will be sent to /dev/null
logfile stdout
# Set the number of databases. The default database is DB 0, you can select
# a different one on a per-connection basis using SELECT <dbid> where
# dbid is a number between 0 and 'databases'-1
databases 16
################################ SNAPSHOTTING #################################
#
# Save the DB on disk:
#
# save <seconds> <changes>
#
# Will save the DB if both the given number of seconds and the given
# number of write operations against the DB occurred.
#
# In the example below the behaviour will be to save:
# after 900 sec (15 min) if at least 1 key changed
# after 300 sec (5 min) if at least 10 keys changed
# after 60 sec if at least 10000 keys changed
#
# Note: you can disable saving at all commenting all the "save" lines.
save 900 1
save 300 10
save 60 10000
# Compress string objects using LZF when dump .rdb databases?
# For default that's set to 'yes' as it's almost always a win.
# If you want to save some CPU in the saving child set it to 'no' but
# the dataset will likely be bigger if you have compressible values or keys.
rdbcompression yes
# The filename where to dump the DB
dbfilename dump.rdb
# The working directory.
#
# The DB will be written inside this directory, with the filename specified
# above using the 'dbfilename' configuration directive.
#
# Also the Append Only File will be created inside this directory.
#
# Note that you must specify a directory here, not a file name.
dir /usr/local/var/db/redis/
################################# REPLICATION #################################
# Master-Slave replication. Use slaveof to make a Redis instance a copy of
# another Redis server. Note that the configuration is local to the slave
# so for example it is possible to configure the slave to save the DB with a
# different interval, or to listen to another port, and so on.
#
# slaveof <masterip> <masterport>
# If the master is password protected (using the "requirepass" configuration
# directive below) it is possible to tell the slave to authenticate before
# starting the replication synchronization process, otherwise the master will
# refuse the slave request.
#
# masterauth <master-password>
################################## SECURITY ###################################
# Require clients to issue AUTH <PASSWORD> before processing any other
# commands. This might be useful in environments in which you do not trust
# others with access to the host running redis-server.
#
# This should stay commented out for backward compatibility and because most
# people do not need auth (e.g. they run their own servers).
#
# Warning: since Redis is pretty fast an outside user can try up to
# 150k passwords per second against a good box. This means that you should
# use a very strong password otherwise it will be very easy to break.
#
# requirepass foobared
################################### LIMITS ####################################
# Set the max number of connected clients at the same time. By default there
# is no limit, and it's up to the number of file descriptors the Redis process
# is able to open. The special value '0' means no limits.
# Once the limit is reached Redis will close all the new connections sending
# an error 'max number of clients reached'.
#
# maxclients 128
# Don't use more memory than the specified amount of bytes.
# When the memory limit is reached Redis will try to remove keys with an
# EXPIRE set. It will try to start freeing keys that are going to expire
# in little time and preserve keys with a longer time to live.
# Redis will also try to remove objects from free lists if possible.
#
# If all this fails, Redis will start to reply with errors to commands
# that will use more memory, like SET, LPUSH, and so on, and will continue
# to reply to most read-only commands like GET.
#
# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
# 'state' server or cache, not as a real DB. When Redis is used as a real
# database the memory usage will grow over the weeks, it will be obvious if
# it is going to use too much memory in the long run, and you'll have the time
# to upgrade. With maxmemory after the limit is reached you'll start to get
# errors for write operations, and this may even lead to DB inconsistency.
#
# maxmemory <bytes>
############################## APPEND ONLY MODE ###############################
# By default Redis asynchronously dumps the dataset on disk. If you can live
# with the idea that the latest records will be lost if something like a crash
# happens this is the preferred way to run Redis. If instead you care a lot
# about your data and don't want to that a single record can get lost you should
# enable the append only mode: when this mode is enabled Redis will append
# every write operation received in the file appendonly.aof. This file will
# be read on startup in order to rebuild the full dataset in memory.
#
# Note that you can have both the async dumps and the append only file if you
# like (you have to comment the "save" statements above to disable the dumps).
# Still if append only mode is enabled Redis will load the data from the
# log file at startup ignoring the dump.rdb file.
#
# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append
# log file in background when it gets too big.
appendonly no
# The name of the append only file (default: "appendonly.aof")
# appendfilename appendonly.aof
# The fsync() call tells the Operating System to actually write data on disk
# instead to wait for more data in the output buffer. Some OS will really flush
# data on disk, some other OS will just try to do it ASAP.
#
# Redis supports three different modes:
#
# no: don't fsync, just let the OS flush the data when it wants. Faster.
# always: fsync after every write to the append only log . Slow, Safest.
# everysec: fsync only if one second passed since the last fsync. Compromise.
#
# The default is "everysec" that's usually the right compromise between
# speed and data safety. It's up to you to understand if you can relax this to
# "no" that will will let the operating system flush the output buffer when
# it wants, for better performances (but if you can live with the idea of
# some data loss consider the default persistence mode that's snapshotting),
# or on the contrary, use "always" that's very slow but a bit safer than
# everysec.
#
# If unsure, use "everysec".
# appendfsync always
appendfsync everysec
# appendfsync no
################################ VIRTUAL MEMORY ###############################
# Virtual Memory allows Redis to work with datasets bigger than the actual
# amount of RAM needed to hold the whole dataset in memory.
# In order to do so very used keys are taken in memory while the other keys
# are swapped into a swap file, similarly to what operating systems do
# with memory pages.
#
# To enable VM just set 'vm-enabled' to yes, and set the following three
# VM parameters accordingly to your needs.
vm-enabled no
# vm-enabled yes
# This is the path of the Redis swap file. As you can guess, swap files
# can't be shared by different Redis instances, so make sure to use a swap
# file for every redis process you are running. Redis will complain if the
# swap file is already in use.
#
# The best kind of storage for the Redis swap file (that's accessed at random)
# is a Solid State Disk (SSD).
#
# *** WARNING *** if you are using a shared hosting the default of putting
# the swap file under /tmp is not secure. Create a dir with access granted
# only to Redis user and configure Redis to create the swap file there.
vm-swap-file /tmp/redis.swap
# vm-max-memory configures the VM to use at max the specified amount of
# RAM. Everything that deos not fit will be swapped on disk *if* possible, that
# is, if there is still enough contiguous space in the swap file.
#
# With vm-max-memory 0 the system will swap everything it can. Not a good
# default, just specify the max amount of RAM you can in bytes, but it's
# better to leave some margin. For instance specify an amount of RAM
# that's more or less between 60 and 80% of your free RAM.
vm-max-memory 0
# Redis swap files is split into pages. An object can be saved using multiple
# contiguous pages, but pages can't be shared between different objects.
# So if your page is too big, small objects swapped out on disk will waste
# a lot of space. If you page is too small, there is less space in the swap
# file (assuming you configured the same number of total swap file pages).
#
# If you use a lot of small objects, use a page size of 64 or 32 bytes.
# If you use a lot of big objects, use a bigger page size.
# If unsure, use the default :)
vm-page-size 32
# Number of total memory pages in the swap file.
# Given that the page table (a bitmap of free/used pages) is taken in memory,
# every 8 pages on disk will consume 1 byte of RAM.
#
# The total swap size is vm-page-size * vm-pages
#
# With the default of 32-bytes memory pages and 134217728 pages Redis will
# use a 4 GB swap file, that will use 16 MB of RAM for the page table.
#
# It's better to use the smallest acceptable value for your application,
# but the default is large in order to work in most conditions.
vm-pages 134217728
# Max number of VM I/O threads running at the same time.
# This threads are used to read/write data from/to swap file, since they
# also encode and decode objects from disk to memory or the reverse, a bigger
# number of threads can help with big objects even if they can't help with
# I/O itself as the physical device may not be able to couple with many
# reads/writes operations at the same time.
#
# The special value of 0 turn off threaded I/O and enables the blocking
# Virtual Memory implementation.
vm-max-threads 4
############################### ADVANCED CONFIG ###############################
# Glue small output buffers together in order to send small replies in a
# single TCP packet. Uses a bit more CPU but most of the times it is a win
# in terms of number of queries per second. Use 'yes' if unsure.
glueoutputbuf yes
# Hashes are encoded in a special way (much more memory efficient) when they
# have at max a given numer of elements, and the biggest element does not
# exceed a given threshold. You can configure this limits with the following
# configuration directives.
hash-max-zipmap-entries 64
hash-max-zipmap-value 512
# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
# order to help rehashing the main Redis hash table (the one mapping top-level
# keys to values). The hash table implementation redis uses (see dict.c)
# performs a lazy rehashing: the more operation you run into an hash table
# that is rhashing, the more rehashing "steps" are performed, so if the
# server is idle the rehashing is never complete and some more memory is used
# by the hash table.
#
# The default is to use this millisecond 10 times every second in order to
# active rehashing the main dictionaries, freeing memory when possible.
#
# If unsure:
# use "activerehashing no" if you have hard latency requirements and it is
# not a good thing in your environment that Redis can reply form time to time
# to queries with 2 milliseconds delay.
#
# use "activerehashing yes" if you don't have such hard requirements but
# want to free memory asap when possible.
activerehashing yes
################################## INCLUDES ###################################
# Include one or more other config files here. This is useful if you
# have a standard template that goes to all redis server but also need
# to customize a few per-server settings. Include files can include
# other files, so use this wisely.
#
# include /path/to/local.conf
# include /path/to/other.conf

View file

@ -1 +0,0 @@
{"recipes": ["debian::main" ] }

View file

@ -1,49 +0,0 @@
execute "bundler deps" do
command "apt-get install -y cpio"
end
execute "nokogiri deps" do
command "apt-get install -y libxml2-dev libxslt-dev"
end
execute "eventmachine deps" do
command "apt-get install -y gcc-c++"
end
execute "ssl lib" do
command "apt-get install -y libssl-dev libopenssl-ruby"
end
execute "curl" do
command "apt-get install -y libcurl4-openssl-dev"
end
execute "ffi" do
command "apt-get install -y libffi-ruby"
end
execute "htop" do
command "apt-get install -y htop psmisc screen"
end
execute "rvm deps" do
command "apt-get install -y bzip2"
end
def harden_ruby(ruby_string)
Dir.glob("/usr/local/rvm/wrappers/#{ruby_string}/*").each do |file|
link "/usr/local/bin/#{file.split('/').last}" do
to file
end
end
Dir.glob("/usr/local/rvm/gems/#{ruby_string}/bin/*").each do |file|
link "/usr/local/bin/#{file.split('/').last}" do
to file
end
end
end
harden_ruby("ree-1.8.7-2010.02")
include_recipe "debian::post_bootstrap"

View file

@ -1,3 +0,0 @@
execute "install ImageMagick" do
command "apt-get install -y imagemagick libmagick9-dev"
end

View file

@ -1,3 +0,0 @@
execute "JAVA!! for Jammit" do
command "apt-get install -y java"
end

View file

@ -1,5 +0,0 @@
include_recipe "debian::image_magick"
include_recipe "debian::mysql"
include_recipe "common::main"
include_recipe "debian::nginx"
include_recipe "debian::redis"

View file

@ -1,3 +0,0 @@
execute "install mysql" do
command "apt-get install -y mysql-server libmysqlclient-dev libmysql-ruby"
end

View file

@ -1,4 +0,0 @@
execute "pcre dependency" do
command "apt-get install -y libpcre3 libpcre3-dev"
end
include_recipe "common::nginx"

View file

@ -1 +0,0 @@
include_recipe "debian::java"

View file

@ -1,19 +0,0 @@
execute "download redis" do
command "wget http://ftp.us.debian.org/debian/pool/main/r/redis/redis-server_2.2.2-1_amd64.deb"
end
execute "install redis" do
command "dpkg -i redis-server_2.2.2-1_amd64.deb"
end
cookbook_file "/usr/local/etc/redis.conf" do
source "redis.conf"
end
execute "change redis.conf permissions" do
command "chmod 755 /usr/local/etc/redis.conf"
end
execute "make the redis db directory" do
command "mkdir -p /usr/local/var/db/redis"
end

View file

@ -0,0 +1 @@
{"run_list": ["recipe[diaspora::bootstrap"]}

View file

@ -0,0 +1 @@
{"run_list": ["recipe[diaspora::main]"]}

View file

@ -0,0 +1,16 @@
cron "user stats" do
minute 42
hour 15
command "cd /usr/local/app/diaspora && exec /usr/local/bin/ruby /usr/local/bin/bundle exec rake --trace statistics:users_splunk &> /usr/local/app/diaspora/log/stats.log"
end
cron "content stats" do
minute 42
hour 15
command "cd /usr/local/app/diaspora && exec /usr/local/bin/ruby /usr/local/bin/bundle exec rake --trace statistics:content_splunk &> /usr/local/app/diaspora/log/stats.log"
end
cron "backup mysql" do
minute 0
command "cd /usr/local/app/diaspora && exec /usr/local/bin/ruby /usr/local/bin/bundle exec rake --trace backup:mysql"
end

View file

@ -0,0 +1,56 @@
common_pkgs = [
"cpio",
"gcc-c++",
"htop",
"psmisc",
"screen",
"bzip2"
]
dev_pkgs = value_for_platform(
"debian" => {
"default" => [
"libxml2-dev",
"libxslt-dev",
"libssl-dev",
"libcurl4-openssl-dev"
]
},
"centos" => {
"default" => [
"libxml2-devel",
"libxslt-devel",
"openssl-devel",
]
}
)
execute "apt-get update" do
action :nothing
end.run_action(:run) if platform?("debian")
common_pkgs.each do |pkg|
package pkg
end
dev_pkgs do |pkg|
package pkg
end
def harden_ruby(ruby_string)
Dir.glob("/usr/local/rvm/wrappers/#{ruby_string}/*").each do |file|
link "/usr/local/bin/#{file.split('/').last}" do
to file
end
end
Dir.glob("/usr/local/rvm/gems/#{ruby_string}/bin/*").each do |file|
link "/usr/local/bin/#{file.split('/').last}" do
to file
end
end
end
harden_ruby("ree-1.8.7-2010.02")
include_recipe "diaspora::java"

View file

@ -48,4 +48,4 @@ execute 'rebundle' do
command 'bundle install'
end
include_recipe "centos::startcom_bundle"
include_recipe "diaspora::startcom_bundle"

View file

@ -9,6 +9,7 @@ end
execute "mysql run" do
command "mkdir -p /service/mysql && echo '#!/bin/sh' > /service/mysql/run && echo 'exec /usr/libexec/mysqld --datadir=/var/lib/mysql --socket=/var/lib/mysql/mysql.sock --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid --user=mysql' >> /service/mysql/run"
end
execute "executable" do
command "chmod -R 755 /service/mysql"
end

View file

@ -0,0 +1,12 @@
package "imagemagick" do
case node['platform']
when "debian"
package_name "imagemagick"
when "centos"
package_name "ImageMagick"
end
end
if platform?("debian")
package "libmagick9-dev"
end

View file

@ -0,0 +1,12 @@
if platform?("centos")
cookbook_file "/etc/sysconfig/iptables" do
source "iptables"
notifies :run, "execute[restart iptables]", :immediately
end
execute "restart iptables" do
command "/etc/init.d/iptables restart"
end
end

View file

@ -0,0 +1 @@
package "java"

View file

@ -0,0 +1,9 @@
include_recipe "diaspora::image_magick"
include_recipe "diaspora::mysql"
include_recipe "diaspora::iptables"
include_recipe "diaspora::daemontools"
include_recipe "diaspora::splunk"
include_recipe "diaspora::backup"
include_recipe "diaspora::nginx"
include_recipe "diaspora::redis"
include_recipe "diaspora::curl" if platform?("centos")

View file

@ -0,0 +1,16 @@
mysql_pkgs = value_for_platform(
"debian" => { "default" => %w[mysql-server libmysqlclient-dev libmysql-ruby] },
"centos" => { "default" => %w[mysql mysql-server mysql-devel] }
)
if platform?("centos")
execute "start mysql service to create the system tables" do
command "service mysqld start"
end
execute "stop service again" do
command "service mysqld stop"
end
end

View file

@ -0,0 +1,29 @@
case node['platform']
when "debian"
remote_file "#{Chef::Config[:file_cache_path]}/redis-server_2.2.2-1_amd64.deb" do
source "wget http://ftp.us.debian.org/debian/pool/main/r/redis/redis-server_2.2.2-1_amd64.deb"
end
dpkg_package "redis-server" do
source "#{Chef::Config[:file_cache_path]}/redis-server_2.2.2-1_amd64.deb"
end
when "centos"
execute "refresh yum" do
command "yum update -y"
end
package "redis"
end
cookbook_file "/usr/local/etc/redis.conf" do
source "redis.conf"
mode 0755
end
directory "/usr/local/var/db/redis" do
recursive true
end