├── .gitignore ├── LICENSE ├── README.md ├── Rakefile ├── bin ├── amy └── stem ├── chrysanthemum ├── config.ru ├── postgres-prototype │ ├── TODO │ ├── config.json │ ├── packet │ │ ├── pg_hba.conf │ │ ├── postgresql.conf-8.4.template │ │ ├── recovery.conf.erb │ │ └── s3cfg.template │ └── userdata.sh ├── postgres-server │ ├── config.json │ ├── config.json.mustache │ ├── userdata.sh │ ├── userdata.sh.mustache │ └── userdata.sh.yaml └── web.rb ├── examples └── lxc-server │ ├── lxc-server.json │ ├── packet │ └── etc │ │ ├── environment │ │ ├── fstab │ │ └── profile.d │ │ └── ruby.sh │ └── userdata.sh ├── lib ├── stem.rb └── stem │ ├── cli.rb │ ├── family.rb │ ├── group.rb │ ├── image.rb │ ├── instance.rb │ ├── instance_types.rb │ ├── ip.rb │ ├── key_pair.rb │ ├── tag.rb │ ├── userdata.rb │ └── util.rb ├── spec ├── family_spec.rb ├── fixtures │ ├── userdata │ │ └── userdata.sh │ └── vcr_cassettes │ │ ├── Stem_Family │ │ └── member_.yml │ │ ├── Stem_Image │ │ ├── describe.yml │ │ ├── describe_tagged.yml │ │ └── tagged.yml │ │ ├── family.yml │ │ └── image.yml ├── image_spec.rb ├── spec_helper.rb ├── userdata_spec.rb └── util_spec.rb └── stem.gemspec /.gitignore: -------------------------------------------------------------------------------- 1 | .rvmrc 2 | *.sw* 3 | */*.sw* 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010 Peter van Hardenberg 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #Stem 2 | ##EC2 made easy. 3 | 4 | ##Introduction 5 | 6 | Stem is a thin, light-weight EC2 instance management library which abstracts the Amazon EC2 API and provides an intuitive interface for designing, launching, and managing running instances. 7 | 8 | Stem is named after the model it encourages -- simple AMIs created on demand with many running copies derived from that. 9 | 10 | ##Configuration 11 | 12 | Stem relies on the Swirl library, which needs to be passed your AWS credentials to do its magic. There are two ways to set these. 13 | 14 | In your environment: 15 | 16 | export AWS_ACCESS_KEY_ID=my_access_key 17 | export AWS_SECRET_ACCESS_KEY=my_secret_key 18 | 19 | or in ~/.swirl: 20 | 21 | --- 22 | :default: 23 | :aws_access_key_id: my_access_key 24 | :aws_secret_access_key: my_secret_key 25 | 26 | ##Usage 27 | 28 | You can use Stem to manage your instances either from the commandline or directly via the library. You should create an instance which will serve as your "stem" and be converted into an AMI. Once you have tested this instance, create a snapshot of the instance, then use it by name to launch new instances with their own individual configuration. 29 | 30 | Here's a simple example from the command line. Begin by launching the example prototype instance. 31 | 32 | $ bin/stem launch chrysanthemum/postgres-prototype/config.json chrysanthemum/postgres-prototype/userdata.sh 33 | 34 | The config.json file specifies which AMI to start from, and what kind of EBS drive configuration to use. It is important that the drives are specified in the configuration file as any drives attached to the instance after launch will not become part of the eventual AMI you are creating 35 | 36 | You can monitor the instance's fabrication process via 37 | 38 | $ stem list 39 | 40 | The instance you created will boot, install some packages on top of a stock Ubuntu 10.4 AMI, then (if everything goes according to plan) shut itself down and go into a "stopped" state that indicates success. If any part of the stem fabrication fails, the instance will remain running. Once the instance reaches stopped, type 41 | 42 | $ stem create postgres-server appserver,current,tag_3 43 | 44 | The AMI may take as long as half an hour to build, depending on how the gremlins in EC2 are behaving on any given day. You can check on their progress with 45 | 46 | $ bin/amy 47 | 48 | If the AMI fabrication reaches the state "failed" you will have to manually reissue the `create` command and hope that the gremlins are more forgiving the second time around. 49 | 50 | Now that you have a simple postgres-server, you'll want to boot it up and create a database on it with some unique credentials! One of the simplest ways to solve this problem is to provide the instance with a templated userdata script which will perform per-instance configuration. I like mustache for this purpose. 51 | 52 | $ mustache userdata.sh.yaml postgres-server/userdata.sh.mustache > postgres-server/userdata.sh 53 | $ stem launch postgres-server/config.json postgres-server/userdata.sh 54 | 55 | You can, of course, delete the produced userdata.sh once the instance is launched. 56 | 57 | ##Inspiration and Thanks 58 | 59 | Stem is almost entirely based on Orion Henry's Judo gem, and Blake Mizerany's work on variously patton, carealot, napkin, and several other experiments. Thanks also for feedback, testing and patches from Adam Wiggins, Mark McGranahan, Noah Zoschke, Jason Dusek, and Blake Gentry. 60 | 61 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | 2 | $gemspec_file = nil 3 | $gemspec = nil 4 | 5 | task :gemspec_file do 6 | $gemspec_file = Dir['*.gemspec'].first unless $gemspec_file 7 | end 8 | 9 | task :gemspec do 10 | load $gemspec_file 11 | $gemspec = $spec 12 | end 13 | task :gemspec => :gemspec_file 14 | 15 | desc 'Create a gem for this project.' 16 | task :gem do 17 | system "gem build #{$gemspec_file}" 18 | end 19 | task :gem => :gemspec_file 20 | 21 | desc 'Install the gem for this project.' 22 | task :install => :gem do 23 | system "gem install #{$gemspec.name}-#{$gemspec.version}.gem" 24 | end 25 | task :install => [ :gem, :gemspec ] 26 | 27 | desc 'Remove the gem installed by this project.' 28 | task :uninstall do 29 | system "gem uninstall #{$gemspec.name} --version #{$gemspec.version}" 30 | end 31 | task :uninstall => :gemspec 32 | 33 | desc 'Delete gems and docs.' 34 | task :clean do 35 | system 'rm -f *.gem' 36 | system 'rm -rf doc' 37 | end 38 | 39 | -------------------------------------------------------------------------------- /bin/amy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'swirl' 3 | c = Swirl::EC2.new( 4 | :aws_access_key_id => ENV['AWS_ACCESS_KEY_ID'], 5 | :aws_secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'] 6 | ) 7 | 8 | images = c.call "DescribeImages", "Owner" => "self" 9 | puts images["imagesSet"].inject({}) { |them, img| them[ img["name"] ] = String.new(img["imageId"]); them }.to_yaml 10 | 11 | -------------------------------------------------------------------------------- /bin/stem: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $:.unshift File.dirname(__FILE__) + "/../lib" 3 | require 'stem' 4 | 5 | # Help is the default. 6 | ARGV << '-h' if ARGV.empty? && $stdin.tty? 7 | 8 | # Process options 9 | Stem::CLI.parse_options(ARGV) if $stdin.tty? 10 | 11 | # Still here - run the command 12 | Stem::CLI.dispatch_command(ARGV.shift, ARGV) 13 | 14 | -------------------------------------------------------------------------------- /chrysanthemum/config.ru: -------------------------------------------------------------------------------- 1 | require 'web' 2 | 3 | map "/heroku" do 4 | run Stem::HerokuApi 5 | end 6 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-prototype/TODO: -------------------------------------------------------------------------------- 1 | TODO 2 | ----- 3 | backups / restores not really thought through or tested yet 4 | 5 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-prototype/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "ami":"ami-714ba518", // public ubuntu 10.04 ami - 32 bit 3 | "availability_zone" : "us-east-1a", 4 | 5 | "volumes" : [ 6 | { "device" : "/dev/sde1", "size" : 4 }, 7 | { "device" : "/dev/sdf1", "size" : 8 }, 8 | { "device" : "/dev/sdf2", "size" : 8 }, 9 | { "device" : "/dev/sdf3", "size" : 8 }, 10 | { "device" : "/dev/sdf4", "size" : 8 }, 11 | { "device" : "/dev/sdf5", "size" : 8 }, 12 | { "device" : "/dev/sdf6", "size" : 8 }, 13 | { "device" : "/dev/sdf7", "size" : 8 }, 14 | { "device" : "/dev/sdf8", "size" : 8 } 15 | ] 16 | } 17 | 18 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-prototype/packet/pg_hba.conf: -------------------------------------------------------------------------------- 1 | local all postgres ident 2 | local all all ident 3 | host all all 0.0.0.0/0 md5 4 | host all all ::1/128 md5 5 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-prototype/packet/postgresql.conf-8.4.template: -------------------------------------------------------------------------------- 1 | # ----------------------------- 2 | # PostgreSQL configuration file 3 | # ----------------------------- 4 | # 5 | # This file consists of lines of the form: 6 | # 7 | # name = value 8 | # 9 | # (The "=" is optional.) Whitespace may be used. Comments are introduced with 10 | # "#" anywhere on a line. The complete list of parameter names and allowed 11 | # values can be found in the PostgreSQL documentation. 12 | # 13 | # The commented-out settings shown in this file represent the default values. 14 | # Re-commenting a setting is NOT sufficient to revert it to the default value; 15 | # you need to reload the server. 16 | # 17 | # This file is read on server startup and when the server receives a SIGHUP 18 | # signal. If you edit the file on a running system, you have to SIGHUP the 19 | # server for the changes to take effect, or use "pg_ctl reload". Some 20 | # parameters, which are marked below, require a server shutdown and restart to 21 | # take effect. 22 | # 23 | # Any parameter can also be given as a command-line option to the server, e.g., 24 | # "postgres -c log_connections=on". Some parameters can be changed at run time 25 | # with the "SET" SQL command. 26 | # 27 | # Memory units: kB = kilobytes Time units: ms = milliseconds 28 | # MB = megabytes s = seconds 29 | # GB = gigabytes min = minutes 30 | # h = hours 31 | # d = days 32 | 33 | 34 | #------------------------------------------------------------------------------ 35 | # FILE LOCATIONS 36 | #------------------------------------------------------------------------------ 37 | 38 | # The default values of these variables are driven from the -D command-line 39 | # option or PGDATA environment variable, represented here as ConfigDir. 40 | 41 | data_directory = '${DATA_DIR}' # use data in another directory 42 | # (change requires restart) 43 | hba_file = '/etc/postgresql/8.4/main/pg_hba.conf' # host-based authentication file 44 | # (change requires restart) 45 | ident_file = '/etc/postgresql/8.4/main/pg_ident.conf' # ident configuration file 46 | # (change requires restart) 47 | 48 | # If external_pid_file is not explicitly set, no extra PID file is written. 49 | external_pid_file = '/var/run/postgresql/8.4-main.pid' # write an extra PID file 50 | # (change requires restart) 51 | 52 | 53 | #------------------------------------------------------------------------------ 54 | # CONNECTIONS AND AUTHENTICATION 55 | #------------------------------------------------------------------------------ 56 | 57 | # - Connection Settings - 58 | 59 | listen_addresses = '*' # what IP address(es) to listen on; 60 | # comma-separated list of addresses; 61 | # defaults to 'localhost', '*' = all 62 | # (change requires restart) 63 | port = 5432 # (change requires restart) 64 | max_connections = 500 # (change requires restart) 65 | # Note: Increasing max_connections costs ~400 bytes of shared memory per 66 | # connection slot, plus lock space (see max_locks_per_transaction). 67 | #superuser_reserved_connections = 3 # (change requires restart) 68 | unix_socket_directory = '/var/run/postgresql' # (change requires restart) 69 | #unix_socket_group = '' # (change requires restart) 70 | #unix_socket_permissions = 0777 # begin with 0 to use octal notation 71 | # (change requires restart) 72 | #bonjour_name = '' # defaults to the computer name 73 | # (change requires restart) 74 | 75 | # - Security and Authentication - 76 | 77 | #authentication_timeout = 1min # 1s-600s 78 | ssl = true # (change requires restart) 79 | #ssl_ciphers = 'ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH' # allowed SSL ciphers 80 | # (change requires restart) 81 | #password_encryption = on 82 | #db_user_namespace = off 83 | 84 | # Kerberos and GSSAPI 85 | #krb_server_keyfile = '' 86 | #krb_srvname = 'postgres' # (Kerberos only) 87 | #krb_caseins_users = off 88 | 89 | # - TCP Keepalives - 90 | # see "man 7 tcp" for details 91 | 92 | #tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; 93 | # 0 selects the system default 94 | #tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; 95 | # 0 selects the system default 96 | #tcp_keepalives_count = 0 # TCP_KEEPCNT; 97 | # 0 selects the system default 98 | 99 | 100 | #------------------------------------------------------------------------------ 101 | # RESOURCE USAGE (except WAL) 102 | #------------------------------------------------------------------------------ 103 | 104 | # - Memory - 105 | 106 | shared_buffers = ${PG_SHARED_BUFFERS} # min 128kB 107 | # (change requires restart) 108 | #temp_buffers = 8MB # min 800kB 109 | max_prepared_transactions = 500 # zero disables the feature 110 | # (change requires restart) 111 | # Note: Increasing max_prepared_transactions costs ~600 bytes of shared memory 112 | # per transaction slot, plus lock space (see max_locks_per_transaction). 113 | # It is not advisable to set max_prepared_transactions nonzero unless you 114 | # actively intend to use prepared transactions. 115 | work_mem = 100MB # min 64kB 116 | maintenance_work_mem = 64MB # min 1MB 117 | #max_stack_depth = 2MB # min 100kB 118 | 119 | # - Kernel Resource Usage - 120 | 121 | #max_files_per_process = 1000 # min 25 122 | # (change requires restart) 123 | #shared_preload_libraries = '' # (change requires restart) 124 | 125 | # - Cost-Based Vacuum Delay - 126 | 127 | #vacuum_cost_delay = 0ms # 0-100 milliseconds 128 | #vacuum_cost_page_hit = 1 # 0-10000 credits 129 | #vacuum_cost_page_miss = 10 # 0-10000 credits 130 | #vacuum_cost_page_dirty = 20 # 0-10000 credits 131 | #vacuum_cost_limit = 200 # 1-10000 credits 132 | 133 | # - Background Writer - 134 | 135 | #bgwriter_delay = 200ms # 10-10000ms between rounds 136 | #bgwriter_lru_maxpages = 100 # 0-1000 max buffers written/round 137 | #bgwriter_lru_multiplier = 2.0 # 0-10.0 multipler on buffers scanned/round 138 | 139 | # - Asynchronous Behavior - 140 | 141 | #effective_io_concurrency = 1 # 1-1000. 0 disables prefetching 142 | 143 | 144 | #------------------------------------------------------------------------------ 145 | # WRITE AHEAD LOG 146 | #------------------------------------------------------------------------------ 147 | 148 | # - Settings - 149 | 150 | #fsync = on # turns forced synchronization on or off 151 | #synchronous_commit = on # immediate fsync at commit 152 | #wal_sync_method = fsync # the default is the first option 153 | # supported by the operating system: 154 | # open_datasync 155 | # fdatasync 156 | # fsync 157 | # fsync_writethrough 158 | # open_sync 159 | wal_buffers = 8MB # min 32kB 160 | # (change requires restart) 161 | #wal_writer_delay = 200ms # 1-10000 milliseconds 162 | 163 | #commit_delay = 0 # range 0-100000, in microseconds 164 | #commit_siblings = 5 # range 1-1000 165 | 166 | # - Checkpoints - 167 | 168 | checkpoint_segments = 40 # in logfile segments, min 1, 16MB each 169 | #checkpoint_timeout = 5min # range 30s-1h 170 | checkpoint_completion_target = 0.7 # checkpoint target duration, 0.0 - 1.0 171 | #checkpoint_warning = 30s # 0 disables 172 | 173 | # - Archiving - 174 | 175 | full_page_writes = off 176 | archive_mode = on 177 | archive_command = '${PG_ARCHIVE_COMMAND}' 178 | 179 | archive_timeout = 1min # force a logfile segment switch after this 180 | # number of seconds; 0 disables 181 | 182 | 183 | #------------------------------------------------------------------------------ 184 | # QUERY TUNING 185 | #------------------------------------------------------------------------------ 186 | 187 | # - Planner Method Configuration - 188 | 189 | #enable_bitmapscan = on 190 | #enable_hashagg = on 191 | #enable_hashjoin = on 192 | #enable_indexscan = on 193 | #enable_mergejoin = on 194 | #enable_nestloop = on 195 | #enable_seqscan = on 196 | #enable_sort = on 197 | #enable_tidscan = on 198 | 199 | # - Planner Cost Constants - 200 | 201 | #seq_page_cost = 1.0 # measured on an arbitrary scale 202 | #random_page_cost = 4.0 # same scale as above 203 | cpu_tuple_cost = 0.0030 # same scale as above 204 | cpu_index_tuple_cost = 0.001 # same scale as above 205 | cpu_operator_cost = 0.0005 # same scale as above 206 | effective_cache_size = ${PG_EFFECTIVE_CACHE_SIZE} 207 | 208 | # - Genetic Query Optimizer - 209 | 210 | #geqo = on 211 | #geqo_threshold = 12 212 | #geqo_effort = 5 # range 1-10 213 | #geqo_pool_size = 0 # selects default based on effort 214 | #geqo_generations = 0 # selects default based on effort 215 | #geqo_selection_bias = 2.0 # range 1.5-2.0 216 | 217 | # - Other Planner Options - 218 | 219 | #default_statistics_target = 100 # range 1-10000 220 | #constraint_exclusion = partition # on, off, or partition 221 | #cursor_tuple_fraction = 0.1 # range 0.0-1.0 222 | #from_collapse_limit = 8 223 | #join_collapse_limit = 8 # 1 disables collapsing of explicit 224 | # JOIN clauses 225 | 226 | 227 | #------------------------------------------------------------------------------ 228 | # ERROR REPORTING AND LOGGING 229 | #------------------------------------------------------------------------------ 230 | 231 | # - Where to Log - 232 | 233 | log_destination = 'stderr' # Valid values are combinations of 234 | # stderr, csvlog, syslog and eventlog, 235 | # depending on platform. csvlog 236 | # requires logging_collector to be on. 237 | 238 | # This is used when logging to stderr: 239 | logging_collector = on # Enable capturing of stderr and csvlog 240 | # into log files. Required to be on for 241 | # csvlogs. 242 | # (change requires restart) 243 | 244 | # These are only used if logging_collector is on: 245 | log_directory = 'pg_log' # directory where log files are written, 246 | # can be absolute or relative to PGDATA 247 | #log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, 248 | # can include strftime() escapes 249 | #log_truncate_on_rotation = off # If on, an existing log file of the 250 | # same name as the new log file will be 251 | # truncated rather than appended to. 252 | # But such truncation only occurs on 253 | # time-driven rotation, not on restarts 254 | # or size-driven rotation. Default is 255 | # off, meaning append to existing files 256 | # in all cases. 257 | #log_rotation_age = 1d # Automatic rotation of logfiles will 258 | # happen after that time. 0 disables. 259 | #log_rotation_size = 10MB # Automatic rotation of logfiles will 260 | # happen after that much log output. 261 | # 0 disables. 262 | 263 | # These are relevant when logging to syslog: 264 | #syslog_facility = 'LOCAL0' 265 | #syslog_ident = 'postgres' 266 | 267 | #silent_mode = off # Run server silently. 268 | # DO NOT USE without syslog or 269 | # logging_collector 270 | # (change requires restart) 271 | 272 | 273 | # - When to Log - 274 | 275 | client_min_messages = notice # values in order of decreasing detail: 276 | # debug5 277 | # debug4 278 | # debug3 279 | # debug2 280 | # debug1 281 | # log 282 | # notice 283 | # warning 284 | # error 285 | 286 | log_min_messages = notice # values in order of decreasing detail: 287 | # debug5 288 | # debug4 289 | # debug3 290 | # debug2 291 | # debug1 292 | # info 293 | # notice 294 | # warning 295 | # error 296 | # log 297 | # fatal 298 | # panic 299 | 300 | #log_error_verbosity = default # terse, default, or verbose messages 301 | 302 | #log_min_error_statement = error # values in order of decreasing detail: 303 | # debug5 304 | # debug4 305 | # debug3 306 | # debug2 307 | # debug1 308 | # info 309 | # notice 310 | # warning 311 | # error 312 | # log 313 | # fatal 314 | # panic (effectively off) 315 | 316 | #log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements 317 | # and their durations, > 0 logs only 318 | # statements running at least this number 319 | # of milliseconds 320 | 321 | 322 | # - What to Log - 323 | 324 | #debug_print_parse = off 325 | #debug_print_rewritten = off 326 | #debug_print_plan = off 327 | #debug_pretty_print = on 328 | log_checkpoints = on 329 | #log_connections = off 330 | #log_disconnections = off 331 | #log_duration = off 332 | #log_hostname = off 333 | log_line_prefix = '%t %d ' # special values: 334 | # %u = user name 335 | # %d = database name 336 | # %r = remote host and port 337 | # %h = remote host 338 | # %p = process ID 339 | # %t = timestamp without milliseconds 340 | # %m = timestamp with milliseconds 341 | # %i = command tag 342 | # %c = session ID 343 | # %l = session line number 344 | # %s = session start timestamp 345 | # %v = virtual transaction ID 346 | # %x = transaction ID (0 if none) 347 | # %q = stop here in non-session 348 | # processes 349 | # %% = '%' 350 | #log_lock_waits = off # log lock waits >= deadlock_timeout 351 | #log_statement = 'none' # none, ddl, mod, all 352 | #log_temp_files = -1 # log temporary files equal or larger 353 | # than the specified size in kilobytes; 354 | # -1 disables, 0 logs all temp files 355 | #log_timezone = unknown # actually, defaults to TZ environment 356 | # setting 357 | 358 | 359 | #------------------------------------------------------------------------------ 360 | # RUNTIME STATISTICS 361 | #------------------------------------------------------------------------------ 362 | 363 | # - Query/Index Statistics Collector - 364 | 365 | #track_activities = on 366 | #track_counts = on 367 | #track_functions = none # none, pl, all 368 | #track_activity_query_size = 1024 369 | #update_process_title = on 370 | #stats_temp_directory = 'pg_stat_tmp' 371 | 372 | 373 | # - Statistics Monitoring - 374 | 375 | #log_parser_stats = off 376 | #log_planner_stats = off 377 | #log_executor_stats = off 378 | #log_statement_stats = off 379 | 380 | 381 | #------------------------------------------------------------------------------ 382 | # AUTOVACUUM PARAMETERS 383 | #------------------------------------------------------------------------------ 384 | 385 | #autovacuum = on # Enable autovacuum subprocess? 'on' 386 | # requires track_counts to also be on. 387 | #log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and 388 | # their durations, > 0 logs only 389 | # actions running at least this number 390 | # of milliseconds. 391 | #autovacuum_max_workers = 3 # max number of autovacuum subprocesses 392 | #autovacuum_naptime = 1min # time between autovacuum runs 393 | #autovacuum_vacuum_threshold = 50 # min number of row updates before 394 | # vacuum 395 | #autovacuum_analyze_threshold = 50 # min number of row updates before 396 | # analyze 397 | #autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum 398 | #autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze 399 | #autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum 400 | # (change requires restart) 401 | #autovacuum_vacuum_cost_delay = 20ms # default vacuum cost delay for 402 | # autovacuum, in milliseconds; 403 | # -1 means use vacuum_cost_delay 404 | #autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for 405 | # autovacuum, -1 means use 406 | # vacuum_cost_limit 407 | 408 | 409 | #------------------------------------------------------------------------------ 410 | # CLIENT CONNECTION DEFAULTS 411 | #------------------------------------------------------------------------------ 412 | 413 | # - Statement Behavior - 414 | 415 | #search_path = '"$user",public' # schema names 416 | #default_tablespace = '' # a tablespace name, '' uses the default 417 | #temp_tablespaces = '' # a list of tablespace names, '' uses 418 | # only default tablespace 419 | #check_function_bodies = on 420 | #default_transaction_isolation = 'read committed' 421 | #default_transaction_read_only = off 422 | #session_replication_role = 'origin' 423 | #statement_timeout = 0 # in milliseconds, 0 is disabled 424 | #vacuum_freeze_min_age = 50000000 425 | #vacuum_freeze_table_age = 150000000 426 | #xmlbinary = 'base64' 427 | #xmloption = 'content' 428 | 429 | # - Locale and Formatting - 430 | 431 | datestyle = 'iso, mdy' 432 | #intervalstyle = 'postgres' 433 | #timezone = unknown # actually, defaults to TZ environment 434 | # setting 435 | #timezone_abbreviations = 'Default' # Select the set of available time zone 436 | # abbreviations. Currently, there are 437 | # Default 438 | # Australia 439 | # India 440 | # You can create your own file in 441 | # share/timezonesets/. 442 | #extra_float_digits = 0 # min -15, max 2 443 | #client_encoding = sql_ascii # actually, defaults to database 444 | # encoding 445 | 446 | # These settings are initialized by initdb, but they can be changed. 447 | lc_messages = 'en_US.UTF-8' # locale for system error message 448 | # strings 449 | lc_monetary = 'en_US.UTF-8' # locale for monetary formatting 450 | lc_numeric = 'en_US.UTF-8' # locale for number formatting 451 | lc_time = 'en_US.UTF-8' # locale for time formatting 452 | 453 | # default configuration for text search 454 | default_text_search_config = 'pg_catalog.english' 455 | 456 | # - Other Defaults - 457 | 458 | #dynamic_library_path = '$libdir' 459 | #local_preload_libraries = '' 460 | 461 | 462 | #------------------------------------------------------------------------------ 463 | # LOCK MANAGEMENT 464 | #------------------------------------------------------------------------------ 465 | 466 | #deadlock_timeout = 1s 467 | #max_locks_per_transaction = 64 # min 10 468 | # (change requires restart) 469 | # Note: Each lock table slot uses ~270 bytes of shared memory, and there are 470 | # max_locks_per_transaction * (max_connections + max_prepared_transactions) 471 | # lock table slots. 472 | 473 | 474 | #------------------------------------------------------------------------------ 475 | # VERSION/PLATFORM COMPATIBILITY 476 | #------------------------------------------------------------------------------ 477 | 478 | # - Previous PostgreSQL Versions - 479 | 480 | #add_missing_from = off 481 | #array_nulls = on 482 | #backslash_quote = safe_encoding # on, off, or safe_encoding 483 | #default_with_oids = off 484 | #escape_string_warning = on 485 | #regex_flavor = advanced # advanced, extended, or basic 486 | #sql_inheritance = on 487 | #standard_conforming_strings = off 488 | #synchronize_seqscans = on 489 | 490 | # - Other Platforms and Clients - 491 | 492 | #transform_null_equals = off 493 | 494 | 495 | #------------------------------------------------------------------------------ 496 | # CUSTOMIZED OPTIONS 497 | #------------------------------------------------------------------------------ 498 | 499 | #custom_variable_classes = '' # list of custom variable class names 500 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-prototype/packet/recovery.conf.erb: -------------------------------------------------------------------------------- 1 | restore_command = '<%= ENV["PG_RESTORE_COMMAND"] %>' 2 | <% if !(t = ENV["PG_RECOVER_AT"]).empty? %> 3 | recovery_target_time = '<%= t %>' 4 | <% end %> 5 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-prototype/packet/s3cfg.template: -------------------------------------------------------------------------------- 1 | [default] 2 | access_key = ${PG_AWS_ACCESS_KEY_ID} 3 | secret_key = ${PG_AWS_SECRET_KEY_ID} 4 | acl_public = False 5 | bucket_location = US 6 | cloudfront_host = cloudfront.amazonaws.com 7 | cloudfront_resource = /2008-06-30/distribution 8 | default_mime_type = binary/octet-stream 9 | delete_removed = False 10 | dry_run = False 11 | encoding = UTF-8 12 | encrypt = False 13 | force = False 14 | get_continue = False 15 | gpg_command = /usr/bin/gpg 16 | gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s 17 | gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s 18 | gpg_passphrase = ${PG_SECRET} 19 | guess_mime_type = True 20 | host_base = s3.amazonaws.com 21 | host_bucket = %(bucket)s.s3.amazonaws.com 22 | human_readable_sizes = False 23 | list_md5 = False 24 | preserve_attrs = True 25 | progress_meter = True 26 | proxy_host = 27 | proxy_port = 0 28 | recursive = False 29 | recv_chunk = 4096 30 | send_chunk = 4096 31 | simpledb_host = sdb.amazonaws.com 32 | skip_existing = False 33 | use_https = False 34 | verbosity = WARNING 35 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-prototype/userdata.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | # template() 7 | # 8 | # stone-stupid templating with only perl 9 | # replaces ${VARIABLE} with an environment variable value where available 10 | # 11 | 12 | function template() { 13 | perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' $1 14 | } 15 | 16 | function configure_drives() { 17 | echo "--- CONFIGURE DRIVES" 18 | 19 | apt-get install -y mdadm xfsprogs 20 | 21 | echo "--- configuring write ahead log drive" 22 | echo deadline > /sys/block/sde1/queue/scheduler 23 | mkfs.ext3 -q -L /wal /dev/sde1 24 | mkdir -p /wal && mount -o noatime -t ext3 /dev/sde1 /wal 25 | echo "/dev/sde1 /wal ext3 nodev,nosuid,noatime" >> /etc/fstab 26 | 27 | echo "--- creating raid for database cluster" 28 | for i in /sys/block/sdf{1..8}/queue/scheduler; do 29 | echo "deadline" > $i 30 | done 31 | # a bug with udev conflicts with mdadm 32 | service udev stop 33 | mdadm --create /dev/md0 -n 8 -l 0 -c 256 /dev/sdf{1..8} 34 | mdadm -Es >>/etc/mdadm/mdadm.conf 35 | service udev start 36 | echo "blockdev --setra 256 /dev/md0" >> /etc/rc.local 37 | modprobe dm-mod 38 | echo dm-mod >> /etc/modules 39 | pvcreate /dev/md0 40 | vgcreate vgdb /dev/md0 41 | lvcreate -n lvdb vgdb -l `/sbin/vgdisplay vgdb | grep Free | awk '{print $5}'` 42 | mkfs.xfs -q -L /database /dev/vgdb/lvdb 43 | echo "/dev/vgdb/lvdb /database xfs noatime,nosuid,nodev" >> /etc/fstab 44 | mkdir -p /database && mount /database 45 | } 46 | 47 | 48 | function userdata() { 49 | exec 2>&1 50 | 51 | echo "" > /etc/rc.local ## truncate this file - default is 'exit 0' which breaks append ops 52 | 53 | echo "--- BEGIN" 54 | export DEBIAN_FRONTEND="noninteractive" 55 | export DEBIAN_PRIORITY="critical" 56 | 57 | configure_drives 58 | 59 | echo "--- POSTGRESQL INSTALL" 60 | apt-get -y install thin postgresql-8.4 postgresql-server-dev-8.4 libpq-dev libgeos-dev proj 61 | service postgresql-8.4 stop 62 | 63 | echo "--- POSTGRESQL VARS" 64 | export DATA_DIR="/database" 65 | export WAL_DIR="/wal" 66 | export SYSTEM_MEMORY_KB=$(cat /proc/meminfo | awk '/MemTotal/ { print $2 }') 67 | export PG_EFFECTIVE_CACHE_SIZE=$(($SYSTEM_MEMORY_KB * 9 / 10 / 1024))MB 68 | export PG_SHARED_BUFFERS=$(($SYSTEM_MEMORY_KB / 4 / 1024))MB 69 | #export PG_ARCHIVE_COMMAND=XXX TODO 70 | 71 | echo "--- POSTGRESQL CONFIGURE" 72 | 73 | echo "--- templating and placing config files." 74 | 75 | # TODO maybe don't move this directory, but refer to the files directly 76 | mv dedicated/prototype/packet ~ 77 | cd ~ 78 | 79 | # setup firewall 80 | cp packet/iptables.rules /etc/iptables.rules 81 | chmod 600 /etc/iptables.rules 82 | echo '/sbin/iptables-restore /etc/iptables.rules' >> /etc/rc.local 83 | /sbin/iptables-restore /etc/iptables.rules 84 | useradd -m ingress 85 | mkdir ~ingress/.ssh 86 | cp packet/ingress.authorized ~ingress/.ssh/authorized_keys 87 | chown -R ingress:ingress ~ingress/.ssh 88 | chmod 700 ~ingress/.ssh 89 | chmod 600 ~ingress/.ssh/authorized_keys 90 | echo "ingress ALL=(ALL) NOPASSWD:/sbin/iptables -[AD] local --src [0-9.]* -p tcp --dport postgresql -m comment --comment [a-z0-9]* -j ACCEPT" >> /etc/sudoers 91 | 92 | cp packet/pg_hba.conf /etc/postgresql/9.0/main/ 93 | template packet/postgresql.conf-9.0.template > /etc/postgresql/9.0/main/postgresql.conf 94 | 95 | # shmmax -> one third of system RAM in bytes 96 | echo "kernel.shmmax=$((SYSTEM_MEMORY_KB * 1024 / 3))" >> /etc/sysctl.conf 97 | sysctl -p /etc/sysctl.conf 98 | 99 | template packet/s3cfg.template > /etc/s3cfg 100 | chown postgres:postgres /etc/s3cfg 101 | 102 | echo "--- creating database cluster" 103 | mkdir -p $DATA_DIR 104 | mkdir -p $WAL_DIR 105 | chown postgres:postgres $DATA_DIR $WAL_DIR 106 | chmod 700 $DATA_DIR $WAL_DIR 107 | su - postgres -c "/usr/lib/postgresql/8.4/bin/initdb -D $DATA_DIR" 108 | mv $DATA_DIR/pg_xlog/ $WAL_DIR 109 | ln -s $WAL_DIR/pg_xlog $DATA_DIR/pg_xlog 110 | ln -s /etc/ssl/certs/ssl-cert-snakeoil.pem $DATA_DIR/server.crt 111 | ln -s /etc/ssl/private/ssl-cert-snakeoil.key $DATA_DIR/server.key 112 | 113 | echo "--- starting postgres service" 114 | service postgresql-8.4 start 115 | 116 | echo "--- FINISHED, SHUTTING DOWN" 117 | shutdown -h now 118 | 119 | } 120 | 121 | userdata > /var/log/kuzushi.log 122 | 123 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-server/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "ami-name": "postgres-server" 3 | } 4 | 5 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-server/config.json.mustache: -------------------------------------------------------------------------------- 1 | // some options need to be over-written - instance size for instance 2 | // some on the otherhand need to merge - such as packages or users 3 | // how do we know what is what? - hard code for now 4 | { 5 | "ami":"ami-000be369", // public ubuntu 10.04 ami - 32 bit 6 | "security_group" : "pg", // (UNUSED) 7 | "user" : "ubuntu", // (UNUSED) 8 | 9 | "availability_zone" : "us-east-1a", 10 | "elastic_ip" : true // UNUSED 11 | } 12 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-server/userdata.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | export PG_ADMIN_PASSWORD=admin 7 | export PG_ROLE=role 8 | export PG_PASSWORD=password 9 | 10 | function userdata() { 11 | exec 2>&1 12 | 13 | echo "+++ POSTGRESQL CREATE" 14 | su - postgres -c "psql -c \"ALTER ROLE postgres WITH UNENCRYPTED PASSWORD '$PG_ADMIN_PASSWORD';\"" 15 | su - postgres -c "psql -c \"CREATE ROLE $PG_ROLE;\"" 16 | su - postgres -c "psql -c \"ALTER ROLE $PG_ROLE WITH LOGIN UNENCRYPTED PASSWORD '$PG_PASSWORD' NOSUPERUSER NOCREATEDB NOCREATEROLE;\"" 17 | su - postgres -c "psql -c \"CREATE DATABASE $PG_ROLE OWNER $PG_ROLE;\"" 18 | su - postgres -c "psql -c \"REVOKE ALL ON DATABASE $PG_ROLE FROM PUBLIC;\"" 19 | su - postgres -c "psql -c \"GRANT CONNECT ON DATABASE $PG_ROLE TO $PG_ROLE;\"" 20 | su - postgres -c "psql -c \"GRANT ALL ON DATABASE $PG_ROLE TO $PG_ROLE;\"" 21 | 22 | } 23 | 24 | userdata > /var/log/userdata.log 25 | 26 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-server/userdata.sh.mustache: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | export PG_ADMIN_PASSWORD={{{ pg_admin_password }}} 7 | export PG_ROLE={{{ db_role }}} 8 | export PG_PASSWORD={{{ db_password }}} 9 | 10 | function userdata() { 11 | exec 2>&1 12 | 13 | echo "+++ POSTGRESQL CREATE" 14 | su - postgres -c "psql -c \"ALTER ROLE postgres WITH UNENCRYPTED PASSWORD '$PG_ADMIN_PASSWORD';\"" 15 | su - postgres -c "psql -c \"CREATE ROLE $PG_ROLE;\"" 16 | su - postgres -c "psql -c \"ALTER ROLE $PG_ROLE WITH LOGIN UNENCRYPTED PASSWORD '$PG_PASSWORD' NOSUPERUSER NOCREATEDB NOCREATEROLE;\"" 17 | su - postgres -c "psql -c \"CREATE DATABASE $PG_ROLE OWNER $PG_ROLE;\"" 18 | su - postgres -c "psql -c \"REVOKE ALL ON DATABASE $PG_ROLE FROM PUBLIC;\"" 19 | su - postgres -c "psql -c \"GRANT CONNECT ON DATABASE $PG_ROLE TO $PG_ROLE;\"" 20 | su - postgres -c "psql -c \"GRANT ALL ON DATABASE $PG_ROLE TO $PG_ROLE;\"" 21 | 22 | } 23 | 24 | userdata > /var/log/userdata.log 25 | 26 | -------------------------------------------------------------------------------- /chrysanthemum/postgres-server/userdata.sh.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pg_admin_password: admin 3 | db_role: role 4 | db_password: password 5 | --- 6 | 7 | -------------------------------------------------------------------------------- /chrysanthemum/web.rb: -------------------------------------------------------------------------------- 1 | require 'sinatra' 2 | require 'json' 3 | require 'lib/stem' 4 | require 'mustache' 5 | 6 | module Stem 7 | class HerokuApi < Sinatra::Base 8 | post "/resources" do 9 | params = JSON.parse(request.body.string) 10 | id = params["heroku_id"] 11 | plan = params["plan"] 12 | callback_url = params["callback_url"] 13 | 14 | config = JSON.parse( File.read("postgres-server/config.json") ) 15 | template = File.read("postgres-server/userdata.sh.mustache") 16 | data = { 17 | "db_role" => (10 + rand(26)).to_s(36) + rand(2**64).to_s(36), 18 | "db_password" => rand(2**128).to_s(36), 19 | "pg_admin_password" => rand(2**128).to_s(36) 20 | } 21 | userdata = Mustache.render(template, data) 22 | 23 | puts "Allocating an IP" 24 | ip = Stem.allocate_ip 25 | instance = Stem.create(config, userdata) 26 | 27 | while true 28 | i = Stem.inspect(instance) 29 | break if i["instanceState"]["name"] == "running" 30 | sleep 1 31 | puts "Waiting... #{ i["instanceState"]["name"] }" 32 | end 33 | 34 | Stem.associate_ip(ip, instance) 35 | 36 | {:id => id, :config => {"STEM_URL" => 37 | "postgres://#{data["db_role"]}:#{data["db_password"]}#{ip}/#{data["db_role"]}"}}.to_json 38 | end 39 | 40 | delete "/resources/:name" do |name| 41 | @server = @sample.server(name) 42 | @sample.destroy_server_async(name) 43 | "ok" 44 | end 45 | 46 | end 47 | end 48 | 49 | -------------------------------------------------------------------------------- /examples/lxc-server/lxc-server.json: -------------------------------------------------------------------------------- 1 | { "ami" : "ami-1634de7f", 2 | "instance_type" : "m1.large", 3 | "key_name" : "ops_testing" 4 | } 5 | 6 | -------------------------------------------------------------------------------- /examples/lxc-server/packet/etc/environment: -------------------------------------------------------------------------------- 1 | PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" 2 | LANG=en_DK.UTF-8 3 | LC_ALL=en_DK.UTF-8 4 | 5 | -------------------------------------------------------------------------------- /examples/lxc-server/packet/etc/fstab: -------------------------------------------------------------------------------- 1 | # /etc/fstab: static file system information. 2 | # 3 | proc /proc proc nodev,noexec,nosuid 0 0 4 | /dev/sda1 / ext3 defaults,noatime 0 0 5 | /dev/sdb /tmp auto defaults,noatime 0 0 6 | /tmp /var/tmp bind defaults,bind 0 0 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /examples/lxc-server/packet/etc/profile.d/ruby.sh: -------------------------------------------------------------------------------- 1 | export PATH="$PATH:/var/lib/gems/1.8/bin" 2 | export RUBYOPT="-Ku -rubygems" 3 | 4 | -------------------------------------------------------------------------------- /examples/lxc-server/userdata.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note that `-o' means enable while `+o' means disable. 4 | set -o errexit 5 | set -o nounset 6 | set -o pipefail 7 | set -o functrace 8 | set -o errtrace 9 | set +o histexpand 10 | 11 | 12 | declare -ar packages=( lxc nmap git-core 13 | irb ruby rubygems1.8 14 | libopenssl-ruby libjson-ruby ) 15 | aptitude install --assume-yes "${packages[@]}" 16 | 17 | /usr/bin/updatedb 18 | 19 | locale-gen en_DK.UTF-8 20 | 21 | cat packet/etc/fstab > /etc/fstab 22 | cat packet/etc/environment > /etc/environment 23 | cat packet/etc/profile.d/ruby.sh > /etc/profile.d/ruby.sh 24 | 25 | -------------------------------------------------------------------------------- /lib/stem.rb: -------------------------------------------------------------------------------- 1 | $:.unshift File.dirname(__FILE__) 2 | 3 | require 'swirl/aws' 4 | require 'json' 5 | 6 | require 'stem/cli' 7 | require 'stem/util' 8 | require 'stem/group' 9 | require 'stem/userdata' 10 | require 'stem/instance' 11 | require 'stem/instance_types' 12 | require 'stem/image' 13 | require 'stem/ip' 14 | require 'stem/key_pair' 15 | require 'stem/tag' 16 | 17 | module Stem 18 | autoload :Family, 'stem/family' 19 | end 20 | -------------------------------------------------------------------------------- /lib/stem/cli.rb: -------------------------------------------------------------------------------- 1 | require 'optparse' 2 | 3 | require 'stem/userdata' 4 | 5 | module Stem 6 | Version = "0.2.1" 7 | 8 | module CLI 9 | extend self 10 | 11 | # Return a structure describing the options. 12 | def parse_options(args) 13 | opts = OptionParser.new do |opts| 14 | opts.banner = "Usage: stem COMMAND ..." 15 | 16 | opts.separator " " 17 | 18 | opts.separator "Examples:" 19 | opts.separator " $ stem launch prototype.config prototype-userdata.sh" 20 | opts.separator " $ stem launch examples/lxc-server/lxc-server.json examples/lxc-server/" 21 | opts.separator " $ stem list" 22 | opts.separator " $ stem create ami-name instance-id ami_tag1,ami_tag2" 23 | opts.separator " $ stem destroy ami-name" 24 | 25 | opts.separator " " 26 | opts.separator "Options:" 27 | 28 | opts.on("-v", "--version", "Print the version") do |v| 29 | puts "Stem v#{Stem::Version}" 30 | exit 31 | end 32 | 33 | opts.on_tail("-h", "--help", "Show this message") do 34 | puts opts 35 | exit 36 | end 37 | end 38 | 39 | opts.separator "" 40 | 41 | opts.parse!(args) 42 | end 43 | 44 | def dispatch_command command, arguments 45 | case command 46 | when "launch" 47 | launch(*arguments) 48 | when "create" 49 | create(*arguments) 50 | when "list" 51 | list(*arguments) 52 | when "describe" 53 | describe(*arguments) 54 | when "destroy" 55 | destroy(*arguments) 56 | when nil 57 | puts "Please provide a command." 58 | else 59 | puts "Command \"#{command}\" not recognized." 60 | end 61 | end 62 | 63 | def launch config_file = nil, userdata_file = nil 64 | abort "No config file" unless config_file 65 | userdata = case 66 | when userdata_file.nil? 67 | nil 68 | when File.directory?(userdata_file) 69 | Userdata.compile(userdata_file) 70 | when File.file?(userdata_file) 71 | File.new(userdata_file).read 72 | else 73 | abort 'Unable to interpret userdata object.' 74 | end 75 | conf = JSON.parse(File.new(config_file).read) 76 | instance = Stem::Instance.launch(conf, userdata) 77 | puts "New instance ID: #{instance}" 78 | end 79 | 80 | def create name = nil, instance = nil, tag_list = nil 81 | abort "Usage: create ami-name instance-to-capture ami_tag1,ami_tag2" unless name && instance 82 | tags = tag_list ? tag_list.split(',') : [] 83 | image_id = Stem::Image::create(name, instance, tags) 84 | puts "New image ID: #{image_id}" 85 | end 86 | 87 | def describe what 88 | require 'pp' 89 | if (what[0..2] == "ami") 90 | pp Stem::Image::describe(what) 91 | elsif 92 | pp Stem::Instance::describe(what) 93 | end 94 | end 95 | 96 | def destroy instance = nil 97 | abort "Usage: destroy instance-id" unless instance 98 | Stem::Instance::destroy(instance) 99 | end 100 | 101 | def list *arguments 102 | Stem::Instance::list(*arguments) 103 | end 104 | end 105 | end 106 | 107 | -------------------------------------------------------------------------------- /lib/stem/family.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module Family 3 | include Util 4 | extend self 5 | 6 | def ami_for(family, release, architecture = "x86_64") 7 | amis = Stem::Image::tagged(:family => family, 8 | :release => release, 9 | :architecture => architecture) 10 | throw "More than one AMI matched release." if amis.length > 1 11 | amis[0] 12 | end 13 | 14 | def unrelease family, release_name 15 | prev = Stem::Image::tagged(:family => family, :release => release_name) 16 | prev.each { |ami| Stem::Tag::destroy(ami, :release => release_name) } 17 | end 18 | 19 | def member? family, ami 20 | desc = Stem::Image::describe(ami) 21 | throw "AMI #{ami} does not exist" if desc.nil? 22 | tagset_to_hash(desc["tagSet"])["family"] == family 23 | end 24 | 25 | def members family, architecture = nil 26 | opts = architecture ? { "architecture" => architecture } : {} 27 | Stem::Image.tagged(opts.merge("family" => family)) 28 | end 29 | 30 | def describe_members family, architecture = nil 31 | opts = architecture ? { "architecture" => architecture } : {} 32 | Stem::Image.describe_tagged(opts.merge("family" => family)) 33 | end 34 | 35 | def release family, release_name, *amis 36 | amis.each do |ami| 37 | throw "#{ami} not part of #{family}" unless member?(family, ami) 38 | end 39 | unrelease family, release_name 40 | amis.each { |ami| Stem::Tag::create(ami, :release => release_name) } 41 | end 42 | 43 | def build_image(family, config, userdata) 44 | log = lambda { |msg| 45 | puts "[#{family}|#{Time.now.to_s}] #{msg}" 46 | } 47 | 48 | aggregate_hash_options_for_ami!(config) 49 | sha1 = image_hash(config, userdata) 50 | 51 | log.call "Beginning to build image for #{family}" 52 | log.call "Config:\n------\n#{ config.inspect }\n-------" 53 | instance_id = Stem::Instance.launch(config, userdata) 54 | 55 | log.call "Booting #{instance_id} to produce your prototype instance" 56 | wait_for_stopped instance_id, log 57 | 58 | timestamp = Time.now.utc.iso8601 59 | image_id = Stem::Image.create("#{family}-#{timestamp.gsub(':', '_')}", 60 | instance_id, 61 | { 62 | :created => timestamp, 63 | :family => family, 64 | :sha1 => sha1, 65 | :source_ami => config["ami"] 66 | }) 67 | log.call "Image ID is #{image_id}" 68 | 69 | wait_for_available(image_id, log) 70 | 71 | log.call "Terminating #{instance_id} now that the image is captured" 72 | Stem::Instance::destroy(instance_id) 73 | image_id 74 | end 75 | 76 | def image_already_built?(family, config, userdata) 77 | aggregate_hash_options_for_ami!(config) 78 | sha1 = image_hash(config, userdata) 79 | !Stem::Image.tagged(:family => family, :sha1 => sha1).empty? 80 | end 81 | 82 | def image_hash(config, userdata) 83 | Digest::SHA1::hexdigest([config.to_s, userdata].join(' ')) 84 | end 85 | 86 | protected 87 | 88 | def wait_for_stopped(instance_id, log) 89 | log.call "waiting for instance to reach state stopped -- " 90 | while sleep 10 91 | state = Stem::Instance.describe(instance_id)["instanceState"]["name"] 92 | log.call "instance #{instance_id} #{state}" 93 | break if state == "stopped" 94 | end 95 | end 96 | 97 | def wait_for_available(image_id, log) 98 | log.call "Waiting for image to finish capturing..." 99 | while sleep 10 100 | begin 101 | state = Stem::Image.describe(image_id)["imageState"] 102 | log.call "Image #{image_id} #{state}" 103 | case state 104 | when "available" 105 | log.call("Image capturing succeeded") 106 | break 107 | when "pending" #continue 108 | when "terminated" 109 | log "Image capture failed (#{image_id})" 110 | return false 111 | else throw "Image unexpectedly entered #{state}"; 112 | end 113 | rescue Swirl::InvalidRequest => e 114 | raise unless e.message =~ /does not exist/ 115 | end 116 | end 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /lib/stem/group.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module Group 3 | include Util 4 | extend self 5 | 6 | ## Example Rules 7 | 8 | ## icmp://1.2.3.4/32 9 | ## icmp://GroupName 10 | ## icmp://GroupName@UserId 11 | ## icmp://@UserId 12 | ## tcp://0.0.0.0/0:22 13 | ## tcp://0.0.0.0/0:22-23 14 | ## tcp://10.0.0.0/8: (this imples 0-65535 15 | ## udp://GroupName:4567 16 | ## udp://GroupName@UserID:4567-9999 17 | 18 | def get(name) 19 | swirl.call("DescribeSecurityGroups", "GroupName.1" => name)["securityGroupInfo"].first 20 | rescue Swirl::InvalidRequest => e 21 | raise e unless e.message =~ /The security group '\S+' does not exist/ 22 | nil 23 | end 24 | 25 | def create(name, rules = nil, description = "") 26 | create!(name, rules, description) 27 | true 28 | rescue Swirl::InvalidRequest => e 29 | raise e unless e.message =~ /The security group '\S+' already exists/ 30 | false 31 | end 32 | 33 | def create!(name, rules = nil, description = "") 34 | swirl.call "CreateSecurityGroup", "GroupName" => name, "GroupDescription" => description 35 | auth(name, rules) if rules 36 | end 37 | 38 | def destroy(name) 39 | destroy!(name) 40 | true 41 | rescue Swirl::InvalidRequest => e 42 | puts "===> #{e.class}" 43 | puts "===> #{e.message}" 44 | puts "#{e.backtrace.join("\n")}" 45 | false 46 | end 47 | 48 | def destroy!(name) 49 | swirl.call "DeleteSecurityGroup", "GroupName" => name 50 | end 51 | 52 | def auth(name, rules) 53 | index = 0 54 | args = rules.inject({"GroupName" => name}) do |i,rule| 55 | index += 1; 56 | rule_hash = gen_authorize(index, rule) 57 | i.merge(rule_hash) 58 | end 59 | swirl.call "AuthorizeSecurityGroupIngress", args 60 | end 61 | 62 | def revoke(name, rules) 63 | index = 0 64 | args = rules.inject({"GroupName" => name}) do |i,rule| 65 | index += 1; 66 | rule_hash = gen_authorize(index, rule) 67 | i.merge(rule_hash) 68 | end 69 | swirl.call "RevokeSecurityGroupIngress", args 70 | end 71 | 72 | def gen_authorize_target(index, target) 73 | if target =~ /^\d+\.\d+\.\d+.\d+\/\d+$/ 74 | { "IpPermissions.#{index}.IpRanges.1.CidrIp" => target } 75 | elsif target =~ /^(.+)@(\w+)$/ 76 | { "IpPermissions.#{index}.Groups.1.GroupName" => $1, 77 | "IpPermissions.#{index}.Groups.1.UserId" => $2 } 78 | elsif target =~ /^@(\w+)$/ 79 | { "IpPermissions.#{index}.Groups.1.UserId" => $1 } 80 | else 81 | { "IpPermissions.#{index}.Groups.1.GroupName" => target } 82 | end 83 | end 84 | 85 | def gen_authorize_ports(index, ports) 86 | if ports =~ /^(\d+)-(\d+)$/ 87 | { "IpPermissions.#{index}.FromPort" => $1, 88 | "IpPermissions.#{index}.ToPort" => $2 } 89 | elsif ports =~ /^(\d+)$/ 90 | { "IpPermissions.#{index}.FromPort" => $1, 91 | "IpPermissions.#{index}.ToPort" => $1 } 92 | elsif ports == "" 93 | { "IpPermissions.#{index}.FromPort" => "0", 94 | "IpPermissions.#{index}.ToPort" => "65535" } 95 | else 96 | raise "bad ports: #{rule}" 97 | end 98 | end 99 | 100 | def gen_authorize(index, rule) 101 | if rule =~ /icmp:\/\/(.+)/ 102 | { "IpPermissions.#{index}.IpProtocol" => "icmp", 103 | "IpPermissions.#{index}.FromPort" => "-1", 104 | "IpPermissions.#{index}.ToPort" => "-1" }.merge(gen_authorize_target(index,$1)) 105 | elsif rule =~ /(tcp|udp):\/\/(.*):(.*)/ 106 | { "IpPermissions.#{index}.IpProtocol" => $1 }.merge(gen_authorize_target(index,$2)).merge(gen_authorize_ports(index,$3)) 107 | else 108 | raise "bad rule: #{rule}" 109 | end 110 | end 111 | end 112 | end 113 | 114 | -------------------------------------------------------------------------------- /lib/stem/image.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module Image 3 | include Util 4 | extend self 5 | 6 | def create name, instance, tags = {} 7 | image_id = swirl.call("CreateImage", "Name" => name, "InstanceId" => instance)["imageId"] 8 | unless tags.empty? 9 | # We'll retry this once if necessary due to consistency issues on the AWS side 10 | i = 0 11 | begin 12 | Tag::create(image_id, tags) 13 | rescue Swirl::InvalidRequest => e 14 | if i < 5 && e.message =~ /does not exist/ 15 | i += 1 16 | retry 17 | end 18 | raise e 19 | end 20 | end 21 | image_id 22 | end 23 | 24 | def deregister image 25 | swirl.call("DeregisterImage", "ImageId" => image)["return"] 26 | end 27 | 28 | def named name 29 | opts = get_filter_opts("name" => name).merge("Owner" => "self") 30 | i = swirl.call("DescribeImages", opts)["imagesSet"].first 31 | i ? i["imageId"] : nil 32 | end 33 | 34 | def tagged tags 35 | return if tags.empty? 36 | opts = tags_to_filter(tags).merge("Owner" => "self") 37 | res = swirl.call("DescribeImages", opts)['imagesSet'] 38 | res ? res.map {|image| image['imageId'] } : [] 39 | end 40 | 41 | def describe_tagged tags 42 | opts = tags_to_filter(tags).merge("Owner" => "self") 43 | images = swirl.call("DescribeImages", opts)["imagesSet"] 44 | if images 45 | images.each {|image| image["tags"] = tagset_to_hash(image["tagSet"]) } 46 | images 47 | else 48 | [] 49 | end 50 | end 51 | 52 | def describe image 53 | swirl.call("DescribeImages", "ImageId" => image)["imagesSet"][0] 54 | rescue Swirl::InvalidRequest => e 55 | raise e unless e.message =~ /does not exist/ 56 | nil 57 | end 58 | end 59 | end 60 | 61 | -------------------------------------------------------------------------------- /lib/stem/instance.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module Instance 3 | include Util 4 | extend self 5 | 6 | def launch config, userdata = nil 7 | throw "No config provided" unless config 8 | config = aggregate_hash_options_for_ami!(config) 9 | ami = config["ami"] 10 | 11 | opt = { 12 | "SecurityGroup.#" => config["groups"] || [], 13 | "MinCount" => "1", 14 | "MaxCount" => "1", 15 | "KeyName" => config["key_name"] || "default", 16 | "InstanceType" => config["instance_type"] || "m1.small", 17 | "ImageId" => ami 18 | } 19 | 20 | opt.merge! "Placement.AvailabilityZone" => config["availability_zone"] if config["availability_zone"] 21 | 22 | if config["volumes"] 23 | devices = [] 24 | sizes = [] 25 | config["volumes"].each do |v| 26 | puts "Adding a volume of #{v["size"]} to be mounted at #{v["device"]}." 27 | devices << v["device"] 28 | sizes << v["size"].to_s 29 | end 30 | 31 | opt.merge! "BlockDeviceMapping.#.Ebs.VolumeSize" => sizes, 32 | "BlockDeviceMapping.#.DeviceName" => devices 33 | end 34 | 35 | if userdata 36 | puts "Userdata provided, encoded and sent to the instance." 37 | opt.merge!({ "UserData" => Base64.encode64(userdata)}) 38 | end 39 | 40 | response = swirl.call "RunInstances", opt 41 | instance_id = response["instancesSet"].first["instanceId"] 42 | 43 | if config['tags'] && !config['tags'].empty? 44 | i = 0 45 | begin 46 | Tag::create(instance_id, config['tags']) 47 | rescue Swirl::InvalidRequest => e 48 | if i < 5 && e.message =~ /does not exist/ 49 | i += 1 50 | retry 51 | end 52 | raise e 53 | end 54 | end 55 | instance_id 56 | end 57 | 58 | def restart instance_id 59 | swirl.call "RebootInstances", "InstanceId" => instance_id 60 | end 61 | 62 | def destroy instance_id 63 | swirl.call "TerminateInstances", "InstanceId" => instance_id 64 | end 65 | 66 | def stop instance_id, force = false 67 | swirl.call "StopInstances", "InstanceId" => instance_id, "Force" => force.to_s 68 | end 69 | 70 | def start instance_id 71 | swirl.call "StartInstances", "InstanceId" => instance_id 72 | end 73 | 74 | def describe instance 75 | throw "You must provide an instance ID to describe" unless instance 76 | swirl.call("DescribeInstances", "InstanceId" => instance)["reservationSet"][0]["instancesSet"][0] 77 | end 78 | 79 | def console_output instance 80 | swirl.call("GetConsoleOutput", "InstanceId" => instance) 81 | end 82 | 83 | def list 84 | instances = swirl.call("DescribeInstances") 85 | 86 | lookup = {} 87 | instances["reservationSet"].each do |r| 88 | r["instancesSet"].each do |i| 89 | lookup[i["imageId"]] = nil 90 | end 91 | end 92 | amis = swirl.call("DescribeImages", "ImageId" => lookup.keys)["imagesSet"] 93 | 94 | amis.each do |ami| 95 | name = ami["name"] || ami["imageId"] 96 | if !ami["description"] || ami["description"][0..1] != "%%" 97 | # only truncate ugly names from other people (never truncate ours) 98 | name.gsub!(/^(.{8}).+(.{8})/) { $1 + "..." + $2 } 99 | name = "(foreign) " + name 100 | end 101 | lookup[ami["imageId"]] = name 102 | end 103 | 104 | reservations = instances["reservationSet"] 105 | unless reservations.nil? or reservations.empty? 106 | puts "------------------------------------------" 107 | puts "Instances" 108 | puts "------------------------------------------" 109 | reservations.each do |r| 110 | groups = r["groupSet"].map { |g| g["groupId"] }.join(",") 111 | r["instancesSet"].each do |i| 112 | name = lookup[i["imageId"]] 113 | puts "%-15s %-15s %-15s %-20s %s" % [ i["instanceId"], i["ipAddress"] || "no ip", i["instanceState"]["name"], groups, name ] 114 | end 115 | end 116 | end 117 | 118 | result = swirl.call "DescribeImages", "Owner" => "self" 119 | images = result["imagesSet"].select { |img| img["name"] } 120 | unless images.nil? or images.empty? 121 | puts "------------------------------------------" 122 | puts "AMIs" 123 | puts "------------------------------------------" 124 | iwidth = images.map { |img| img["name"].length }.max + 1 125 | images.each do |img| 126 | puts "%-#{iwidth}s %s" % [ img["name"], img["imageId"] ] 127 | end 128 | end 129 | end 130 | 131 | def tagged *tags 132 | return if tags.empty? 133 | opts = { "tag-key" => tags.map {|t| t.to_s } } 134 | instances = swirl.call "DescribeInstances", get_filter_opts(opts) 135 | 136 | ids = [] 137 | instances["reservationSet"].each do |r| 138 | r["instancesSet"].each do |i| 139 | ids << i["instanceId"] 140 | end 141 | end 142 | ids 143 | end 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /lib/stem/instance_types.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module Instance 3 | Types = 4 | {"m1.small" => { 5 | "architecture" => "i386"}, 6 | "m1.large" => { 7 | "architecture" => "x86_64"}, 8 | "m1.xlarge" => { 9 | "architecture" => "x86_64"}, 10 | "t1.micro" => { 11 | "architecture" => "i386"}, 12 | "m2.xlarge" => { 13 | "architecture" => "x86_64"}, 14 | "m2.2xlarge" => { 15 | "architecture" => "x86_64"}, 16 | "m2.4xlarge" => { 17 | "architecture" => "x86_64"}, 18 | "c1.medium" => { 19 | "architecture" => "i386"}, 20 | "c1.xlarge" => { 21 | "architecture" => "x86_64"}, 22 | "cc1.4xlarge" => { 23 | "architecture" => "x86_64"}, 24 | "cg1.4xlarge" => { 25 | "architecture" => "x86_64"} 26 | } 27 | end 28 | end 29 | 30 | -------------------------------------------------------------------------------- /lib/stem/ip.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module Ip 3 | include Util 4 | 5 | extend self 6 | def allocate 7 | swirl.call("AllocateAddress")["publicIp"] 8 | end 9 | 10 | def associate ip, instance 11 | result = swirl.call("AssociateAddress", "InstanceId" => instance, "PublicIp" => ip)["return"] 12 | result == "true" 13 | end 14 | 15 | def disassociate ip 16 | result = swirl.call("DisassociateAddress", "PublicIp" => ip) 17 | end 18 | 19 | def release ip 20 | result = swirl.call("ReleaseAddress", "PublicIp" => ip) 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/stem/key_pair.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module KeyPair 3 | include Util 4 | extend self 5 | 6 | def create(name) 7 | swirl.call('CreateKeyPair', 'KeyName' => name) 8 | true 9 | rescue Swirl::InvalidRequest => e 10 | raise e unless e.message =~ /The keypair '\S+' already exists/ 11 | false 12 | end 13 | 14 | def destroy(name) 15 | destroy!(name) 16 | true 17 | rescue Swirl::InvalidRequest => e 18 | puts "===> #{e.class}" 19 | puts "===> #{e.message}" 20 | puts "#{e.backtrace.join("\n")}" 21 | false 22 | end 23 | 24 | def destroy!(name) 25 | swirl.call('DeleteKeyPair', 'KeyName' => name) 26 | end 27 | 28 | def describe(names) 29 | swirl.call('DescribeKeyPairs', 'KeyName' => names) 30 | end 31 | 32 | def exists?(name) 33 | true if describe(name) 34 | rescue Swirl::InvalidRequest => e 35 | raise e unless e.message.match(/does not exist$/) 36 | false 37 | end 38 | 39 | def import(name, key_string) 40 | require 'base64' 41 | swirl.call('ImportKeyPair', { 42 | 'KeyName' => name, 43 | 'PublicKeyMaterial' => Base64.encode64(key_string) 44 | }) 45 | true 46 | rescue Swirl::InvalidRequest => e 47 | raise e unless e.message =~ /The keypair '\S+' already exists/ 48 | false 49 | end 50 | 51 | end 52 | end -------------------------------------------------------------------------------- /lib/stem/tag.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module Tag 3 | include Util 4 | extend self 5 | 6 | def create resource_ids, tags 7 | resource_ids = [ resource_ids ] unless resource_ids.is_a? Array 8 | swirl.call("CreateTags", tag_opts(tags).merge("ResourceId" => resource_ids) ) 9 | end 10 | 11 | def destroy resource_ids, tags 12 | resource_ids = [ resource_ids ] unless resource_ids.is_a? Array 13 | swirl.call("DeleteTags", tag_opts(tags).merge("ResourceId" => resource_ids) ) 14 | end 15 | 16 | def tag_opts(tags) 17 | if tags.is_a? Hash 18 | { "Tag.#.Key" => tags.keys.map(&:to_s), 19 | "Tag.#.Value" => tags.values.map(&:to_s) } 20 | elsif tags.is_a? Array 21 | { 22 | "Tag.#.Key" => tags.map(&:to_s), 23 | "Tag.#.Value" => (1..tags.size).map { '' } 24 | } 25 | else 26 | { "Tag.1.Key" => tags.to_s, "Tag.1.Value" => '' } 27 | end 28 | end 29 | 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/stem/userdata.rb: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | require 'fileutils' 3 | 4 | module Stem 5 | module Userdata 6 | CREATE_ONLY = File::CREAT|File::EXCL|File::WRONLY 7 | 8 | extend self 9 | 10 | def compile(path, opts = {}) 11 | raise "No absolute paths please" if path.index("/") == 0 12 | raise "must be a directory" unless File.directory?(path) 13 | Dir.mktmpdir do |tmp_path| 14 | # trailing dot copies directory contents to match actual cp -r semantics... go figure 15 | FileUtils.cp_r("#{path}/.", tmp_path) 16 | Dir.chdir tmp_path do 17 | process_erb(opts[:erb_binding]) 18 | process_mustache(opts[:mustache_vars]) 19 | raise "must contain a userdata.sh" unless File.exists?("userdata.sh") 20 | # Set constant timestamps for all files to guarantee hashing consistency 21 | set_all_timestamps(tmp_path, "200210272016") 22 | make_zip_shell 23 | end 24 | end 25 | end 26 | 27 | # todo: make this & process_mustache both use binding 28 | def process_erb(binding) 29 | Dir["**/*.erb.stem"].each do |file| 30 | raise "must pass :erb_binding when using .erb.stem files" unless binding 31 | require 'erb' 32 | puts "erb ... #{file}" 33 | File.open(file.gsub(/.erb.stem$/,""), CREATE_ONLY) do |fff| 34 | fff.write ERB.new(File.read(file), 0, '<>').result(binding) 35 | end 36 | end 37 | end 38 | 39 | def process_mustache(vars) 40 | Dir["**/*.mustache.stem"].each do |file| 41 | raise "must pass :mustache_vars when using .mustache.stem files" unless vars 42 | require 'mustache' 43 | puts "mustache ... #{file}" 44 | File.open(file.gsub(/.mustache.stem$/,""), CREATE_ONLY) do |fff| 45 | fff.write Mustache.render(File.read(file), vars) 46 | end 47 | end 48 | end 49 | 50 | def make_zip_shell 51 | # We'll comment outside here, to keep from wasting valuable userdata bytes. 52 | # we decompress into /root/userdata, then run userdata.sh 53 | header = <<-SHELL 54 | #!/bin/bash 55 | exec >> /var/log/userdata.log 2>&1 56 | date --utc '+BOOTING %FT%TZ' 57 | UD=~/userdata 58 | mkdir -p $UD 59 | sed '1,/^#### THE END$/ d' "$0" | tar -jx -C $UD 60 | cd $UD 61 | exec bash userdata.sh 62 | #### THE END 63 | SHELL 64 | header + %x{tar --exclude \\*.stem -cv . | bzip2 --best -} 65 | end 66 | 67 | def set_all_timestamps(file_or_dir, time) 68 | Dir.foreach(file_or_dir) do |item| 69 | path = file_or_dir + '/' + item 70 | if File.directory?(path) && item != '.' 71 | next if item == '..' 72 | set_all_timestamps(path, time) 73 | else 74 | `touch -t #{time} #{path}` 75 | end 76 | end 77 | end 78 | 79 | end 80 | end 81 | 82 | -------------------------------------------------------------------------------- /lib/stem/util.rb: -------------------------------------------------------------------------------- 1 | module Stem 2 | module Util 3 | def swirl 4 | @swirl ||= Swirl::AWS.new :ec2, load_config 5 | end 6 | 7 | def tagset_to_hash(tagset) 8 | if tagset.is_a?(Hash) 9 | {tagset["item"]["key"] => tagset["item"]["value"]} 10 | else 11 | tagset.inject({}) do |h,item| 12 | k, v = item["key"], item["value"] 13 | h[k] = v 14 | h 15 | end 16 | end 17 | end 18 | 19 | def tags_to_filter(tags) 20 | if tags.is_a? Hash 21 | tags = tags.inject({}) do |h, (k, v)| 22 | # this is really awful. how can i make this non-awful? 23 | k = "tag:#{k}" unless k.to_s == "architecture" 24 | h[k.to_s] = v 25 | h 26 | end 27 | get_filter_opts(tags) 28 | elsif tags.is_a? Array 29 | get_filter_opts( { "tag-key" => tags.map(&:to_s) }) 30 | else 31 | get_filter_opts( { "tag-key" => [tags.to_s] }) 32 | end 33 | end 34 | 35 | def get_filter_opts(filters) 36 | opts = {} 37 | filters.keys.sort.each_with_index do |k, n| 38 | v = filters[k] 39 | opts["Filter.#{n}.Name"] = k.to_s 40 | v = [ v ] unless v.is_a? Array 41 | v.each_with_index do |v, i| 42 | opts["Filter.#{n}.Value.#{i}"] = v.to_s 43 | end 44 | end 45 | opts 46 | end 47 | 48 | protected 49 | 50 | def load_config 51 | account = "default" 52 | etc = "#{ENV["HOME"]}/.swirl" 53 | if ENV["AWS_ACCESS_KEY_ID"] && ENV["AWS_SECRET_ACCESS_KEY"] 54 | { 55 | :aws_access_key_id => ENV["AWS_ACCESS_KEY_ID"], 56 | :aws_secret_access_key => ENV["AWS_SECRET_ACCESS_KEY"], 57 | :version => "2010-08-31" 58 | } 59 | else 60 | account = account.to_sym 61 | 62 | if File.exists?(etc) 63 | data = YAML.load_file(etc) 64 | else 65 | abort("I was expecting to find a .swirl file in your home directory.") 66 | end 67 | 68 | if data.key?(account) 69 | data[account] 70 | else 71 | abort("I don't see the account you're looking for") 72 | end 73 | end 74 | end 75 | 76 | def aggregate_hash_options_for_ami!(config) 77 | if config["ami"] 78 | return config 79 | elsif config["ami-name"] 80 | name = config.delete("ami-name") 81 | config["ami"] = Image::named(name) 82 | throw "AMI named #{name} was not found. (Does it need creating?)" unless config["ami"] 83 | elsif config["ami-tags"] 84 | tags = config.delete('ami-tags') 85 | config["ami"] = Image::tagged(tags)[0] 86 | throw "AMI tagged with #{tags.inspect} was not found. (Does it need creating?)" unless config["ami"] 87 | else 88 | throw "No AMI specified." 89 | end 90 | config 91 | end 92 | 93 | end 94 | end 95 | 96 | -------------------------------------------------------------------------------- /spec/family_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Stem::Family do 4 | 5 | describe "ami_for" do 6 | it { should respond_to :ami_for } 7 | 8 | it "should return an AMI id when given the right input" do 9 | Stem::Image.should_receive(:tagged).with( 10 | :family => "postgres", 11 | :release => "production", 12 | :architecture => "x86_64" 13 | ).and_return(["ami-XXXXXX"]) 14 | Stem::Family.ami_for("postgres", "production").should == "ami-XXXXXX" 15 | end 16 | 17 | it "should allow you to specify i386 architecture" do 18 | Stem::Image.should_receive(:tagged).with( 19 | :family => "postgres", 20 | :release => "production", 21 | :architecture => "i386" 22 | ).and_return(["ami-XXXXXX"]) 23 | Stem::Family.ami_for("postgres", "production", "i386").should == "ami-XXXXXX" 24 | end 25 | 26 | it "should throw an error if there is more than one AMI matching a release" do 27 | Stem::Image.should_receive(:tagged).and_return(["ami-XXXXXX", "ami-BADBEEF"]) 28 | lambda { Stem::Family.ami_for("postgres", "production", "i386") }.should raise_error 29 | end 30 | end 31 | 32 | describe "unrelease" do 33 | it { should respond_to :unrelease} 34 | 35 | it "can unrelease nothing" do 36 | Stem::Image::should_receive(:tagged).and_return([]) 37 | Stem::Family::unrelease("postgres", "dummy") 38 | end 39 | 40 | it "should look up released images using tags" do 41 | Stem::Image.should_receive(:tagged).with({ 42 | :family => 'postgres', 43 | :release => 'production' 44 | }).and_return([]) 45 | Stem::Family.unrelease("postgres", "production") 46 | end 47 | 48 | it "can unrelease the previous release" do 49 | amis = ["ami-F00D", "ami-BEEF"] 50 | Stem::Image::should_receive(:tagged).and_return(amis) 51 | Stem::Tag::should_receive(:destroy).ordered.with(amis[0], :release => "production") 52 | Stem::Tag::should_receive(:destroy).ordered.with(amis[1], :release => "production") 53 | Stem::Family::unrelease("postgres", "production") 54 | end 55 | end 56 | 57 | describe "member?" do 58 | use_vcr_cassette 59 | 60 | it { should respond_to :member? } 61 | 62 | it "throws an exception for missing AMIs" do 63 | Stem::Image::should_receive(:describe).and_return(nil) 64 | lambda { Stem::Family::member?("postgres", "ami-BADAMI") }.should raise_error 65 | end 66 | 67 | it "returns true for AMIs in a family" do 68 | Stem::Family.member?("logplex", "ami-0286766b").should be_true 69 | end 70 | 71 | it "returns false for AMIs not in a family" do 72 | Stem::Family.member?("logplex", "ami-0686766f").should be_false 73 | end 74 | end 75 | 76 | describe "members" do 77 | use_vcr_cassette 78 | 79 | it { should respond_to :members } 80 | 81 | it "should call Stem::Image.tagged with the family tag" do 82 | f = "postgres" 83 | Stem::Image.should_receive(:tagged).with("family" => f).and_return([]) 84 | Stem::Family.members(f) 85 | end 86 | 87 | it "should return an empty array when no members exist" do 88 | Stem::Image.should_receive(:tagged).and_return([]) 89 | Stem::Family.members("postgres-protoss").should == [] 90 | end 91 | 92 | it "should return the AMI IDs when members exist" do 93 | amis = [ "ami-00000001", "ami-00000002" ] 94 | Stem::Image.should_receive(:tagged).and_return(amis) 95 | Stem::Family.members("postgres").should == amis 96 | end 97 | 98 | it "should accept an architecture argument" do 99 | f, arch = "postgres", "x86_64" 100 | Stem::Image.should_receive(:tagged).with({ 101 | "family" => f, 102 | "architecture" => arch 103 | }) 104 | Stem::Family.members(f, "x86_64") 105 | end 106 | end 107 | 108 | describe "describe_members" do 109 | use_vcr_cassette 110 | 111 | it { should respond_to :describe_members } 112 | 113 | it "should call Stem::Image.describe_tagged with the family tag" do 114 | f = "postgres" 115 | Stem::Image.should_receive(:describe_tagged).with("family" => f) 116 | Stem::Family.describe_members(f) 117 | end 118 | 119 | it "should accept an architecture argument" do 120 | f, arch = "postgres", "x86_64" 121 | Stem::Image.should_receive(:describe_tagged).with({ 122 | "family" => f, 123 | "architecture" => arch 124 | }) 125 | Stem::Family.describe_members(f, "x86_64") 126 | end 127 | end 128 | 129 | describe "release" do 130 | it { should respond_to :release } 131 | 132 | context "image exists" do 133 | before do 134 | Stem::Family.stub(:member?).and_return(true) 135 | Stem::Family.stub(:unrelease).and_return(true) 136 | end 137 | 138 | it "should unrelease the existing release prior to tagging a new one" do 139 | Stem::Family.should_receive(:unrelease).ordered 140 | Stem::Tag.should_receive(:create).ordered 141 | Stem::Family::release("postgres", "production", "ami-XXXXXX") 142 | end 143 | 144 | it "should tag a new release for 1 AMI" do 145 | Stem::Tag.should_receive(:create).with( 146 | "ami-XXXXXX", 147 | :release => 'production' 148 | ).and_return('true') 149 | Stem::Family::release("postgres", "production", "ami-XXXXXX") 150 | end 151 | 152 | it "should tag a new release for 2 AMIs" do 153 | amis = ["ami-XXXXX1", "ami-XXXXX2"] 154 | Stem::Tag.should_receive(:create).with( 155 | amis[0], 156 | :release => 'production' 157 | ).ordered.and_return('true') 158 | Stem::Tag.should_receive(:create).with( 159 | amis[1], 160 | :release => 'production' 161 | ).ordered.and_return('true') 162 | Stem::Family::release("postgres", "production", *amis) 163 | end 164 | 165 | it "should become the release for a version" 166 | end 167 | 168 | end 169 | 170 | describe "build_image" do 171 | it { should respond_to :build_image } 172 | end 173 | 174 | describe "image_hash" do 175 | it { should respond_to :image_hash } 176 | 177 | it "should return the SHA1 hash of the config.to_s and userdata joined by a space" do 178 | sha1 = Digest::SHA1.hexdigest("config userdata") 179 | Stem::Family.image_hash("config", "userdata").should == sha1 180 | end 181 | end 182 | 183 | describe "image_already_built?" do 184 | before do 185 | @family = "great_beers" 186 | @config = { 187 | "arch" => "32", 188 | "ami" => "ami-714ba518", 189 | "instance_type" => "c1.medium" 190 | } 191 | @userdata = "echo 'some useful script'" 192 | @sha1 ||= Stem::Family.image_hash(@config, @userdata) 193 | @ami_id = "ami-STOUT2" 194 | Stem::Image.stub!(:tagged).and_return([@ami_id]) 195 | end 196 | 197 | it { should respond_to :image_already_built? } 198 | 199 | it "should look for images tagged with the family and sha1" do 200 | Stem::Image.should_receive(:tagged).with({ 201 | :family => @family, 202 | :sha1 => @sha1 203 | }).and_return([]) 204 | Stem::Family.image_already_built?(@family, @config, @userdata) 205 | end 206 | 207 | it "should return true if an image with the given family and sha1 exists" do 208 | Stem::Image.stub!(:tagged).and_return([@ami_id]) 209 | Stem::Family.image_already_built?(@family, @config, @userdata).should == true 210 | end 211 | 212 | it "should return false if there's no image with the given family and sha1" do 213 | Stem::Image.stub!(:tagged).and_return([]) 214 | Stem::Family.image_already_built?(@family, @config, @userdata).should == false 215 | end 216 | end 217 | end 218 | 219 | # tags 220 | # :family => "postgresql", 221 | # :release => "production", 222 | # :created_at => "10/10/10 10:10", 223 | -------------------------------------------------------------------------------- /spec/fixtures/userdata/userdata.sh: -------------------------------------------------------------------------------- 1 | echo "Goliath online!" -------------------------------------------------------------------------------- /spec/fixtures/vcr_cassettes/Stem_Family/member_.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - !ruby/struct:VCR::HTTPInteraction 3 | request: !ruby/struct:VCR::Request 4 | method: :post 5 | uri: https://ec2.amazonaws.com:443/ 6 | body: AWSAccessKeyId=AKIAABCDEFGHIJKLMNOP&Action=DescribeImages&ImageId=ami-0286766b&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2002-10-28T04%3A16%3A00Z&Version=2010-08-31&Signature=fakesignature 7 | headers: 8 | content-type: 9 | - application/x-www-form-urlencoded 10 | response: !ruby/struct:VCR::Response 11 | status: !ruby/struct:VCR::ResponseStatus 12 | code: 200 13 | message: OK 14 | headers: 15 | content-type: 16 | - text/xml;charset=UTF-8 17 | date: 18 | - Mon, 14 Feb 2011 05:45:35 GMT 19 | server: 20 | - AmazonEC2 21 | transfer-encoding: 22 | - chunked 23 | body: |- 24 | 25 | 26 | 7f3c8bd5-13e0-440a-a726-d3288fcb12a2 27 | 28 | 29 | ami-0286766b 30 | 646412345678/logplex-bd003cb96bb122203db388192f65f2c2b7a4c61d 31 | available 32 | 646412345678 33 | false 34 | i386 35 | machine 36 | aki-754aa41c 37 | logplex-bd003cb96bb122203db388192f65f2c2b7a4c61d 38 | ebs 39 | /dev/sda1 40 | 41 | 42 | /dev/sda1 43 | 44 | snap-a7622dca 45 | 15 46 | true 47 | 48 | 49 | 50 | paravirtual 51 | 52 | 53 | family 54 | logplex 55 | 56 | 57 | 58 | 59 | 60 | http_version: "1.1" 61 | - !ruby/struct:VCR::HTTPInteraction 62 | request: !ruby/struct:VCR::Request 63 | method: :post 64 | uri: https://ec2.amazonaws.com:443/ 65 | body: AWSAccessKeyId=AKIAABCDEFGHIJKLMNOP&Action=DescribeImages&ImageId=ami-0686766f&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2002-10-28T04%3A16%3A00Z&Version=2010-08-31&Signature=fakesignature 66 | headers: 67 | content-type: 68 | - application/x-www-form-urlencoded 69 | response: !ruby/struct:VCR::Response 70 | status: !ruby/struct:VCR::ResponseStatus 71 | code: 200 72 | message: OK 73 | headers: 74 | content-type: 75 | - text/xml;charset=UTF-8 76 | server: 77 | - AmazonEC2 78 | date: 79 | - Mon, 14 Feb 2011 05:45:36 GMT 80 | transfer-encoding: 81 | - chunked 82 | body: |- 83 | 84 | 85 | 254ee6a5-083d-4e0e-8e0e-e78f3fb64c16 86 | 87 | 88 | ami-0686766f 89 | 646412345678/logplex-ec3e8c94ece2a26b46c948bf02c1b787b7a43569 90 | available 91 | 646412345678 92 | false 93 | x86_64 94 | machine 95 | aki-0b4aa462 96 | logplex-ec3e8c94ece2a26b46c948bf02c1b787b7a43569 97 | ebs 98 | /dev/sda1 99 | 100 | 101 | /dev/sda1 102 | 103 | snap-1b612e76 104 | 15 105 | true 106 | 107 | 108 | 109 | paravirtual 110 | 111 | 112 | family 113 | zergling 114 | 115 | 116 | 117 | 118 | 119 | http_version: "1.1" 120 | -------------------------------------------------------------------------------- /spec/fixtures/vcr_cassettes/Stem_Image/describe.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - !ruby/struct:VCR::HTTPInteraction 3 | request: !ruby/struct:VCR::Request 4 | method: :post 5 | uri: https://ec2.amazonaws.com:443/ 6 | body: AWSAccessKeyId=AKIAABCDEFGHIJKLMNOP&Action=DescribeImages&ImageId=ami-aaaaaaaa&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2002-10-28T04%3A16%3A00Z&Version=2010-08-31&Signature=fakesignature 7 | headers: 8 | content-type: 9 | - application/x-www-form-urlencoded 10 | response: !ruby/struct:VCR::Response 11 | status: !ruby/struct:VCR::ResponseStatus 12 | code: 400 13 | message: Bad Request 14 | headers: 15 | date: 16 | - Mon, 14 Feb 2011 18:40:18 GMT 17 | server: 18 | - AmazonEC2 19 | transfer-encoding: 20 | - chunked 21 | body: |- 22 | 23 | InvalidAMIID.NotFoundThe AMI ID 'ami-aaaaaaaa' does not existb53e9f3c-3a57-4cf8-bfbe-b5fd0de8c2d0 24 | http_version: "1.1" 25 | - !ruby/struct:VCR::HTTPInteraction 26 | request: !ruby/struct:VCR::Request 27 | method: :post 28 | uri: https://ec2.amazonaws.com:443/ 29 | body: AWSAccessKeyId=AKIAABCDEFGHIJKLMNOP&Action=DescribeImages&ImageId=ami-e67a8a8f&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2002-10-28T04%3A16%3A00Z&Version=2010-08-31&Signature=fakesignature 30 | headers: 31 | content-type: 32 | - application/x-www-form-urlencoded 33 | response: !ruby/struct:VCR::Response 34 | status: !ruby/struct:VCR::ResponseStatus 35 | code: 200 36 | message: OK 37 | headers: 38 | content-type: 39 | - text/xml;charset=UTF-8 40 | server: 41 | - AmazonEC2 42 | date: 43 | - Mon, 14 Feb 2011 18:40:18 GMT 44 | transfer-encoding: 45 | - chunked 46 | body: |- 47 | 48 | 49 | 10995ecc-010d-41a3-8d78-cf4ac9ed945d 50 | 51 | 52 | ami-e67a8a8f 53 | 646412345678/routing_transient_redis-57e6055179710692885fc12fa0025da14ca1ca04 54 | available 55 | 646412345678 56 | false 57 | x86_64 58 | machine 59 | aki-0b4aa462 60 | routing_transient_redis-57e6055179710692885fc12fa0025da14ca1ca04 61 | ebs 62 | /dev/sda1 63 | 64 | 65 | /dev/sda1 66 | 67 | snap-2b2a2946 68 | 15 69 | true 70 | 71 | 72 | 73 | paravirtual 74 | 75 | 76 | family 77 | postgres 78 | 79 | 80 | release 81 | production 82 | 83 | 84 | 85 | 86 | 87 | http_version: "1.1" 88 | -------------------------------------------------------------------------------- /spec/fixtures/vcr_cassettes/Stem_Image/describe_tagged.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - !ruby/struct:VCR::HTTPInteraction 3 | request: !ruby/struct:VCR::Request 4 | method: :post 5 | uri: https://ec2.amazonaws.com:443/ 6 | body: AWSAccessKeyId=AKIAABCDEFGHIJKLMNOP&Action=DescribeImages&Filter.0.Name=tag%3Afamily&Filter.0.Value.0=fake_family&Owner=self&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2002-10-28T04%3A16%3A00Z&Version=2010-08-31&Signature=fakesignature 7 | headers: 8 | content-type: 9 | - application/x-www-form-urlencoded 10 | response: !ruby/struct:VCR::Response 11 | status: !ruby/struct:VCR::ResponseStatus 12 | code: 200 13 | message: OK 14 | headers: 15 | content-type: 16 | - text/xml;charset=UTF-8 17 | date: 18 | - Mon, 14 Feb 2011 19:14:10 GMT 19 | server: 20 | - AmazonEC2 21 | transfer-encoding: 22 | - chunked 23 | body: |- 24 | 25 | 26 | f9e7b7b8-ffd2-4521-b91a-5e90cbefca38 27 | 28 | 29 | http_version: "1.1" 30 | - !ruby/struct:VCR::HTTPInteraction 31 | request: !ruby/struct:VCR::Request 32 | method: :post 33 | uri: https://ec2.amazonaws.com:443/ 34 | body: AWSAccessKeyId=AKIAABCDEFGHIJKLMNOP&Action=DescribeImages&Filter.0.Name=tag%3Afamily&Filter.0.Value.0=postgres&Owner=self&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2002-10-28T04%3A16%3A00Z&Version=2010-08-31&Signature=fakesignature 35 | headers: 36 | content-type: 37 | - application/x-www-form-urlencoded 38 | response: !ruby/struct:VCR::Response 39 | status: !ruby/struct:VCR::ResponseStatus 40 | code: 200 41 | message: OK 42 | headers: 43 | content-type: 44 | - text/xml;charset=UTF-8 45 | server: 46 | - AmazonEC2 47 | date: 48 | - Mon, 14 Feb 2011 19:14:10 GMT 49 | transfer-encoding: 50 | - chunked 51 | body: |- 52 | 53 | 54 | c23cf86c-eb5a-4281-bfa7-115d883669bd 55 | 56 | 57 | ami-e67a8a8f 58 | 646412345678/routing_transient_redis-57e6055179710692885fc12fa0025da14ca1ca04 59 | available 60 | 646412345678 61 | false 62 | x86_64 63 | machine 64 | aki-0b4aa462 65 | routing_transient_redis-57e6055179710692885fc12fa0025da14ca1ca04 66 | ebs 67 | /dev/sda1 68 | 69 | 70 | /dev/sda1 71 | 72 | snap-2b2a2946 73 | 15 74 | true 75 | 76 | 77 | 78 | paravirtual 79 | 80 | 81 | family 82 | postgres 83 | 84 | 85 | release 86 | production 87 | 88 | 89 | 90 | 91 | 92 | http_version: "1.1" 93 | -------------------------------------------------------------------------------- /spec/fixtures/vcr_cassettes/Stem_Image/tagged.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - !ruby/struct:VCR::HTTPInteraction 3 | request: !ruby/struct:VCR::Request 4 | method: :post 5 | uri: https://ec2.amazonaws.com:443/ 6 | body: AWSAccessKeyId=AKIAABCDEFGHIJKLMNOP&Action=DescribeImages&Filter.0.Name=tag%3Afaketag&Filter.0.Value.0=does_not_exist&Owner=self&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2002-10-28T04%3A16%3A00Z&Version=2010-08-31&Signature=fakesignature 7 | headers: 8 | content-type: 9 | - application/x-www-form-urlencoded 10 | response: !ruby/struct:VCR::Response 11 | status: !ruby/struct:VCR::ResponseStatus 12 | code: 200 13 | message: OK 14 | headers: 15 | content-type: 16 | - text/xml;charset=UTF-8 17 | date: 18 | - Mon, 14 Feb 2011 05:06:50 GMT 19 | server: 20 | - AmazonEC2 21 | transfer-encoding: 22 | - chunked 23 | body: |- 24 | 25 | 26 | d43ded72-3669-4a7f-b6a3-d48732ae3284 27 | 28 | 29 | http_version: "1.1" 30 | - !ruby/struct:VCR::HTTPInteraction 31 | request: !ruby/struct:VCR::Request 32 | method: :post 33 | uri: https://ec2.amazonaws.com:443/ 34 | body: AWSAccessKeyId=AKIAABCDEFGHIJKLMNOP&Action=DescribeImages&Filter.0.Name=architecture&Filter.0.Value.0=x86_64&Filter.1.Name=tag%3Aslot&Filter.1.Value.0=logplex&Owner=self&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2002-10-28T04%3A16%3A00Z&Version=2010-08-31&Signature=fakesignature 35 | headers: 36 | content-type: 37 | - application/x-www-form-urlencoded 38 | response: !ruby/struct:VCR::Response 39 | status: !ruby/struct:VCR::ResponseStatus 40 | code: 200 41 | message: OK 42 | headers: 43 | content-type: 44 | - text/xml;charset=UTF-8 45 | server: 46 | - AmazonEC2 47 | date: 48 | - Mon, 14 Feb 2011 05:06:51 GMT 49 | transfer-encoding: 50 | - chunked 51 | body: |- 52 | 53 | 54 | 24c2d8de-68d8-446b-bf47-dbada676c3e2 55 | 56 | 57 | ami-0686766f 58 | 646412345678/logplex-ec3e8c94ece2a26b46c948bf02c1b787b7a43569 59 | available 60 | 646412345678 61 | false 62 | x86_64 63 | machine 64 | aki-0b4aa462 65 | logplex-ec3e8c94ece2a26b46c948bf02c1b787b7a43569 66 | ebs 67 | /dev/sda1 68 | 69 | 70 | /dev/sda1 71 | 72 | snap-1b612e76 73 | 15 74 | true 75 | 76 | 77 | 78 | paravirtual 79 | 80 | 81 | slot 82 | logplex 83 | 84 | 85 | 86 | 87 | ami-2e06f647 88 | 646412345678/logplex-12350a613f9a51cd1bc1d0a6d49a7e66a078ec64 89 | available 90 | 646412345678 91 | false 92 | x86_64 93 | machine 94 | aki-0b4aa462 95 | logplex-12350a613f9a51cd1bc1d0a6d49a7e66a078ec64 96 | ebs 97 | /dev/sda1 98 | 99 | 100 | /dev/sda1 101 | 102 | snap-7b291316 103 | 15 104 | true 105 | 106 | 107 | 108 | paravirtual 109 | 110 | 111 | slot 112 | logplex 113 | 114 | 115 | 116 | 117 | 118 | http_version: "1.1" 119 | -------------------------------------------------------------------------------- /spec/fixtures/vcr_cassettes/family.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - !ruby/struct:VCR::HTTPInteraction 3 | request: !ruby/struct:VCR::Request 4 | method: :post 5 | uri: https://ec2.amazonaws.com:443/ 6 | body: AWSAccessKeyId=AKIAJCMZCYU5RS674NOQ&Action=DescribeImages&Filter.0.Name=architecture&Filter.0.Value.0=x86_64&Filter.1.Name=tag%3Afamily&Filter.1.Value.0=postgres&Filter.2.Name=tag%3Arelease&Filter.2.Value.0=production&Owner=self&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-02-11T01%3A45%3A28Z&Version=2010-08-31&Signature=idWIJU%2FV4kKZmGl%2BeAdR7zsrqaBXGmHIgJdDxZcNwYc%3D 7 | headers: 8 | content-type: 9 | - application/x-www-form-urlencoded 10 | response: !ruby/struct:VCR::Response 11 | status: !ruby/struct:VCR::ResponseStatus 12 | code: 200 13 | message: OK 14 | headers: 15 | content-type: 16 | - text/xml;charset=UTF-8 17 | date: 18 | - Fri, 11 Feb 2011 01:45:28 GMT 19 | server: 20 | - AmazonEC2 21 | transfer-encoding: 22 | - chunked 23 | body: |- 24 | 25 | 26 | 9a8ad5fb-9688-423e-99a2-08532e152ec6 27 | 28 | 29 | http_version: "1.1" 30 | - !ruby/struct:VCR::HTTPInteraction 31 | request: !ruby/struct:VCR::Request 32 | method: :post 33 | uri: https://ec2.amazonaws.com:443/ 34 | body: AWSAccessKeyId=AKIAJCMZCYU5RS674NOQ&Action=DescribeImages&Filter.0.Name=architecture&Filter.0.Value.0=x86_64&Filter.1.Name=tag%3Afamily&Filter.1.Value.0=postgres&Filter.2.Name=tag%3Arelease&Filter.2.Value.0=production&Owner=self&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-02-11T01%3A50%3A39Z&Version=2010-08-31&Signature=N5jmfUH2HdJs8EPs45UCTB9CSA2FTV5npAb%2B1GWO4wo%3D 35 | headers: 36 | content-type: 37 | - application/x-www-form-urlencoded 38 | response: !ruby/struct:VCR::Response 39 | status: !ruby/struct:VCR::ResponseStatus 40 | code: 200 41 | message: OK 42 | headers: 43 | content-type: 44 | - text/xml;charset=UTF-8 45 | server: 46 | - AmazonEC2 47 | date: 48 | - Fri, 11 Feb 2011 01:50:40 GMT 49 | transfer-encoding: 50 | - chunked 51 | body: |- 52 | 53 | 54 | 158e6b7f-4c7a-4f56-be86-c258ee517f16 55 | 56 | 57 | ami-e67a8a8f 58 | 646476705259/postgres-57e6055179710692885fc12fa0025da14ca1ca04 59 | available 60 | 646476705259 61 | false 62 | x86_64 63 | machine 64 | aki-0b4aa462 65 | postgres-57e6055179710692885fc12fa0025da14ca1ca04 66 | ebs 67 | /dev/sda1 68 | 69 | 70 | /dev/sda1 71 | 72 | snap-2b2a2946 73 | 15 74 | true 75 | 76 | 77 | 78 | paravirtual 79 | 80 | 81 | family 82 | postgres 83 | 84 | 85 | source 86 | ami-4b4ba522 87 | 88 | 89 | created_at 90 | 2011-02-04T23:00:58Z 91 | 92 | 93 | release 94 | production 95 | 96 | 97 | sha1 98 | 57e6055179710692885fc12fa0025da14ca1ca04 99 | 100 | 101 | 102 | 103 | 104 | http_version: "1.1" 105 | - !ruby/struct:VCR::HTTPInteraction 106 | request: !ruby/struct:VCR::Request 107 | method: :post 108 | uri: https://ec2.amazonaws.com:443/ 109 | body: AWSAccessKeyId=AKIAJCMZCYU5RS674NOQ&Action=DescribeImages&ImageId=ami-e67a8a8f&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-02-11T01%3A55%3A47Z&Version=2010-08-31&Signature=1WIK2FTDxrkitUxWvugVRMi3rgozNowL8ULNLrSGE94%3D 110 | headers: 111 | content-type: 112 | - application/x-www-form-urlencoded 113 | response: !ruby/struct:VCR::Response 114 | status: !ruby/struct:VCR::ResponseStatus 115 | code: 200 116 | message: OK 117 | headers: 118 | content-type: 119 | - text/xml;charset=UTF-8 120 | server: 121 | - AmazonEC2 122 | date: 123 | - Fri, 11 Feb 2011 01:55:48 GMT 124 | transfer-encoding: 125 | - chunked 126 | body: |- 127 | 128 | 129 | 06aff765-c53c-48f0-a458-f48aa6f84118 130 | 131 | 132 | ami-e67a8a8f 133 | 646476705259/routing_transient_redis-57e6055179710692885fc12fa0025da14ca1ca04 134 | available 135 | 646476705259 136 | false 137 | x86_64 138 | machine 139 | aki-0b4aa462 140 | routing_transient_redis-57e6055179710692885fc12fa0025da14ca1ca04 141 | ebs 142 | /dev/sda1 143 | 144 | 145 | /dev/sda1 146 | 147 | snap-2b2a2946 148 | 15 149 | true 150 | 151 | 152 | 153 | paravirtual 154 | 155 | 156 | os 157 | Ubuntu 10.04 158 | 159 | 160 | arch 161 | 64 162 | 163 | 164 | family 165 | postgres 166 | 167 | 168 | source 169 | ami-4b4ba522 170 | 171 | 172 | created_at 173 | 2011-02-04T23:00:58Z 174 | 175 | 176 | release 177 | production 178 | 179 | 180 | sha1 181 | 57e6055179710692885fc12fa0025da14ca1ca04 182 | 183 | 184 | 185 | 186 | 187 | http_version: "1.1" 188 | -------------------------------------------------------------------------------- /spec/fixtures/vcr_cassettes/image.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pvh/stem/f9f6e65caa6dabacf85d6a5356e2718b101d4824/spec/fixtures/vcr_cassettes/image.yml -------------------------------------------------------------------------------- /spec/image_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Stem::Image do 4 | 5 | context "create" do 6 | it { should respond_to :create } 7 | 8 | it "should raise an exception when an image with that name already exists" do 9 | image_name = "boring_old_ami" 10 | swirl.should_receive(:call) do 11 | raise Swirl::InvalidRequest.new("AMI name #{image_name} is already in use by AMI ami-12341234") 12 | end 13 | lambda { Stem::Image.create(image_name, "ami-12345678") }. 14 | should raise_exception 15 | end 16 | 17 | it "should call swirl with the correct arguments" do 18 | name, source_ami = "new_image", "ami-12345678" 19 | Stem::Image.swirl.should_receive(:call).at_least(:once). 20 | with("CreateImage", { 21 | "Name" => name, 22 | "InstanceId" => source_ami 23 | }).and_return("imageId" => nil) 24 | Stem::Image.create(name, source_ami) 25 | end 26 | 27 | it "should not tag the image when no tags are passed in" do 28 | Stem::Tag.should_not_receive(:create) 29 | swirl.should_receive(:call).and_return("imageId" => nil) 30 | Stem::Image.create("new_image", "ami-12345678") 31 | end 32 | 33 | it "should tag the image when tags are passed in" do 34 | ami_id, tags = "ami-12345678", {"family" => "postgres"} 35 | Stem::Tag.should_receive(:create).with(ami_id, tags) 36 | swirl.should_receive(:call).and_return("imageId" => ami_id) 37 | Stem::Image.create("new_image", ami_id, tags) 38 | end 39 | 40 | it "should return the ami id" do 41 | output_ami = "ami-22222222" 42 | swirl.should_receive(:call).and_return("imageId" => output_ami) 43 | Stem::Image.create("new_image", "ami-11111111").should == output_ami 44 | end 45 | end 46 | 47 | context "describe" do 48 | use_vcr_cassette 49 | 50 | it { should respond_to :describe } 51 | 52 | it "should return nil when the ami doesn't exist" do 53 | Stem::Image.describe("ami-aaaaaaaa").should be_nil 54 | end 55 | 56 | it "should return the AMI details hash when the ami exists" do 57 | h = Stem::Image.describe("ami-e67a8a8f") 58 | h["imageId"].should == "ami-e67a8a8f" 59 | end 60 | end 61 | 62 | context "tagged" do 63 | use_vcr_cassette 64 | 65 | it { should respond_to :tagged } 66 | 67 | it "should return nil if an empty hash is passed in" do 68 | Stem::Image.tagged([]).should be_nil 69 | end 70 | 71 | it "should return nil if an empty array is passed in" do 72 | Stem::Image.tagged([]).should be_nil 73 | end 74 | 75 | it "should return an empty array when no images exist with the specified tags" do 76 | Stem::Image.tagged(:faketag => 'does_not_exist').should == [] 77 | end 78 | 79 | it "should retun an array of ami IDs when 2 images exist with the specified tags" do 80 | Stem::Image.tagged(:slot => 'logplex', :architecture => 'x86_64') 81 | end 82 | end 83 | 84 | context "describe_tagged" do 85 | use_vcr_cassette 86 | 87 | it { should respond_to :describe_tagged } 88 | 89 | it "should convert the input tags to filters" do 90 | tags = { "family" => "postgres" } 91 | Stem::Image.should_receive(:tags_to_filter).with(tags).and_return({ 92 | "Filter.0.Name" => "tag:family", 93 | "Filter.0.Value.0" => "postgres" 94 | }) 95 | Stem::Image.swirl.stub!(:call).and_return("imagesSet" => []) 96 | Stem::Image.describe_tagged(tags) 97 | end 98 | 99 | it "should call swirl with the correct filters" do 100 | tags = { "family" => "postgres" } 101 | Stem::Image.swirl.should_receive(:call).with("DescribeImages", { 102 | "Owner" => "self", 103 | "Filter.0.Name" => "tag:family", 104 | "Filter.0.Value.0" => "postgres" 105 | }).and_return("imagesSet" => []) 106 | Stem::Image.describe_tagged(tags) 107 | end 108 | 109 | it "should return an empty array when the ami doesn't exist" do 110 | Stem::Image.describe_tagged("family" => "fake_family").should == [] 111 | end 112 | 113 | it "should return the AMI tags at the first level of the image hash" do 114 | images = Stem::Image.describe_tagged("family" => "postgres") 115 | images.first["tags"].should include("family" => "postgres") 116 | end 117 | end 118 | 119 | def swirl 120 | Stem::Image.swirl 121 | end 122 | end 123 | 124 | # tags 125 | # :family => "postgresql", 126 | # :release => "production", 127 | # :created_at => "10/10/10 10:10", 128 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'vcr' 2 | require 'timecop' 3 | require File.join(File.dirname(__FILE__), '../lib/stem') 4 | 5 | RSpec.configure do |c| 6 | c.extend VCR::RSpec::Macros 7 | end 8 | 9 | if ENV["VCR_RECORD"] 10 | puts "******** VCR RECORDING **********" 11 | 12 | VCR.config do |c| 13 | c.cassette_library_dir = 'spec/fixtures/vcr_cassettes' 14 | c.stub_with :webmock 15 | c.default_cassette_options = { :match_requests_on => [:uri, :method, :body], :record => :new_episodes } 16 | c.before_record do |i| 17 | if i.request.uri =~ /ec2\.amazonaws\.com/ 18 | vars_to_strip = { 19 | "AWSAccessKeyId" => "AKIAABCDEFGHIJKLMNOP", 20 | "Signature" => "fakesignature", 21 | "Timestamp" => "2002-10-28T04%3A16%3A00Z" 22 | } 23 | vars_to_strip.each do |k,v| 24 | i.request.body.gsub!(/(#{k}=[^&$]+)(&|$)/, "#{k}=#{v}\\2") 25 | end 26 | end 27 | end 28 | end 29 | else 30 | puts "******** VCR PLAYBACK **********" 31 | 32 | RSpec.configure do |config| 33 | config.before(:each) do 34 | Timecop.freeze(Time.parse("2002-10-28T04:16:00Z")) 35 | end 36 | 37 | config.after(:each) do 38 | Timecop.return 39 | end 40 | end 41 | 42 | VCR.config do |c| 43 | c.cassette_library_dir = 'spec/fixtures/vcr_cassettes' 44 | c.stub_with :webmock 45 | c.default_cassette_options = { :match_requests_on => [:uri, :method, :body], :record => :none } 46 | end 47 | 48 | # Stub out Stem config loading to use constant values 49 | Stem::Util.class_eval do 50 | private 51 | def load_config 52 | { 53 | :aws_access_key_id => "AKIAABCDEFGHIJKLMNOP", 54 | :aws_secret_access_key => "secret_access_key", 55 | :version => "2010-08-31" 56 | } 57 | end 58 | end 59 | 60 | # Stub out signature generation 61 | Swirl::AWS.class_eval do 62 | def compile_signature(method, body) 63 | "fakesignature" 64 | end 65 | end 66 | 67 | end 68 | -------------------------------------------------------------------------------- /spec/userdata_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | 3 | describe Stem::Userdata do 4 | describe '#compile' do 5 | it "should produce identical output with identical input" do 6 | output1 = Stem::Userdata.compile("spec/fixtures/userdata", :erb_binding => binding) 7 | sleep(1) # Necessary to ensure different timestamps 8 | output2 = Stem::Userdata.compile("spec/fixtures/userdata", :erb_binding => binding) 9 | output1.should == output2 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /spec/util_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | class UtilTestClass 4 | include Stem::Util 5 | end 6 | 7 | describe Stem::Util do 8 | include Stem::Util 9 | 10 | subject { UtilTestClass.new } 11 | 12 | describe "tagset_to_hash" do 13 | it { should respond_to(:tagset_to_hash) } 14 | 15 | it "returns a hash when fed a stupid tagset" do 16 | tagset = [ 17 | {"value"=>"postgres9-server", "key"=>"type"}, 18 | {"value"=>"production", "key"=>"version"} 19 | ] 20 | tagset_to_hash(tagset).should == {"type" => "postgres9-server", "version" => "production"} 21 | end 22 | 23 | it "returns a hash when there is only one tag" do 24 | tagset = [{"key" => "type", "value" => "postgres9-server"}] 25 | tagset_to_hash(tagset).should == {"type" => "postgres9-server"} 26 | end 27 | 28 | it "returns nil for tags with no value" do 29 | tagset = [{"key" => "deprecated"}] 30 | tagset_to_hash(tagset).should == {"deprecated" => nil} 31 | end 32 | end 33 | 34 | describe "get_filter_opts" do 35 | it { should respond_to(:get_filter_opts) } 36 | 37 | it "translates a hash into amazon FilterOpts" do 38 | tags = {"tag:version" => "production", 39 | "tag:family" => "postgres9-server"} 40 | 41 | get_filter_opts(tags).should == { "Filter.1.Value.0"=>"production", 42 | "Filter.0.Value.0"=>"postgres9-server", 43 | "Filter.1.Name"=>"tag:version", 44 | "Filter.0.Name"=>"tag:family" } 45 | end 46 | 47 | it "supports multiple values for the query" do 48 | tags = {"tag:version" => "production", 49 | "tag:family" => ["postgres84-server", "postgres9-server"]} 50 | 51 | get_filter_opts(tags).should == { "Filter.1.Value.0"=>"production", 52 | "Filter.0.Value.0"=>"postgres84-server", 53 | "Filter.0.Value.1"=>"postgres9-server", 54 | "Filter.1.Name"=>"tag:version", 55 | "Filter.0.Name"=>"tag:family" } 56 | end 57 | end 58 | 59 | describe "tags_to_filter" do 60 | it { should respond_to(:tags_to_filter) } 61 | 62 | it "translates tags into the aws style" do 63 | tags = {:version => "production", 64 | :family => "postgres9-server"} 65 | tags_to_filter(tags).should == get_filter_opts({"tag:version" => "production", 66 | "tag:family" => "postgres9-server"}) 67 | end 68 | it "special-cases architecture tag" do 69 | tags = {:architecture => "i386"} 70 | tags_to_filter(tags).should == get_filter_opts({"architecture" => "i386"}) 71 | end 72 | end 73 | 74 | describe "aggregate_hash_options_for_ami!" do 75 | it { should respond_to(:aggregate_hash_options_for_ami!) } 76 | 77 | it "shouldn't alter the hash if 'ami' is in the config hash" do 78 | config = {'ami' => 'ami-STOUT1'} 79 | aggregate_hash_options_for_ami!(config).should == config 80 | end 81 | 82 | it "should raise an exception if no valid ami options are in the input" do 83 | lambda do 84 | aggregate_hash_options_for_ami!({}) 85 | end.should raise_exception 86 | end 87 | 88 | describe "ami-name" do 89 | before do 90 | @config = { 'ami-name' => 'speedway_stout' } 91 | @ami_id = 'ami-STOUT1' 92 | Stem::Image.stub!(:named).and_return(@ami_id) 93 | end 94 | 95 | it "should look up the ami by name if 'ami-name' is in the input hash" do 96 | Stem::Image.should_receive(:named).and_return(@ami_id) 97 | aggregate_hash_options_for_ami!(@config) 98 | end 99 | 100 | it "should not include 'ami-name' in the result" do 101 | aggregate_hash_options_for_ami!(@config).keys.should_not include('ami-name') 102 | end 103 | 104 | it "should include the ami in the result" do 105 | aggregate_hash_options_for_ami!(@config)['ami'].should == @ami_id 106 | end 107 | 108 | it "should raise an exception if no ami matches the name" do 109 | Stem::Image.should_receive(:named).and_return(nil) 110 | lambda do 111 | aggregate_hash_options_for_ami!(@config) 112 | end.should raise_exception 113 | end 114 | end 115 | 116 | describe "ami-tags" do 117 | before do 118 | @config = { 'ami-tags' => { :brewery => 'alesmith' } } 119 | @ami_id = 'ami-SCOTCH' 120 | Stem::Image.stub!(:tagged).and_return([@ami_id]) 121 | end 122 | 123 | it "should look up the ami by name if 'ami-tags' is in the input hash" do 124 | Stem::Image.should_receive(:tagged).and_return([@ami_id]) 125 | aggregate_hash_options_for_ami!(@config) 126 | end 127 | 128 | it "should not include 'ami-tags' in the result" do 129 | aggregate_hash_options_for_ami!(@config).keys.should_not include('ami-name') 130 | end 131 | 132 | it "shuld include the ami in the result" do 133 | aggregate_hash_options_for_ami!(@config)['ami'].should == @ami_id 134 | end 135 | 136 | it "should raise an exception if no ami matches the tags" do 137 | Stem::Image.should_receive(:tagged).and_return([]) 138 | lambda do 139 | aggregate_hash_options_for_ami!(@config) 140 | end.should raise_exception 141 | end 142 | end 143 | end 144 | end 145 | -------------------------------------------------------------------------------- /stem.gemspec: -------------------------------------------------------------------------------- 1 | $spec = Gem::Specification.new do |s| 2 | s.specification_version = 2 if s.respond_to? :specification_version= 3 | s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= 4 | 5 | s.name = 'stem' 6 | s.version = '0.5.7' 7 | s.date = '2011-03-09' 8 | 9 | s.description = "minimalist EC2 instance management" 10 | s.summary = "an EC2 instance management library designed to get out of your way and give you instances" 11 | 12 | s.authors = ["Peter van Hardenberg", "Orion Henry", "Blake Gentry"] 13 | s.email = ["pvh@heroku.com", "orion@heroku.com", "b@heroku.com"] 14 | 15 | # = MANIFEST = 16 | s.files = %w[LICENSE README.md] + Dir["lib/**/*.rb"] 17 | 18 | s.executables = ["stem"] 19 | 20 | # = MANIFEST = 21 | s.add_dependency 'swirl', '~> 1.7.5' 22 | s.add_development_dependency 'rspec', '~> 2.5.0' 23 | s.add_development_dependency 'rspec-core', '~> 2.5.0' 24 | s.add_development_dependency 'rspec-expectations', '~> 2.5.0' 25 | s.add_development_dependency 'rspec-mocks', '~> 2.5.0' 26 | s.add_development_dependency 'timecop', '~> 0.3.5' 27 | s.add_development_dependency 'vcr', '~> 1.6.0' 28 | s.add_development_dependency 'webmock', '~> 1.6.2' 29 | s.homepage = "http://github.com/pvh/stem" 30 | s.require_paths = %w[lib] 31 | end 32 | --------------------------------------------------------------------------------