Merge pull request #44 from Lytro/cycle_multiple_instances_at_once
Cycle multiple instances at once
This commit is contained in:
commit
769547b854
|
@ -2,7 +2,7 @@ PATH
|
|||
remote: .
|
||||
specs:
|
||||
aws-missing-tools (0.0.1)
|
||||
aws-sdk
|
||||
aws-sdk (~> 1.11)
|
||||
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
|
@ -29,4 +29,4 @@ PLATFORMS
|
|||
|
||||
DEPENDENCIES
|
||||
aws-missing-tools!
|
||||
rspec
|
||||
rspec (~> 2.12)
|
||||
|
|
|
@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
|
|||
gem.require_paths = %w(lib)
|
||||
gem.version = AwsMissingTools::VERSION
|
||||
|
||||
gem.add_dependency 'aws-sdk'
|
||||
gem.add_dependency 'aws-sdk', '~> 1.11'
|
||||
|
||||
gem.add_development_dependency 'rspec'
|
||||
gem.add_development_dependency 'rspec', '~> 2.12'
|
||||
end
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
require 'timeout'
|
||||
require 'optparse'
|
||||
|
||||
module AwsMissingTools
|
||||
|
@ -22,6 +21,8 @@ module AwsMissingTools
|
|||
raise ArgumentError, "The Auto Scaling Group named #{@opts[:as_group_name]} does not exist in #{@opts[:region]}."
|
||||
end
|
||||
|
||||
@opts[:num_simultaneous_instances] = Integer(@opts[:num_simultaneous_instances]) rescue @group.auto_scaling_instances.count
|
||||
|
||||
@max_size_change = 0
|
||||
@time_spent_inservice = 0
|
||||
end
|
||||
|
@ -31,7 +32,8 @@ module AwsMissingTools
|
|||
region: 'us-east-1',
|
||||
elb_timeout: 60,
|
||||
inservice_time_allowed: 300,
|
||||
min_inservice_time: 30
|
||||
min_inservice_time: 30,
|
||||
num_simultaneous_instances: 1
|
||||
}
|
||||
|
||||
OptionParser.new('Usage: aws-ha-release.rb -a <group name> [options]', 50) do |opts|
|
||||
|
@ -62,6 +64,10 @@ module AwsMissingTools
|
|||
opts.on('-s', '--aws_secret_key AWS_SECRET_KEY', 'AWS Secret Key') do |v|
|
||||
options[:aws_secret_key] = v
|
||||
end
|
||||
|
||||
opts.on('-n', '--num-simultaneous-instances NUM', 'Number of instances to simultaneously bring up per iteration') do |v|
|
||||
options[:num_simultaneous_instances] = v
|
||||
end
|
||||
end.parse!(arguments)
|
||||
|
||||
raise OptionParser::MissingArgument, 'You must specify the AutoScaling Group Name: aws-ha-release.rb -a <group name>' if options[:as_group_name].nil?
|
||||
|
@ -85,55 +91,53 @@ module AwsMissingTools
|
|||
|
||||
@group.suspend_processes PROCESSES_TO_SUSPEND
|
||||
|
||||
if @group.max_size == @group.desired_capacity
|
||||
puts "#{@group.name} has a max-size of #{@group.max_size}. In order to recycle instances max-size will be temporarily increased by 1."
|
||||
@group.update(max_size: @group.max_size + 1)
|
||||
@max_size_change = 1
|
||||
@max_size_change = determine_max_size_change
|
||||
if @max_size_change > 0
|
||||
puts "#{@group.name} has a max-size of #{@group.max_size}. In order to recycle instances max-size will be temporarily increased by #{@max_size_change}."
|
||||
@group.update(max_size: @group.max_size + @max_size_change)
|
||||
end
|
||||
|
||||
@group.update(desired_capacity: @group.desired_capacity + 1)
|
||||
@group.update(desired_capacity: @group.desired_capacity + @opts[:num_simultaneous_instances])
|
||||
|
||||
puts "The list of instances in Auto Scaling Group #{@group.name} that will be terminated is:\n#{@group.auto_scaling_instances.map{ |i| i.ec2_instance.id }.to_ary}"
|
||||
@group.auto_scaling_instances.each do |instance|
|
||||
puts "The number of instances that will be brought up simultaneously is: #{@opts[:num_simultaneous_instances]}"
|
||||
@group.auto_scaling_instances.to_a.each_slice(@opts[:num_simultaneous_instances]) do |instances|
|
||||
time_taken = 0
|
||||
|
||||
begin
|
||||
Timeout::timeout(@opts[:inservice_time_allowed]) do
|
||||
|
||||
until all_instances_inservice_for_time_period?(@group.load_balancers, INSERVICE_POLLING_TIME)
|
||||
puts "#{time_taken} seconds have elapsed while waiting for all instances to be InService for a minimum of #{@opts[:min_inservice_time]} seconds."
|
||||
|
||||
time_taken += INSERVICE_POLLING_TIME
|
||||
sleep INSERVICE_POLLING_TIME
|
||||
end
|
||||
|
||||
puts "\nThe new instance was found to be healthy; one old instance will now be removed from the load balancers."
|
||||
deregister_instance instance.ec2_instance, @group.load_balancers
|
||||
end
|
||||
rescue Timeout::Error => e
|
||||
if time_taken >= @opts[:inservice_time_allowed]
|
||||
puts "\nDuring the last #{time_taken} seconds, a new AutoScaling instance failed to become healthy."
|
||||
puts "The following settings were changed and will not be changed back by this script:\n"
|
||||
|
||||
puts "AutoScaling processes #{PROCESSES_TO_SUSPEND} were suspended."
|
||||
puts "The desired capacity was changed from #{@group.desired_capacity - 1} to #{@group.desired_capacity}."
|
||||
puts "The desired capacity was changed from #{@group.desired_capacity - @opts[:num_simultaneous_instances]} to #{@group.desired_capacity}."
|
||||
|
||||
if @max_size_change > 0
|
||||
puts "The maximum size was changed from #{@group.max_size - @max_size_change} to #{@group.max_size}"
|
||||
end
|
||||
|
||||
raise
|
||||
else
|
||||
time_taken += INSERVICE_POLLING_TIME
|
||||
sleep INSERVICE_POLLING_TIME
|
||||
end
|
||||
end
|
||||
|
||||
puts "\nThe new instance(s) was/were found to be healthy; #{@opts[:num_simultaneous_instances]} old instance(s) will now be removed from the load balancers."
|
||||
instances.each { |instance| deregister_instance(instance.ec2_instance, @group.load_balancers) }
|
||||
|
||||
puts "Sleeping for the ELB Timeout period of #{@opts[:elb_timeout]}"
|
||||
sleep @opts[:elb_timeout]
|
||||
|
||||
puts "\nInstance #{instance.id} will now be terminated. By terminating this instance, the actual capacity will be decreased to 1 under desired-capacity."
|
||||
instance.terminate false
|
||||
puts "\nInstance(s) #{instances.map{ |i| i.ec2_instance.id }.join(', ')} will now be terminated. By terminating this/these instance(s), the actual capacity will be decreased to #{@opts[:num_simultaneous_instances]} under desired-capacity."
|
||||
instances.each { |instance| instance.terminate false }
|
||||
end
|
||||
|
||||
puts "\n#{@group.name} had its desired-capacity increased temporarily by 1 to a desired-capacity of #{@group.desired_capacity}."
|
||||
puts "The desired-capacity of #{@group.name} will now be returned to its original desired-capacity of #{@group.desired_capacity - 1}."
|
||||
@group.update(desired_capacity: @group.desired_capacity - 1)
|
||||
puts "\n#{@group.name} had its desired-capacity increased temporarily by #{@opts[:num_simultaneous_instances]} to a desired-capacity of #{@group.desired_capacity}."
|
||||
puts "The desired-capacity of #{@group.name} will now be returned to its original desired-capacity of #{@group.desired_capacity - @opts[:num_simultaneous_instances]}."
|
||||
@group.update(desired_capacity: @group.desired_capacity - @opts[:num_simultaneous_instances])
|
||||
|
||||
if @max_size_change > 0
|
||||
puts "\n#{@group.name} had its max_size increased temporarily by #{@max_size_change} to a max_size of #{@group.max_size}."
|
||||
|
@ -146,6 +150,14 @@ module AwsMissingTools
|
|||
@group.resume_all_processes
|
||||
end
|
||||
|
||||
def determine_max_size_change
|
||||
if @group.max_size - @group.desired_capacity < @opts[:num_simultaneous_instances]
|
||||
@group.desired_capacity + @opts[:num_simultaneous_instances] - @group.max_size
|
||||
else
|
||||
0
|
||||
end
|
||||
end
|
||||
|
||||
def deregister_instance(instance, load_balancers)
|
||||
load_balancers.each do |load_balancer|
|
||||
load_balancer.instances.deregister instance
|
||||
|
|
|
@ -28,6 +28,22 @@ describe 'aws-ha-release' do
|
|||
AwsMissingTools::AwsHaRelease.new(opts)
|
||||
}.should raise_error
|
||||
end
|
||||
|
||||
context 'number of simultaneous instances' do
|
||||
before do
|
||||
as.groups.create opts[1]
|
||||
opts.push('--num-simultaneous-instances')
|
||||
end
|
||||
it 'with MAX, sets the option to the number of active instances' do
|
||||
opts.push('MAX')
|
||||
expect(AwsMissingTools::AwsHaRelease.new(opts).instance_variable_get('@opts')[:num_simultaneous_instances]).to eq 2
|
||||
end
|
||||
|
||||
it 'with an integer, sets the option to that integer' do
|
||||
opts.push('1')
|
||||
expect(AwsMissingTools::AwsHaRelease.new(opts).instance_variable_get('@opts')[:num_simultaneous_instances]).to eq 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#parse_options' do
|
||||
|
@ -45,6 +61,7 @@ describe 'aws-ha-release' do
|
|||
expect(options[:aws_access_key]).not_to be_nil
|
||||
expect(options[:aws_secret_key]).not_to be_nil
|
||||
expect(options[:min_inservice_time]).not_to be_nil
|
||||
expect(options[:num_simultaneous_instances]).not_to be_nil
|
||||
end
|
||||
|
||||
context 'optional params' do
|
||||
|
@ -80,6 +97,20 @@ describe 'aws-ha-release' do
|
|||
expect(AwsMissingTools::AwsHaRelease.parse_options(options)[:min_inservice_time]).to eq 30
|
||||
end
|
||||
end
|
||||
|
||||
context 'number of instances to simultaneously bring up' do
|
||||
it 'recognizes integer inputs' do
|
||||
[%w(-a test_group -n 2), %w(-a test_group --num-simultaneous-instances 2)].each do |options|
|
||||
expect(AwsMissingTools::AwsHaRelease.parse_options(options)[:num_simultaneous_instances]).to eq '2'
|
||||
end
|
||||
end
|
||||
|
||||
it 'recognizes the MAX keyword' do
|
||||
[%w(-a test_group -n MAX), %w(-a test_group --num-simultaneous-instances MAX)].each do |options|
|
||||
expect(AwsMissingTools::AwsHaRelease.parse_options(options)[:num_simultaneous_instances]).to eq 'MAX'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
@ -264,4 +295,37 @@ describe 'aws-ha-release' do
|
|||
expect(elb_two.instances).not_to include instance_one
|
||||
end
|
||||
end
|
||||
|
||||
describe '#determine_max_size_change' do
|
||||
before do
|
||||
@group = as.groups.create opts[1]
|
||||
end
|
||||
|
||||
it 'does not change the desired capacity by default' do
|
||||
@group.update(max_size: 4, desired_capacity: 2)
|
||||
aws_ha_release = AwsMissingTools::AwsHaRelease.new(opts)
|
||||
|
||||
expect(aws_ha_release.determine_max_size_change).to eq 0
|
||||
end
|
||||
|
||||
it 'adjusts the max size when it is equal to the desired capacity' do
|
||||
@group.update(max_size: 2, desired_capacity: 2)
|
||||
aws_ha_release = AwsMissingTools::AwsHaRelease.new(opts)
|
||||
|
||||
expect(aws_ha_release.determine_max_size_change).to eq 1
|
||||
end
|
||||
|
||||
it 'accounts for num_simultaneous_instances' do
|
||||
@group.update(max_size: 2, desired_capacity: 2)
|
||||
aws_ha_release = AwsMissingTools::AwsHaRelease.new(%w(-a test_group --num-simultaneous-instances 2 -o testaccesskey -s testsecretkey -r test_region -i 1 -t 0 -m 5))
|
||||
|
||||
expect(aws_ha_release.determine_max_size_change).to eq 2
|
||||
|
||||
@group.update(max_size: 3, desired_capacity: 2)
|
||||
expect(aws_ha_release.determine_max_size_change).to eq 1
|
||||
|
||||
@group.update(max_size: 4, desired_capacity: 2)
|
||||
expect(aws_ha_release.determine_max_size_change).to eq 0
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -48,7 +48,7 @@ module AWS
|
|||
end
|
||||
|
||||
def auto_scaling_instances
|
||||
@auto_scaling_instances ||= [AWS::FakeAutoScaling::Instance.new(self), AWS::FakeAutoScaling::Instance.new(self)]
|
||||
@auto_scaling_instances ||= AWS::FakeCore::Data::List.new [AWS::FakeAutoScaling::Instance.new(self), AWS::FakeAutoScaling::Instance.new(self)]
|
||||
end
|
||||
|
||||
def load_balancers
|
||||
|
|
71
spec/support/fake_core_data_list.rb
Normal file
71
spec/support/fake_core_data_list.rb
Normal file
|
@ -0,0 +1,71 @@
|
|||
module AWS
|
||||
module FakeCore
|
||||
class Data
|
||||
module MethodMissingProxy
|
||||
|
||||
protected
|
||||
|
||||
def method_missing *args, &block
|
||||
if block_given?
|
||||
return_value = @data.send(*args) do |*values|
|
||||
yield(*values.flatten.map{|v| Data.cast(v) })
|
||||
end
|
||||
Data.cast(return_value)
|
||||
else
|
||||
Data.cast(@data.send(*args))
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
include MethodMissingProxy
|
||||
|
||||
def method_missing method_name, *args, &block
|
||||
if
|
||||
args.empty? and !block_given? and
|
||||
key = _remove_question_mark(method_name) and
|
||||
@data.has_key?(key)
|
||||
then
|
||||
Data.cast(@data[key])
|
||||
else
|
||||
super
|
||||
end
|
||||
end
|
||||
|
||||
class << self
|
||||
|
||||
# Given a hash, this method returns a {Data} object. Given
|
||||
# an Array, this method returns a {Data::List} object. Everything
|
||||
# else is returned as is.
|
||||
#
|
||||
# @param [Object] value The value to conditionally wrap.
|
||||
#
|
||||
# @return [Data,Data::List,Object] Wraps hashes and lists with
|
||||
# Data and List objects, all other objects are returned as
|
||||
# is.
|
||||
#
|
||||
def cast value
|
||||
case value
|
||||
when Hash then Data.new(value)
|
||||
when Array then Data::List.new(value)
|
||||
else value
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
class List
|
||||
include MethodMissingProxy
|
||||
|
||||
def initialize(array)
|
||||
@data = array
|
||||
end
|
||||
|
||||
def to_ary
|
||||
@data
|
||||
end
|
||||
alias_method :to_a, :to_ary
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Reference in New Issue
Block a user