From 09cee91c56574be2cedda7f9d7682e0a4f777d7b Mon Sep 17 00:00:00 2001 From: Dave Stern Date: Tue, 9 Apr 2013 17:46:36 -0400 Subject: [PATCH] Region Copy: -g and -d options to tag and initiate region copies -g tested with new backups using: bash ec2-automate-backup.sh -v 'vol-8a1a43f6 vol-2a78d459' -g 'us-west-1 us-west-2' -n -d tested with various options and repeated calls to test idempotent copies: bash ec2-automate-backup.sh -s regioncopy -d 'us-west-1' bash ec2-automate-backup.sh -s regioncopy -d 'us-west-1' bash ec2-automate-backup.sh -s regioncopy -d 'us-west-1 us-west-2' bash ec2-automate-backup.sh -s regioncopy -d 'us-west-2' etc. --- .../Beta/ec2-automate-backup.sh | 103 +++++++++++++++--- 1 file changed, 88 insertions(+), 15 deletions(-) diff --git a/ec2-automate-backup/Beta/ec2-automate-backup.sh b/ec2-automate-backup/Beta/ec2-automate-backup.sh index fa46302..9f823d3 100755 --- a/ec2-automate-backup/Beta/ec2-automate-backup.sh +++ b/ec2-automate-backup/Beta/ec2-automate-backup.sh @@ -27,27 +27,38 @@ get_EBS_List() fi ebs_selection_string="$volumeid" ;; - tag) + tag) if [[ -z $tag ]] then echo "The selected selection_method \"tag\" (-s tag) requires a valid tag (-t key=value) for operation. Correct usage is as follows: \"-s tag -t backup=true\" or \"-s tag -t Name=my_tag.\"" 1>&2 ; exit 64 fi ebs_selection_string="--filter tag:$tag" ;; + regioncopy) + if [[ -z $region_copy_destinations ]] + then echo "The selected selection_method \"regioncopy\" (-s regioncopy) requires valid region names (-d '[region name(s)]') or 'all' for operation. Correct usage is as follows: \"-s regioncopy -d 'us-west-1'\" or \"-s regioncopy -d 'us-west-1 us-west-2'.\"" 1>&2 ; exit 64 + fi + ;; *) echo "If you specify a selection_method (-s selection_method) for selecting EBS volumes you must select either \"volumeid\" (-s volumeid) or \"tag\" (-s tag)." 1>&2 ; exit 64 ;; esac - #creates a list of all ebs volumes that match the selection string from above - ebs_backup_list_complete=`ec2-describe-volumes --show-empty-fields --region $region $ebs_selection_string 2>&1` - #takes the output of the previous command - ebs_backup_list_result=`echo $?` - if [[ $ebs_backup_list_result -gt 0 ]] - then echo -e "An error occured when running ec2-describe-volumes. The error returned is below:\n$ebs_backup_list_complete" 1>&2 ; exit 70 + + if [[ -n $ebs_selection_string ]] + then + #creates a list of all ebs volumes that match the selection string from above + ebs_backup_list_complete=`ec2-describe-volumes --show-empty-fields --region $region $ebs_selection_string 2>&1` + #takes the output of the previous command + ebs_backup_list_result=`echo $?` + if [[ $ebs_backup_list_result -gt 0 ]] + then echo -e "An error occured when running ec2-describe-volumes. The error returned is below:\n$ebs_backup_list_complete" 1>&2 ; exit 70 + fi + ebs_backup_list=`echo "$ebs_backup_list_complete" | grep ^VOLUME | cut -f 2` + #code to right will output list of EBS volumes to be backed up: echo -e "Now outputting ebs_backup_list:\n$ebs_backup_list" fi - ebs_backup_list=`echo "$ebs_backup_list_complete" | grep ^VOLUME | cut -f 2` - #code to right will output list of EBS volumes to be backed up: echo -e "Now outputting ebs_backup_list:\n$ebs_backup_list" } create_EBS_Snapshot_Tags() { + echo "create_EBS_Snapshot_Tags region_copy_scheduled_destinations=$region_copy_scheduled_destinations" + #snapshot tags holds all tags that need to be applied to a given snapshot - by aggregating tags we ensure that ec2-create-tags is called only onece snapshot_tags="" #if $name_tag_create is true then append ec2ab_${ebs_selected}_$current_date to the variable $snapshot_tags @@ -68,6 +79,15 @@ create_EBS_Snapshot_Tags() snapshot_tags="$snapshot_tags --tag Volume=${ebs_selected} --tag Created=$current_date" fi + #if $name_tag_create is true then append RegionCopy=[region]:scheduled,[region]:scheduled to the variable $snapshot_tags + if [[ -n $region_copy_scheduled_destinations ]] + then + ec2_snapshot_resource_id=`echo "$ec2_create_snapshot_result" | cut -f 2` + regions=$(echo $region_copy_scheduled_destinations | sed 's/ */:scheduled,/g') + snapshot_tags="$snapshot_tags --tag RegionCopy=${regions}:scheduled" + echo "tagging $ec2_snapshot_resource_id with $snapshot_tags" + fi + #if $snapshot_tags is not zero length then set the tag on the snapshot using ec2-create-tags if [[ -n $snapshot_tags ]] then echo "Tagging Snapshot $ec2_snapshot_resource_id with the following Tags:" @@ -113,7 +133,7 @@ purge_EBS_Snapshots() snapshot_tag_list=`ec2-describe-tags --show-empty-fields --region $region --filter resource-type=snapshot --filter key=PurgeAllow,PurgeAfterFE` #snapshot_purge_allowed is a list of all snapshot_ids with PurgeAllow=true snapshot_purge_allowed=`echo "$snapshot_tag_list" | grep .*PurgeAllow'\s'true | cut -f 3` - + for snapshot_id_evaluated in $snapshot_purge_allowed do #gets the "PurgeAfterFE" date which is in UTC with UNIX Time format (or xxxxxxxxxx / %s) @@ -133,6 +153,50 @@ purge_EBS_Snapshots() done } +region_copy_EBS_Snapshots() +{ + #snapshots_to_region_copy is a list of all snapshot_ids with RegionCopy=.*:scheduled.* + # --output-delimiter and $'n' substitution used so $IFS doesn't have to be + # manipulated with for loops and the cut command + snapshot_tag_list=$(ec2-describe-tags --show-empty-fields --region $region --filter resource-type=snapshot --filter key=RegionCopy | grep ':scheduled' | cut -f3,5- --output-delimiter='~') + + for snapshot_data in $(echo ${snapshot_tag_list//$'\n'/ } | cut -f1-) + do + snapshot_id=$(echo $snapshot_data | cut -d'~' -f1) + regions=$(echo $snapshot_data | cut -d'~' -f2-) + original_regions=$regions + region_copy_tag='' + for region_data in ${regions//,/ } + do + destination_region=$(echo $region_data | cut -d: -f1) + status=$(echo $region_data | cut -d: -f2-) + region_copy_tag_append="$destination_region:$status" + if [[ "$region_copy_destinations" = "all" || -n $(echo "$region_copy_destinations"| grep $destination_region) ]] + then + current_datetime=$(date +%Y-%m-%d_%H:%M:%S) + if [ "$status" = "scheduled" ] + then + ec2_copy_snapshot_complete=$(ec2-copy-snapshot -r $region -s $snapshot_id --region $destination_region) + ec2_copy_snapshot_result=`echo $?` + if [[ $ec2_copy_snapshot_result -gt 0 ]] + then + echo -e "An error occured when running ec2-copy-snapshot. The error returned is below:\n$ec2_copy_snapshot_complete" 1>&2 ; exit 70 + else + region_copy_tag_append="$destination_region:$current_datetime" + echo "The snapshot \"$snapshot_id\" is being copied to $destination_region." + fi + fi + fi + region_copy_tag="${region_copy_tag},$region_copy_tag_append" + done + + if [[ "${region_copy_tag#,}" != "$original_regions" ]] + then + ec2-create-tags $snapshot_id --region $region --tag RegionCopy=${region_copy_tag#,} + fi + done +} + app_name=`basename $0` #sets defaults @@ -148,7 +212,7 @@ user_tags=false #sets the Purge Snapshot feature to false - this feature will eventually allow the removal of snapshots that have a "PurgeAfterFE" tag that is earlier than current date purge_snapshots=false #handles options processing -while getopts :s:c:r:v:t:k:pnu opt +while getopts :s:c:r:v:t:k:g:d:l:pnu opt do case $opt in s) selection_method="$OPTARG";; @@ -157,6 +221,8 @@ while getopts :s:c:r:v:t:k:pnu opt v) volumeid="$OPTARG";; t) tag="$OPTARG";; k) purge_after_input="$OPTARG";; + g) region_copy_scheduled_destinations="$OPTARG";; + d) region_copy_destinations="$OPTARG";; n) name_tag_create=true;; p) purge_snapshots=true;; u) user_tags=true;; @@ -166,7 +232,7 @@ while getopts :s:c:r:v:t:k:pnu opt #sources "cron_primer" file for running under cron or other restricted environments - this file should contain the variables and environment configuration required for ec2-automate-backup to run correctly if [[ -n $cron_primer ]] - then if [[ -f $cron_primer ]] + then if [[ -f $cron_primer ]] then source $cron_primer else echo "Cron Primer File \"$cron_primer\" Could Not Be Found." 1>&2 ; exit 70 @@ -177,7 +243,7 @@ fi if [[ -z $region ]] #if the environment variable $EC2_REGION is not set set to us-east-1 then if [[ -z $EC2_REGION ]] - #if both + #if both then region="us-east-1" else region=$EC2_REGION @@ -213,7 +279,7 @@ do then echo -e "An error occured when running ec2-create-snapshot. The error returned is below:\n$ec2_create_snapshot_result" 1>&2 ; exit 70 else ec2_snapshot_resource_id=`echo "$ec2_create_snapshot_result" | cut -f 2` - fi + fi create_EBS_Snapshot_Tags done @@ -221,4 +287,11 @@ done if $purge_snapshots then echo "Snapshot Purging is Starting Now." purge_EBS_Snapshots -fi \ No newline at end of file +fi + +#if region_copy_destinations is true, then run region_copy_EBS_Snapshots function +if [[ -n $region_copy_destinations ]] + then echo "Snapshot Copying to regions $region_copy_destinations is Starting Now." + region_copy_EBS_Snapshots +fi +