├── LICENSE
├── README.md
├── mongodb-ebs-snapshot
└── mongodb-oplog-window-size


/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2016 Daniele Valeriani
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # mongodb-tools
 2 | Scripts that I find useful for managing MongoDB
 3 | 
 4 | #### mongodb-ebs-snapshot
 5 | 
 6 | Takes consistent snapshots of MongoDB data directories stored on EBS volumes.
 7 | Then expires old snapshots assuming each snapshot is taken at hh:00.<br />
 8 | The expiration policy is to keep 2 days of hourly snapshots, 2 weeks of daily
 9 | snapshots at 00:00 and one year of monthly snapshots taken at 00:00 on the
10 | first day of the month.
11 | The expirer is still pretty basic but it does the job if you're happy with its policy.
12 | 
13 | #### mongodb-oplog-window-size
14 | 
15 | Reports the MongoDB oplog window size in hours to a local statsd server.
16 | 


--------------------------------------------------------------------------------
/mongodb-ebs-snapshot:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | 
  3 | ################################################################################
  4 | # MongoDB EBS snapshot
  5 | # Version 0.2
  6 | #
  7 | # Takes consistent snapshots of MongoDB data directories stored on EBS volumes.
  8 | # Then it expires old snapshots assuming each snapshot is taken at hh:00.
  9 | # The expiration policy is to keep 2 days of hourly snapshots, 2 weeks of daily
 10 | # snapshots at 00:00 and one year of monthly snapshots taken at 00:00 on the
 11 | # first day of the month.
 12 | ################################################################################
 13 | 
 14 | import argparse
 15 | import boto.utils
 16 | import fcntl
 17 | import logging
 18 | import os
 19 | import signal
 20 | import sys
 21 | from boto import ec2
 22 | from datetime import datetime, timedelta
 23 | from dateutil.relativedelta import relativedelta
 24 | 
 25 | 
 26 | OWNER_ID = 012345543210  # Insert your AWS account id here
 27 | LOCK_FILE = "/var/lock/mongodb-ebs-snapshot.lock"
 28 | 
 29 | lock_fd = open(LOCK_FILE, "w")
 30 | try:
 31 |     fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
 32 | except IOError:
 33 |     sys.exit("Another instance is running")
 34 | 
 35 | def cleanup(*args):
 36 |     fcntl.flock(lock_fd, fcntl.LOCK_UN)
 37 |     # Remove the lock file
 38 |     if os.path.exists(LOCK_FILE):
 39 |         os.remove(LOCK_FILE)
 40 | 
 41 | for sig in (
 42 |         signal.SIGABRT,
 43 |         signal.SIGILL,
 44 |         signal.SIGINT,
 45 |         signal.SIGSEGV,
 46 |         signal.SIGTERM
 47 |     ):
 48 |     signal.signal(sig, cleanup)
 49 | 
 50 | def round_to_minute(dt):
 51 |     """
 52 |     This is needed because there is a slight overhead between the start of the
 53 |     script execution and when it gets to the expiring part.
 54 |     """
 55 |     seconds = (dt - dt.min).seconds
 56 |     rounding = ((seconds + 30) // 60) * 60
 57 |     return dt + timedelta(0, rounding - seconds, -dt.microsecond)
 58 | 
 59 | def generate_dt_list(start, end, delta):
 60 |     """
 61 |     Generate a list of accepted datetimes to use as a mask for the expiration:
 62 |     everything that doesn't fall in this list will be deleted.
 63 |     """
 64 |     res = []
 65 |     curr = start
 66 |     while curr < end:
 67 |         res.append(curr)
 68 |         curr += delta
 69 |     return res
 70 | 
 71 | now = datetime.utcnow()
 72 | 
 73 | # Logging
 74 | logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 75 | 
 76 | # Arguments parser
 77 | parser = argparse.ArgumentParser()
 78 | parser.add_argument("-d", "--device", dest="device_name",
 79 |     help="Device to snapshot. default: /dev/sdd",
 80 |     default="/dev/sdd")
 81 | parser.add_argument("-e", "--env", dest="env",
 82 |     help="Environment. default: prod",
 83 |     default="prod")
 84 | parser.add_argument("-f", "--force", dest="force", action="store_true",
 85 |     help="Perform the lock even on a primary node. default: False",
 86 |     default=False)
 87 | parser.add_argument("-n", "--name", dest="snapshot_name",
 88 |     help="Snapshot name. default: mongodb-data_<ISO time>",
 89 |     default="mongo_{0}".format(datetime.now().isoformat()))
 90 | parser.add_argument("-r", "--region", dest="aws_region",
 91 |     help="AWS region. default: us-east-1",
 92 |     default="us-east-1")
 93 | args = parser.parse_args()
 94 | 
 95 | timestamp_format = "%Y-%m-%dT%H:%M:%S.%fZ"
 96 | instance_id = boto.utils.get_instance_metadata()["instance-id"]
 97 | 
 98 | # Fetch the volume id mapped to the mongodb data device
 99 | conn = ec2.connect_to_region(args.aws_region)
100 | for vol in conn.get_all_volumes(filters={'attachment.instance-id': instance_id}):
101 |     if vol.attach_data.device == args.device_name:
102 |         logging.info("Detected MongoDB volume: {0}".format(vol.id))
103 |         mongo_vol = vol
104 |         break
105 | else:
106 |     logging.critical("No attached volume could be found for {0}".format(args.device_name))
107 |     sys.exit(1)
108 | 
109 | # Take a snapshot
110 | logging.info("Taking a snapshot of {vol_id} and calling it {snap_name}".format(
111 |         vol_id=mongo_vol.id,
112 |         snap_name=args.snapshot_name
113 |     )
114 | )
115 | new_snap = conn.create_snapshot(mongo_vol.id, "MongoDB snapshot for {0} data".format(args.env))
116 | new_snap.add_tag('Name', args.snapshot_name)
117 | new_snap.add_tag('Env', args.env)
118 | 
119 | # Get point-in-time markers
120 | two_days_ago = now + relativedelta(days=-2, minute=0, second=0, microsecond=0)
121 | two_weeks_ago = now + relativedelta(weeks=-2, hour=0, minute=0, second=0, microsecond=0)
122 | a_year_ago = now + relativedelta(years=-1, day=1, hour=0, minute=0, second=0, microsecond=0)
123 | 
124 | # Generate a list of timetamps
125 | allowed_snapshots = []
126 | allowed_snapshots += generate_dt_list(two_days_ago, now, relativedelta(hours=1))
127 | allowed_snapshots += generate_dt_list(two_weeks_ago, two_days_ago, relativedelta(days=1))
128 | allowed_snapshots += generate_dt_list(a_year_ago, two_weeks_ago, relativedelta(months=1))
129 | 
130 | # Now delete the old snapshots
131 | filters = {
132 |     "owner_id": OWNER_ID,
133 |     "description": "MongoDB snapshot for {0} data".format(args.env)
134 | }
135 | snapshots = conn.get_all_snapshots(filters=filters)
136 | deletion_counter = 0
137 | for snapshot in snapshots:
138 |     # The snapshot just taken is too close to now, so we skip it just in case
139 |     if snapshot.id == new_snap.id:
140 |         continue
141 |     snapshot_time = round_to_minute(datetime.strptime(snapshot.start_time, timestamp_format))
142 |     if snapshot_time not in allowed_snapshots:
143 |         logging.info("Deleting {0} - Unnamed from {1}".format(snapshot.id, snapshot.start_time))
144 |         deletion_counter = deletion_counter + 1
145 |         snapshot.delete()
146 | 
147 | logging.info("Deleted {0} snapshots".format(deletion_counter))
148 | cleanup()
149 | 


--------------------------------------------------------------------------------
/mongodb-oplog-window-size:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | ################################################################################
 4 | # MongoDB oplog window size statsd emitter
 5 | # Version 0.1
 6 | #
 7 | # Reports the MongoDB oplog window size in hours to a local statsd server.
 8 | ################################################################################
 9 | 
10 | # Make sure to quit when running on an arbiter
11 | if [[ "$(mongo --quiet <<< 'rs.isMaster().arbiterOnly')" == "true" ]]; then
12 |   exit 0
13 | fi
14 | 
15 | hours=$(mongo --quiet --eval "
16 | var ri = db.getReplicationInfo();
17 | var window = new Date(ri.tLast).valueOf() - new Date(ri.tFirst).valueOf();
18 | print(window / (60*60*1000))
19 | ")
20 | 
21 | # Send the metric to localhost:8125
22 | [[ -n "$hours" ]] && echo -n "mongodb.oplog.window_size:${hours}|g|" >/dev/udp/localhost/8125
23 | 


--------------------------------------------------------------------------------