├── .gitignore ├── LICENSE ├── Makefile ├── Readme.md ├── build-phar.php ├── composer.json ├── composer.lock └── src ├── Application.php ├── CallbackStream.php ├── CliController.php ├── cli.php └── web.php /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor/ 2 | rmq-dump 3 | dump*.json 4 | 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 webreactor 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BINARY=rmq-dump 2 | #======================================================= 3 | 4 | build: vendor 5 | php build-phar.php --bin="$(BINARY)" 6 | 7 | vendor: 8 | composer install --no-dev --optimize-autoloader 9 | 10 | clean: 11 | -rm $(BINARY) 12 | 13 | clean-vendor: clean 14 | -rm -rf vendor 15 | 16 | install: $(BINARY) 17 | cp $(BINARY) /usr/local/bin/ 18 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | RabbitMQ messages backup tool 2 | ============================= 3 | 4 | ### Features 5 | 6 | - Dump messages from RabbitMQ keeping body, routing key and headers 7 | - Load messages to RabbitMQ 8 | - By default will not consume messages. Server state is not changed after dumping. 9 | - Dump all messages from whole RMQ server or specific vhost or a queue 10 | - Alter messages vhost or queue name during dump or load 11 | - Load from a dump only specified vhosts or queues 12 | - Declare needed queues at load if `--declare` option is used 13 | - Work with STDOUT and STDIN streams using json lines format 14 | - Dry run option 15 | - Supports Linux, MacOS, Windows. Installed php is needed 16 | 17 | To backup exchanges, queues and bindings use RMQ webinterface - use RMQ web interface or rabbitmqadmin for it. 18 | 19 | ### Install 20 | 21 | Linux, MacOS: 22 | ```bash 23 | # php dependencies 24 | apt-get install php-cli php-curl php-mbstring php-bcmath 25 | 26 | # Download and install rmq-dump 27 | curl -L https://github.com/webreactor/rmq-dump/releases/download/0.0.4/rmq-dump > /usr/local/bin/rmq-dump 28 | chmod a+x /usr/local/bin/rmq-dump 29 | ``` 30 | 31 | Windows: 32 | 1. Download compiled php script from [releases](https://github.com/webreactor/rmq-dump/releases/) 33 | 2. [Install php](https://windows.php.net/download/) 34 | 3. Create `rmq-dump.bat` in the same folder with rmq-dump: 35 | ```bat 36 | @php "%~dp0rmq-dump" %* 37 | ``` 38 | 4. Make it available in PATH. Or install to same folder with php.exe 39 | 40 | ### Usage 41 | 42 | `rmq-dump command ` 43 | 44 | Commands: 45 | 46 | - `dump`: dumps messages from rabbitmq to STDOUT 47 | - `load`: loads messages from STDIN to rabbitmq 48 | - `dryload`: dry run load will show how `-v`, `-s` and `-a` options will affect messages 49 | - `list`: shows current state in RabbitMQ with -v -s filters. Use as dry run for dump 50 | - `help`: prints help 51 | 52 | 53 | Make dump all messages of all vhosts: 54 | 55 | `rmq-dump dump -H host -u user -p password > dump.json` 56 | 57 | Make dump all messages of vhost /app/live: 58 | 59 | `rmq-dump dump -H host -u user -p password -v /app/live > dump.json` 60 | 61 | Load dump: 62 | 63 | `cat dump.json | rmq-dump load -H host -u user -p password` 64 | 65 | Load dump to vhost /app/test and create queues if needed: 66 | 67 | `cat dump.json | rmq-dump load -H host -u user -p password -d -a /app/test` 68 | 69 | Load dump from big dump only specific vhost /app/prod to vhost /app/test: 70 | 71 | `cat dump.json | rmq-dump load -H host -u user -p password -v /app/prod -a /app/test` 72 | 73 | `-a or --alter` - is a middleware that replaces vhost or/and queue value in a message. Can be used at dump or load process. 74 | 75 | Copy all messages from one queue1 to queue2 qithout storing them: 76 | 77 | `rmq-dump -u user -p pass -v /app/live:queue1 dump | ./rmq-dump -u user -p pass -a :queue2 load` 78 | 79 | Copy all messages from vhost1 to vhost2 not storing them. Note: all queues have exists at vhost2: 80 | 81 | `rmq-dump -u user -p pass -v /vhost1 dump | ./rmq-dump -u user -p pass -a /vhost2 load` 82 | 83 | List all what you have in RMQ: 84 | 85 | `rmq-dump -H host -u user -p password list` 86 | 87 | Print help: 88 | 89 | `rmq-dump help` 90 | 91 | 92 | **Options** 93 | 94 | ``` 95 | Arguments: 96 | Full name | Short | Default | Note 97 | ------------------------------------------------------- 98 | --host -H localhost 99 | --port -P 15672 100 | --binary-port -B 5672 101 | --user -u 102 | --pass -p 103 | --vhost -v vhost[:queue] | :queue 104 | --skip -s vhost[:queue] | :queue 105 | --alter -a vhost[:queue]~vhost[:queue] | vhost[:queue] 106 | --ack -k false acknowlege (delete) messages when dump 107 | --declare -d false declare persistent queues when load 108 | ``` 109 | 110 | `--vhost` `--skip` `--alter` can be used miltiple times 111 | 112 | ### How it works 113 | 114 | **Dumping** 115 | 116 | rmq-dump runs throught all specified vhosts and queues (all, if not specified). 117 | Using basic_get rmq-dump gets all messages qithout acknowledging then closes connection so all messages go back to their queues. 118 | Tested up to 500k messages in a queue. rmq-dump sends all recieved messages in STDOUT using json lines format. Pipe it to a file if you want to store them. 119 | 120 | It you need consistent state backup - stop all consumers and publishers before you make backup. 121 | 122 | Each stored message contains: 123 | 124 | - vhost 125 | - queue 126 | - body 127 | - properties 128 | - headers 129 | 130 | During processing the programm uses STDERR to show status. 131 | 132 | If you care about message order do `tac dump.json > good_dump.json`. Because they stored in way for queue returned them. 133 | 134 | Since it stores messages in json lines format. Use `wc -l dump.json` to get how many messages in a dump file. 135 | 136 | **Loading** 137 | 138 | Loading process expects messages from STDIN. Use examples below. 139 | 140 | When you load them back it creates temporary exchange in order to put message to destination queue qith original routing key. 141 | That means in properties of loaded message original exchange will be replaced with temporary name. 142 | 143 | Loading is a good place to apply `--alter` filters that can modify destination vhost or queue name. 144 | 145 | Only specified source vhost, queue name can be loaded using `-vhost` from a big dump. 146 | 147 | rmq-dump can be piped to another rmq-dump that allows copy messages qithout storing them. 148 | 149 | 150 | ### Build your own and install 151 | 152 | ```bash 153 | make && make install 154 | ``` 155 | -------------------------------------------------------------------------------- /build-phar.php: -------------------------------------------------------------------------------- 1 | 'src/cli.php', 5 | 'web' => 'src/web.php', 6 | 'bin' => 'application.phar', 7 | ); 8 | $temp_file = 'application.phar'; 9 | 10 | $options = getopt('', array('cli::', 'web::', 'bin::')); 11 | $options = array_merge($defaults, $options); 12 | 13 | @unlink($options['bin']); 14 | @unlink($temp_file); 15 | 16 | $phar = new Phar($temp_file); 17 | $phar->buildFromDirectory(__DIR__, '/^((?!\.git|dump.*json).)*$/'); 18 | $defaultStub = $phar->createDefaultStub($options['cli'], $options['web']); 19 | $defaultStub = "#!/usr/bin/env php\n".$defaultStub; 20 | $phar->setStub($defaultStub); 21 | unset($phar); 22 | chmod($temp_file, 0755); 23 | rename($temp_file, $options['bin']); 24 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "RMQDumper", 3 | "license": "MIT", 4 | "autoload": { 5 | "psr-4": { 6 | "RMQDumper\\": "src/" 7 | } 8 | }, 9 | "repositories": [ 10 | { 11 | "type": "vcs", 12 | "url": "https://github.com/webreactor/cli-arguments.git" 13 | } 14 | ], 15 | "require": { 16 | "symfony/yaml": "3.0.1", 17 | "webreactor/cli-arguments": "dev-master", 18 | "php-amqplib/php-amqplib": "*" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /composer.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_readme": [ 3 | "This file locks the dependencies of your project to a known state", 4 | "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", 5 | "This file is @generated automatically" 6 | ], 7 | "hash": "d6ce81f2e4e0473f92f188ebf923d762", 8 | "content-hash": "1e3f292606ed7419624e73fe7c46fe85", 9 | "packages": [ 10 | { 11 | "name": "php-amqplib/php-amqplib", 12 | "version": "v2.6.3", 13 | "source": { 14 | "type": "git", 15 | "url": "https://github.com/php-amqplib/php-amqplib.git", 16 | "reference": "fa2f0d4410a11008cb36b379177291be7ee9e4f6" 17 | }, 18 | "dist": { 19 | "type": "zip", 20 | "url": "https://api.github.com/repos/php-amqplib/php-amqplib/zipball/fa2f0d4410a11008cb36b379177291be7ee9e4f6", 21 | "reference": "fa2f0d4410a11008cb36b379177291be7ee9e4f6", 22 | "shasum": "" 23 | }, 24 | "require": { 25 | "ext-bcmath": "*", 26 | "ext-mbstring": "*", 27 | "php": ">=5.3.0" 28 | }, 29 | "replace": { 30 | "videlalvaro/php-amqplib": "self.version" 31 | }, 32 | "require-dev": { 33 | "phpunit/phpunit": "^4.8", 34 | "scrutinizer/ocular": "^1.1", 35 | "squizlabs/php_codesniffer": "^2.5" 36 | }, 37 | "suggest": { 38 | "ext-sockets": "Use AMQPSocketConnection" 39 | }, 40 | "type": "library", 41 | "extra": { 42 | "branch-alias": { 43 | "dev-master": "2.7-dev" 44 | } 45 | }, 46 | "autoload": { 47 | "psr-4": { 48 | "PhpAmqpLib\\": "PhpAmqpLib/" 49 | } 50 | }, 51 | "notification-url": "https://packagist.org/downloads/", 52 | "license": [ 53 | "LGPL-2.1" 54 | ], 55 | "authors": [ 56 | { 57 | "name": "Alvaro Videla", 58 | "role": "Original Maintainer" 59 | }, 60 | { 61 | "name": "John Kelly", 62 | "email": "johnmkelly86@gmail.com", 63 | "role": "Maintainer" 64 | }, 65 | { 66 | "name": "Raúl Araya", 67 | "email": "nubeiro@gmail.com", 68 | "role": "Maintainer" 69 | } 70 | ], 71 | "description": "Formerly videlalvaro/php-amqplib. This library is a pure PHP implementation of the AMQP protocol. It's been tested against RabbitMQ.", 72 | "homepage": "https://github.com/php-amqplib/php-amqplib/", 73 | "keywords": [ 74 | "message", 75 | "queue", 76 | "rabbitmq" 77 | ], 78 | "time": "2016-04-11 14:30:01" 79 | }, 80 | { 81 | "name": "symfony/yaml", 82 | "version": "v3.0.1", 83 | "source": { 84 | "type": "git", 85 | "url": "https://github.com/symfony/yaml.git", 86 | "reference": "3df409958a646dad2bc5046c3fb671ee24a1a691" 87 | }, 88 | "dist": { 89 | "type": "zip", 90 | "url": "https://api.github.com/repos/symfony/yaml/zipball/3df409958a646dad2bc5046c3fb671ee24a1a691", 91 | "reference": "3df409958a646dad2bc5046c3fb671ee24a1a691", 92 | "shasum": "" 93 | }, 94 | "require": { 95 | "php": ">=5.5.9" 96 | }, 97 | "type": "library", 98 | "extra": { 99 | "branch-alias": { 100 | "dev-master": "3.0-dev" 101 | } 102 | }, 103 | "autoload": { 104 | "psr-4": { 105 | "Symfony\\Component\\Yaml\\": "" 106 | }, 107 | "exclude-from-classmap": [ 108 | "/Tests/" 109 | ] 110 | }, 111 | "notification-url": "https://packagist.org/downloads/", 112 | "license": [ 113 | "MIT" 114 | ], 115 | "authors": [ 116 | { 117 | "name": "Fabien Potencier", 118 | "email": "fabien@symfony.com" 119 | }, 120 | { 121 | "name": "Symfony Community", 122 | "homepage": "https://symfony.com/contributors" 123 | } 124 | ], 125 | "description": "Symfony Yaml Component", 126 | "homepage": "https://symfony.com", 127 | "time": "2015-12-26 13:39:53" 128 | }, 129 | { 130 | "name": "webreactor/cli-arguments", 131 | "version": "dev-master", 132 | "source": { 133 | "type": "git", 134 | "url": "https://github.com/webreactor/cli-arguments.git", 135 | "reference": "bab7e136ef88f982f6882c4cf12e40b72df38595" 136 | }, 137 | "dist": { 138 | "type": "zip", 139 | "url": "https://api.github.com/repos/webreactor/cli-arguments/zipball/bab7e136ef88f982f6882c4cf12e40b72df38595", 140 | "reference": "bab7e136ef88f982f6882c4cf12e40b72df38595", 141 | "shasum": "" 142 | }, 143 | "type": "library", 144 | "autoload": { 145 | "psr-4": { 146 | "Reactor\\CliArguments\\": "src/" 147 | } 148 | }, 149 | "license": [ 150 | "MIT" 151 | ], 152 | "support": { 153 | "source": "https://github.com/webreactor/cli-arguments/tree/master", 154 | "issues": "https://github.com/webreactor/cli-arguments/issues" 155 | }, 156 | "time": "2016-06-24 03:10:53" 157 | } 158 | ], 159 | "packages-dev": [], 160 | "aliases": [], 161 | "minimum-stability": "stable", 162 | "stability-flags": { 163 | "webreactor/cli-arguments": 20 164 | }, 165 | "prefer-stable": false, 166 | "prefer-lowest": false, 167 | "platform": [], 168 | "platform-dev": [] 169 | } 170 | -------------------------------------------------------------------------------- /src/Application.php: -------------------------------------------------------------------------------- 1 | queue_list = array(); 22 | } 23 | 24 | function setCredentials($host, $port, $bport, $user, $pass) { 25 | $this->host = $host; 26 | $this->port = $port; 27 | $this->bport = $bport; 28 | $this->user = $user; 29 | $this->pass = $pass; 30 | } 31 | 32 | function close($delete_exchange = false) { 33 | if ($delete_exchange) { 34 | try { 35 | if (!empty($this->channel)) { 36 | $this->channel->exchange_delete($this->exchange); 37 | } 38 | } catch (AMQPRuntimeException $e) {} 39 | } 40 | if (!empty($this->connection)) { 41 | $this->connection->close(); 42 | } 43 | if (!empty($this->channel)) { 44 | $this->channel->close(); 45 | } 46 | } 47 | 48 | function connectVhost($vhost, $temp_exchange = false) { 49 | if ($this->vhost === $vhost) { 50 | return $this->getQueueList($vhost); 51 | } 52 | $this->close($temp_exchange); 53 | $this->vhost = $vhost; 54 | $this->connection = new AMQPStreamConnection( 55 | $this->host, 56 | $this->bport, 57 | $this->user, 58 | $this->pass, 59 | $vhost 60 | ); 61 | $this->channel = $this->connection->channel(); 62 | $this->channel->basic_qos(0, 10, false); 63 | if ($temp_exchange) { 64 | try { 65 | $this->channel->exchange_delete($this->exchange); 66 | } catch (AMQPRuntimeException $e) {} 67 | $this->channel->exchange_declare($this->exchange, 'fanout', false, false, false); 68 | } 69 | return $this->getQueueList($vhost); 70 | } 71 | 72 | function rmqAPICall($command) { 73 | $command = array_map('rawurlencode', (array)$command); 74 | $command = implode('/', $command); 75 | $json = file_get_contents( 76 | 'http://'. 77 | rawurlencode($this->user).':'. 78 | rawurlencode($this->pass). 79 | '@'.$this->host.':'.$this->port.'/api/'. 80 | $command 81 | ); 82 | $list = json_decode($json, true); 83 | if (empty($list) || isset($list['error'])) { 84 | throw new \Exception($json, 1); 85 | } 86 | return $list; 87 | } 88 | 89 | 90 | function getVhostList() { 91 | $list = $this->rmqAPICall("vhosts"); 92 | $data = array(); 93 | foreach ($list as $key => $value) { 94 | $data[] = $value['name']; 95 | } 96 | return $data; 97 | } 98 | 99 | function getQueueList($vhost) { 100 | if (isset($this->queue_list[$vhost])) { 101 | return $this->queue_list[$vhost]; 102 | } 103 | $list = $this->rmqAPICall("queues"); 104 | $data = array(); 105 | foreach ($list as $key => $value) { 106 | if ($value['vhost'] === $vhost) { 107 | $data[$value['name']] = $value['messages']; 108 | } 109 | } 110 | $this->queue_list[$vhost] = $data; 111 | return $data; 112 | } 113 | 114 | function dumpQueue($queue_name, $ack) { 115 | return new CallbackStream(function () use ($queue_name, $ack) { 116 | $msg = $this->channel->basic_get($queue_name, $ack); 117 | return $this->messageHandler($msg, $queue_name); 118 | }); 119 | } 120 | 121 | function messageHandler($msg, $queue) { 122 | if (empty($msg)) { 123 | return null; 124 | } 125 | $headers = array(); 126 | $props = $msg->get_properties(); 127 | if (isset($props['application_headers'])) { 128 | $headers = $props['application_headers']->getNativeData(); 129 | } 130 | unset($props['application_headers']); 131 | $message_data = array( 132 | 'vhost' => $this->vhost, 133 | 'queue' => $queue, 134 | 'exchange' => $msg->delivery_info['exchange'], 135 | 'routing_key' => $msg->delivery_info['routing_key'], 136 | 'headers' => $headers, 137 | 'properties' => $props, 138 | 'body' => $msg->getBody(), 139 | ); 140 | return $message_data; 141 | } 142 | 143 | function configureExhangeWith($queue) { 144 | if ($this->current_queue === $queue) { 145 | return; 146 | } 147 | if ($this->current_queue !== null) { 148 | $this->channel->queue_unbind($this->current_queue, $this->exchange); 149 | } 150 | if ($this->auto_declare) { 151 | $this->channel->queue_declare( 152 | $queue, 153 | false, //passive 154 | true, //durable 155 | false, //exclusive 156 | false //autodelete 157 | ); 158 | } 159 | $this->channel->queue_bind($queue, $this->exchange); 160 | $this->current_queue = $queue; 161 | } 162 | 163 | function loadMessage($message_data, $dry_run = false) { 164 | $queues = $this->connectVhost($message_data['vhost'], true); 165 | if (!isset($queues[$message_data['queue']]) && !$this->auto_declare) { 166 | throw new \Exception("Missing queue: {$message_data['vhost']}:{$message_data['queue']}. Try add --declare", 1); 167 | } 168 | if (!$dry_run) { 169 | $this->configureExhangeWith($message_data['queue'], $dry_run); 170 | $msg = new AMQPMessage($message_data['body'], $message_data['properties']); 171 | $headers = new AMQPTable($message_data['headers']); 172 | $msg->set('application_headers', $headers); 173 | $this->channel->basic_publish($msg, $this->exchange, $message_data['routing_key']); 174 | } 175 | } 176 | 177 | } 178 | -------------------------------------------------------------------------------- /src/CallbackStream.php: -------------------------------------------------------------------------------- 1 | getter = $getter; 13 | } 14 | 15 | public function current() { 16 | return $this->current; 17 | } 18 | 19 | public function key() { 20 | return $this->key; 21 | } 22 | 23 | public function next() { 24 | $this->current = call_user_func($this->getter); 25 | $this->key++; 26 | } 27 | 28 | public function rewind() { 29 | $this->next(); 30 | $this->key = 0; 31 | } 32 | 33 | public function valid() { 34 | return !empty($this->current); 35 | } 36 | 37 | } -------------------------------------------------------------------------------- /src/CliController.php: -------------------------------------------------------------------------------- 1 | app = $app; 9 | } 10 | 11 | function handle($arguments) { 12 | $this->arguments = $arguments; 13 | $args = $arguments->get('_words_'); 14 | if (isset($args[1])) { 15 | $command = $args[1]; 16 | } else { 17 | $command = 'help'; 18 | } 19 | $command_handler = 'command'.ucfirst(strtolower($command)); 20 | if (method_exists($this, $command_handler)) { 21 | $this->app->setCredentials( 22 | $arguments->get('host'), 23 | $arguments->get('port'), 24 | $arguments->get('binary-port'), 25 | $arguments->get('user'), 26 | $arguments->get('pass') 27 | ); 28 | $this->app->auto_declare = ($arguments->get('declare') == true); 29 | try { 30 | call_user_func_array(array($this, $command_handler), []); 31 | } catch (\PhpAmqpLib\Exception\AMQPProtocolChannelException $e) { 32 | fwrite(STDERR, $e->getMessage()."\n"); 33 | exit(1); 34 | } catch (\Exception $e) { 35 | fwrite(STDERR, $e->getMessage()."\n"); 36 | exit(1); 37 | } 38 | } else { 39 | $this->commandHelp(); 40 | } 41 | } 42 | 43 | function commandHelp() { 44 | 45 | echo "RabbitMQ messages backup tool version ".$this->app->version."\n\n"; 46 | 47 | echo " Can dump messages to json file, load them back. Can modify destination vhost or queue name.\n"; 48 | echo " To backup exchanges, queues, bindings use RMQ webinterface.\n\n"; 49 | 50 | echo "Usage:\n"; 51 | echo " rmq-dump command \n\n"; 52 | 53 | echo "Commands:\n"; 54 | echo " dump - dumps messages from RMQ to STDOUT\n"; 55 | echo " load - loads messages from STDIN to RMQ\n"; 56 | echo " dryload - Dry run load will show how -v -s -a optionas will affect messages\n"; 57 | echo " list - shows current state in RMQ with -v -s filters. Use as dry run for dump\n"; 58 | echo " help - prints help\n\n"; 59 | 60 | echo "Arguments:\n"; 61 | echo " Full name | Short | Default | Note\n"; 62 | echo "-------------------------------------------------------\n"; 63 | 64 | foreach ($this->arguments->definitions as $key => $definition) { 65 | if ($key != '_words_') { 66 | if (!$definition->is_flag) { 67 | echo sprintf(" --%-12s -%-6s %-20s %s\n", 68 | $definition->name, 69 | $definition->short, 70 | $definition->default, 71 | $definition->description 72 | ); 73 | } else { 74 | echo sprintf(" --%-12s -%-6s %-20s %s\n", 75 | $definition->name, 76 | $definition->short, 77 | 'false', 78 | $definition->description 79 | ); 80 | } 81 | } 82 | } 83 | echo "\n"; 84 | echo "Examples:\n"; 85 | 86 | echo "Make dump all messages of all vhosts:\n"; 87 | echo " rmq-dump dump -H host -u user -p password > dump.json\n\n"; 88 | 89 | echo "Make dump all messages of vhost1:\n"; 90 | echo " rmq-dump dump -H host -u user -p password -v vhost1 > dump.json\n\n"; 91 | 92 | echo "Load dump:\n"; 93 | echo " cat dump.json | rmq-dump load -H host -u user -p password \n\n"; 94 | 95 | echo "Load to vhost1 and create queues if needed:\n"; 96 | echo " cat dump.json | rmq-dump load -H host -u user -p password -d -a vhost1 \n\n"; 97 | 98 | echo "Load only messages from vhost1 to vhost2:\n"; 99 | echo " cat dump.json | rmq-dump load -H host -u user -p password -v host1 -a vhost2 \n\n"; 100 | 101 | echo "Dry load to check you filters\n"; 102 | echo " cat dump.json | rmq-dump dryload -H host -u user -p password -v host1 -a vhost2 \n\n"; 103 | 104 | echo "Copy all messages from queue1 to queue2 qithout storing them\n"; 105 | echo "rmq-dump -u user -p pass -v /app/live:queue1 dump | ./rmq-dump -u user -p pass -a :queue2 load\n\n"; 106 | 107 | echo "Copy all messages from vhost1 to vhost2 not storing them. Note: all queues have exists at vhost2\n"; 108 | echo "rmq-dump -u user -p pass -v /vhost1 dump | ./rmq-dump -u user -p pass -a /vhost2 load\n\n"; 109 | } 110 | 111 | function commandList() { 112 | $to_dump = $this->parseVhostsArg($this->arguments->get('vhost')); 113 | $to_skip = $this->parseVhostsArg($this->arguments->get('skip')); 114 | $vhosts = $this->app->getVhostList(); 115 | $cnt_total = 0; 116 | foreach ($vhosts as $vhost) { 117 | if ($this->matchVhostQueue($vhost, null, $to_skip) || !$this->matchVhostQueue($vhost, null, $to_dump, true)) { 118 | continue; 119 | } 120 | 121 | echo "$vhost\n"; 122 | $queues = $this->app->getQueueList($vhost); 123 | foreach ($queues as $queue => $messages) { 124 | if ($this->matchVhostQueue($vhost, $queue, $to_dump, true)) { 125 | if (!$this->matchVhostQueue($vhost, $queue, $to_skip)) { 126 | echo " $queue $messages\n"; 127 | $cnt_total += $messages; 128 | } else { 129 | echo " $queue skipped\n"; 130 | } 131 | } 132 | } 133 | } 134 | echo "Total: $cnt_total\n"; 135 | } 136 | 137 | function commandDump() { 138 | $to_dump = $this->parseVhostsArg($this->arguments->get('vhost')); 139 | $to_skip = $this->parseVhostsArg($this->arguments->get('skip')); 140 | $to_alter = $this->parseAlternation($this->arguments->get('alter')); 141 | $ack = ($this->arguments->get('ack') == true); 142 | fwrite(STDERR, "Dumping with ack ".($ack?'true':'false')." \n"); 143 | $cnt_total = 0; 144 | $this->printAlter($to_alter); 145 | foreach ($this->app->getVhostList() as $vhost) { 146 | $cnt_total += $this->dumpVhost($vhost, $to_dump, $to_skip, $to_alter, $ack); 147 | } 148 | fwrite(STDERR, "Total: $cnt_total\n"); 149 | $this->app->close(); 150 | } 151 | 152 | function dumpVhost($vhost, $to_dump, $to_skip, $to_alter, $ack) { 153 | if ($this->matchVhostQueue($vhost, null, $to_skip) || !$this->matchVhostQueue($vhost, null, $to_dump, true)) { 154 | return 0; 155 | } 156 | $queues = $this->app->connectVhost($vhost); 157 | fwrite(STDERR, "$vhost \n"); 158 | 159 | $cnt_total = 0; 160 | foreach ($queues as $queue => $size) { 161 | if ($this->matchVhostQueue($vhost, $queue, $to_dump, true)) { 162 | if ($this->matchVhostQueue($vhost, $queue, $to_skip)) { 163 | fwrite(STDERR, " $queue skipped\n"); 164 | } elseif (!($size > 0)) { 165 | fwrite(STDERR, " $queue empty\n"); 166 | } else { 167 | $cnt_total += $this->dumpQueue($queue, $size, $to_alter, $ack); 168 | } 169 | } 170 | } 171 | return $cnt_total; 172 | } 173 | 174 | function printAlter($alters) { 175 | foreach($alters as $alternation) { 176 | $src = $alternation['source']; 177 | $dst = $alternation['destination']; 178 | fwrite(STDERR, "Alternation: {$src['vhost']}:{$src['queue']} to {$dst['vhost']}:{$dst['queue']}\n"); 179 | } 180 | } 181 | 182 | function dumpQueue($queue, $expected, $to_alter, $ack) { 183 | $cnt_total = 0; 184 | foreach ($this->app->dumpQueue($queue, $ack) as $key => $message) { 185 | echo json_encode($this->alterMessage($message, $to_alter))."\n"; 186 | $cnt_total++; 187 | fwrite(STDERR, " $queue $cnt_total of $expected\r"); 188 | } 189 | fwrite(STDERR, " $queue $cnt_total of $expected\n"); 190 | return $cnt_total; 191 | } 192 | 193 | function commandDryload() { 194 | $this->commandLoad(true); 195 | } 196 | 197 | function commandLoad($_dry_run = false) { 198 | $cnt_total = $cnt = 0; 199 | $filters = $this->arguments->getAll(); 200 | $queue = $vhost = null; 201 | $to_dump = $this->parseVhostsArg($this->arguments->get('vhost')); 202 | $to_skip = $this->parseVhostsArg($this->arguments->get('skip')); 203 | $to_alter = $this->parseAlternation($this->arguments->get('alter')); 204 | $this->printAlter($to_alter); 205 | while ($t = fgets(STDIN)) { 206 | $message = json_decode($t, true); 207 | if ($this->matchVhostQueue($message['vhost'], $message['queue'], $to_dump, true) && !$this->matchVhostQueue($message['vhost'], $message['queue'], $to_skip)) { 208 | $message = $this->alterMessage($message, $to_alter); 209 | $this->app->loadMessage($message, $_dry_run); 210 | if ($queue != $message['queue'] || $vhost != $message['vhost']) { 211 | if ($queue !== null) { 212 | echo " $queue $cnt\n"; 213 | } 214 | if ($vhost != $message['vhost']) { 215 | $vhost = $message['vhost']; 216 | echo "$vhost\n"; 217 | } 218 | $queue = $message['queue']; 219 | $cnt = 0; 220 | } 221 | echo " $queue $cnt\r"; 222 | $cnt++; 223 | $cnt_total++; 224 | } 225 | } 226 | if ($cnt_total > 0) { 227 | echo " $queue $cnt\n"; 228 | } 229 | echo "Total: $cnt_total\n"; 230 | $this->app->close(true); 231 | } 232 | 233 | function alterMessage($message, $alters) { 234 | foreach($alters as $alternation) { 235 | if ($this->matchesAlternation($message, $alternation['source'])) { 236 | $destination = $alternation['destination']; 237 | if (!empty($destination['vhost'])) { 238 | $message['vhost'] = $destination['vhost']; 239 | } 240 | if (!empty($destination['queue'])) { 241 | $message['queue'] = $destination['queue']; 242 | } 243 | } 244 | } 245 | return $message; 246 | } 247 | 248 | function matchesAlternation($message, $vhost_queue) { 249 | if (!empty($vhost_queue['vhost']) && $message['vhost'] != $vhost_queue['vhost']) { 250 | return false; 251 | } 252 | if (!empty($vhost_queue['queue']) && $message['queue'] != $vhost_queue['queue']) { 253 | return false; 254 | } 255 | return true; 256 | } 257 | 258 | function parseVhostsArg($raw) { 259 | $data = array( 260 | 'vhosts' => array(), 261 | 'queues' => array(), 262 | ); 263 | foreach ($raw as $value) { 264 | $value = $this->parseVhostQueueName($value); 265 | if (empty($value['vhost'])) { 266 | $data['queues'][] = $value['queue']; 267 | } else { 268 | if (empty($value['queue'])) { 269 | $data['vhosts'][$value['vhost']] = array(); 270 | } else { 271 | $data['vhosts'][$value['vhost']][] = $value['queue']; 272 | } 273 | } 274 | } 275 | return $data; 276 | } 277 | 278 | function parseVhostQueueName($name) { 279 | $name = explode(':', $name); 280 | if (!isset($name[1])) { 281 | $name[1] = null; 282 | } 283 | $rez = array( 284 | 'vhost' => $name[0], 285 | 'queue' => $name[1], 286 | ); 287 | return $rez; 288 | } 289 | 290 | function parseAlternation($raw) { 291 | $data = array(); 292 | foreach ($raw as $value) { 293 | $value = explode('~', $value); 294 | if (!isset($value[1])) { 295 | $source = null; 296 | $destination = $this->parseVhostQueueName($value[0]); 297 | } else { 298 | $source = $this->parseVhostQueueName($value[0]); 299 | $destination = $this->parseVhostQueueName($value[1]); 300 | } 301 | $data[] = array( 302 | 'source' => $source, 303 | 'destination' => $destination, 304 | ); 305 | } 306 | return $data; 307 | } 308 | 309 | function matchVhostQueue($vhost, $queue, $stack, $empty_is_match = false) { 310 | if (empty($stack['vhosts']) && empty($stack['queues']) && $empty_is_match) { 311 | return true; 312 | } 313 | if (empty($stack['vhosts']) && $queue == null && $empty_is_match) { 314 | return true; 315 | } 316 | if (in_array($queue, $stack['queues'])) { 317 | return true; 318 | } 319 | 320 | if (isset($stack['vhosts'][$vhost])) { 321 | if (in_array($queue,$stack['vhosts'][$vhost])) { 322 | return true; 323 | } 324 | if ($queue == null) { 325 | return true; 326 | } 327 | if (empty($stack['vhosts'][$vhost])) { 328 | return true; 329 | } 330 | } 331 | return false; 332 | } 333 | 334 | } 335 | -------------------------------------------------------------------------------- /src/cli.php: -------------------------------------------------------------------------------- 1 | addDefinition(new ArgumentDefinition('_words_', '', null, false, true, '')); 14 | $argumets->addDefinition(new ArgumentDefinition('host', 'H', 'localhost')); 15 | $argumets->addDefinition(new ArgumentDefinition('port', 'P', '15672')); 16 | $argumets->addDefinition(new ArgumentDefinition('binary-port', 'B', '5672')); 17 | $argumets->addDefinition(new ArgumentDefinition('user', 'u')); 18 | $argumets->addDefinition(new ArgumentDefinition('pass', 'p')); 19 | $argumets->addDefinition(new ArgumentDefinition('vhost', 'v', null, false, true, 'vhost[:queue] | :queue')); 20 | $argumets->addDefinition(new ArgumentDefinition('skip', 's', null, false, true, 'vhost[:queue] | :queue')); 21 | $argumets->addDefinition(new ArgumentDefinition('alter', 'a', null, false, true, 'vhost[:queue]~vhost[:queue] | vhost[:queue]')); 22 | $argumets->addDefinition(new ArgumentDefinition('ack', 'k', true, true, false, 'acknowlege (delete) messages when dump')); 23 | $argumets->addDefinition(new ArgumentDefinition('declare', 'd', true, true, false, 'declare persistent queues when load')); 24 | $argumets->parse(); 25 | 26 | $app = new Application(); 27 | 28 | $cli_controller = new CliController($app); 29 | $cli_controller->handle($argumets); 30 | -------------------------------------------------------------------------------- /src/web.php: -------------------------------------------------------------------------------- 1 |