├── LICENSE.md ├── CONTRIBUTING.md ├── src └── main │ └── java │ └── org │ └── voltdb │ └── exportclient │ ├── BackOffFactory.java │ ├── FirehoseExportException.java │ ├── BackOff.java │ ├── FirehoseExportLogger.java │ ├── FirehoseSink.java │ └── KinesisFirehoseExportClient.java ├── README.md └── .gitignore /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | ===================== 3 | 4 | Copyright © 2015 VoltDB Inc. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining 7 | a copy of this software and associated documentation files (the 8 | “Software”), to deal in the Software without restriction, including 9 | without limitation the rights to use, copy, modify, merge, publish, 10 | distribute, sublicense, and/or sell copies of the Software, and to 11 | permit persons to whom the Software is furnished to do so, subject to 12 | the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for your interest in contributing. VoltDB uses GitHub to manage reviews of pull requests. We welcome your contributions to VoltDB or to any of the related libraries and tools. 4 | 5 | VoltDB uses the standard github workflow, meaning we make use of forking and pull requests. 6 | 7 | * If you have a trivial fix or improvement, go ahead and create a pull request. 8 | 9 | * If you are interested in contributing something more involved, feel free to discuss your ideas in [VoltDB Public](http://chat.voltdb.com/) on Slack. 10 | 11 | ## Contributor License Agreement 12 | 13 | In order to contribute code to VoltDB, you must first sign the [VoltDB Contributor License Agreement (CLA)](https://www.voltdb.com/contributor-license-agreement/) and email an image or PDF of the document including your hand-written signature to [support@voltdb.com](mailto:support@voltdb.com). VoltDB will sign and return the final copy of the agreement for your records. 14 | 15 | ## How to submit code 16 | 17 | The workflow is essentially the following: 18 | 19 | 1. Fork the VoltDB project 20 | 2. Make a branch. Commit your changes to this branch. (See note below) 21 | 3. Issue a pull request on the VoltDB repository. 22 | 23 | Once you have signed the CLA, a VoltDB engineer will review your pull request. 24 | 25 | Note: 26 | 27 | It will be easier to keep your work merge-able (conflict-free) if you don't work directly on your master branch in your VoltDB fork. Rather, keep your master branch in sync with the VoltDB repository and apply your changes on a branch of your fork. 28 | 29 | For further reading: 30 | 31 | * [How to fork a GitHub repository](https://help.github.com/articles/fork-a-repo) 32 | * [Using pull requests](https://help.github.com/articles/using-pull-requests/) 33 | 34 | ## Additional Resources 35 | 36 | * [VoltDB Wiki](https://github.com/VoltDB/voltdb/wiki) on Github 37 | * [VoltDB Public](http://chat.voltdb.com/) on Slack 38 | * [VoltDB Community Forum](https://forum.voltdb.com/) 39 | -------------------------------------------------------------------------------- /src/main/java/org/voltdb/exportclient/BackOffFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License (MIT) 3 | * 4 | * Copyright (C) 2008-2018 VoltDB Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package org.voltdb.exportclient; 26 | 27 | import java.lang.reflect.Constructor; 28 | import org.hsqldb_voltpatches.lib.StringUtil; 29 | import com.google_voltpatches.common.base.Throwables; 30 | 31 | public class BackOffFactory { 32 | public static BackOff getBackOff(String backOffType, int backOffBase, int backOffCap) { 33 | switch (backOffType) { 34 | case "full": 35 | return new ExpoBackOffFullJitter(backOffBase, backOffCap); 36 | case "equal": 37 | return new ExpoBackOffEqualJitter(backOffBase, backOffCap); 38 | case "decor": 39 | return new ExpoBackOffDecor(backOffBase, backOffCap); 40 | default: 41 | if(!StringUtil.isEmpty(backOffType)){ 42 | try { 43 | Constructor c = Class.forName(backOffType).getConstructor(Integer.TYPE, Integer.TYPE); 44 | BackOff backoff = (BackOff) c.newInstance(backOffBase, backOffCap); 45 | return backoff; 46 | } catch(Throwable t) { 47 | Throwables.propagate(t); 48 | } 49 | } 50 | return new ExpoBackOffDecor(backOffBase, backOffCap); 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /src/main/java/org/voltdb/exportclient/FirehoseExportException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License (MIT) 3 | * 4 | * Copyright (C) 2008-2018 VoltDB Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package org.voltdb.exportclient; 26 | 27 | import java.util.Arrays; 28 | import java.util.IllegalFormatConversionException; 29 | import java.util.MissingFormatArgumentException; 30 | import java.util.UnknownFormatConversionException; 31 | 32 | public class FirehoseExportException extends RuntimeException { 33 | 34 | private static final long serialVersionUID = 4260074108700559787L; 35 | 36 | public FirehoseExportException() { 37 | } 38 | 39 | public FirehoseExportException(String format, Object...args) { 40 | super(format(format, args)); 41 | } 42 | 43 | public FirehoseExportException(Throwable cause) { 44 | super(cause); 45 | } 46 | 47 | public FirehoseExportException(String format, Throwable cause, Object...args) { 48 | super(format(format, args), cause); 49 | } 50 | 51 | static protected String format(String format, Object...args) { 52 | String formatted = null; 53 | try { 54 | formatted = String.format(format, args); 55 | } catch (MissingFormatArgumentException|IllegalFormatConversionException| 56 | UnknownFormatConversionException ignoreThem) { 57 | } 58 | finally { 59 | if (formatted == null) { 60 | formatted = "Format: " + format + ", arguments: " + Arrays.toString(args); 61 | } 62 | } 63 | return formatted; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/org/voltdb/exportclient/BackOff.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License (MIT) 3 | * 4 | * Copyright (C) 2008-2018 VoltDB Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package org.voltdb.exportclient; 26 | 27 | import java.util.concurrent.ThreadLocalRandom; 28 | 29 | abstract class BackOff { 30 | int base; 31 | int cap; 32 | 33 | public BackOff(int base, int cap) { 34 | this.base = base; 35 | this.cap = cap; 36 | } 37 | 38 | public int expo(int n) { 39 | return Math.min(cap, base * (1< gradle.properties 32 | ``` 33 | 34 | * Invoke gradle to compile artifacts 35 | 36 | ```bash 37 | gradle shadowJar 38 | ``` 39 | 40 | * To setup an eclipse project run gradle as follows 41 | 42 | ```bash 43 | gradle cleanEclipse eclipse 44 | ``` 45 | then import it into your eclipse workspace by using File->Import projects menu option 46 | 47 | ## Configuration 48 | 49 | * Copy the built jar from `build/libs` to `lib/extension` under your VoltDB installation directory 50 | 51 | * Edit your deployment file and use the following export XML stanza as a template 52 | 53 | ```xml 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 63 | us-east-1 64 | streamtest 65 | 66 | 67 | 68 | 69 | 70 | ``` 71 | 72 | This tells VoltDB to write to the alerts stream and send the content to the Amazon Kinesis Firehose stream 73 | with the name streamtest. If the client created with the supplied access.key and secret.key have access 74 | to this stream then this stream will be successfully created. In this example we create the VoltDB export 75 | with the definition: 76 | 77 | ```sql 78 | CREATE STREAM alerts EXPORT TO TARGET default ( 79 | id integer not null, 80 | msg varchar(128), 81 | continent varchar(64), 82 | country varchar(64) 83 | ); 84 | ``` 85 | 86 | Then data can be inserted into this export stream using the command: 87 | 88 | ```sql 89 | INSERT INTO ALERTS (ID,MSG,CONTINENT,COUNTRY) VALUES (1,'fab-02 inoperable','EU','IT'); 90 | ``` 91 | 92 | ## Configuration Properties 93 | 94 | - `region` (mandatory) designates the AWS region where the Kinesis Firehose stream is defined 95 | - `stream.name` (mandatory) Kinesis Firehose stream name 96 | - `access.key` (mandatory) user's access key 97 | - `secret.key` (mandatory) user's secret key 98 | - `timezone` (optional, _default:_ local timezone) timezone used to format timestamp values 99 | - `stream.limit` (optional, _default:_ 5000 records/s) Firehose Delivery Stream limit on AWS side, i.e. how many records per second it can accept 100 | - `concurrent.writers` (optional, _default:_ 1) The number of writers in the concurrent writer pool, sharing the connection to AWS 101 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | **/build/ 3 | **/tmp/ 4 | loader.cfg 5 | 6 | # Ignore Gradle GUI config 7 | gradle-app.setting 8 | *.class 9 | 10 | # Generated Eclipse stuff 11 | .project 12 | .classpath 13 | 14 | # Package Files # 15 | *.jar 16 | *.war 17 | *.ear 18 | 19 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 20 | hs_err_pid* 21 | 22 | .DS_Store 23 | .AppleDouble 24 | .LSOverride 25 | 26 | # Icon must end with two \r 27 | Icon 28 | 29 | 30 | # Thumbnails 31 | ._* 32 | 33 | # Files that might appear on external disk 34 | .Spotlight-V100 35 | .Trashes 36 | 37 | # Directories potentially created on remote AFP share 38 | .AppleDB 39 | .AppleDesktop 40 | Network Trash Folder 41 | Temporary Items 42 | .apdisk 43 | 44 | *~ 45 | \#*\# 46 | /.emacs.desktop 47 | /.emacs.desktop.lock 48 | *.elc 49 | auto-save-list 50 | tramp 51 | .\#* 52 | 53 | # Org-mode 54 | .org-id-locations 55 | *_archive 56 | 57 | # flymake-mode 58 | *_flymake.* 59 | 60 | # eshell files 61 | /eshell/history 62 | /eshell/lastdir 63 | 64 | # elpa packages 65 | /elpa/ 66 | 67 | # reftex files 68 | *.rel 69 | 70 | # AUCTeX auto folder 71 | /auto/ 72 | 73 | # cask packages 74 | .cask/ 75 | 76 | # cache files for sublime text 77 | *.tmlanguage.cache 78 | *.tmPreferences.cache 79 | *.stTheme.cache 80 | 81 | # workspace files are user-specific 82 | *.sublime-workspace 83 | 84 | # project files should be checked into the repository, unless a significant 85 | # proportion of contributors will probably not be using SublimeText 86 | # *.sublime-project 87 | 88 | # sftp configuration file 89 | sftp-config.json 90 | 91 | # KDE directory preferences 92 | .directory 93 | 94 | *.pydevproject 95 | .metadata 96 | .gradle 97 | gradle.properties 98 | bin/ 99 | obj/ 100 | tmp/ 101 | *.tmp 102 | *.bak 103 | *.swp 104 | *~.nib 105 | local.properties 106 | .settings/ 107 | .loadpath 108 | 109 | # External tool builders 110 | .externalToolBuilders/ 111 | 112 | # Locally stored "Eclipse launch configurations" 113 | *.launch 114 | 115 | # CDT-specific 116 | .cproject 117 | 118 | # PDT-specific 119 | .buildpath 120 | 121 | # sbteclipse plugin 122 | .target 123 | 124 | # TeXlipse plugin 125 | .texlipse 126 | 127 | nbproject/private/ 128 | build/ 129 | nbbuild/ 130 | dist/ 131 | nbdist/ 132 | nbactions.xml 133 | nb-configuration.xml 134 | 135 | # It's better to unpack these files and commit the raw source because 136 | # git has its own built in compression methods. 137 | *.7z 138 | *.jar 139 | *.rar 140 | *.zip 141 | *.gz 142 | *.bzip 143 | #*.bz2 144 | *.xz 145 | *.lzma 146 | *.cab 147 | 148 | #packing-only formats 149 | *.iso 150 | *.tar 151 | 152 | #package management formats 153 | *.dmg 154 | *.xpi 155 | *.gem 156 | *.egg 157 | *.deb 158 | *.rpm 159 | *.msi 160 | *.msm 161 | *.msp 162 | 163 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 164 | 165 | *.iml 166 | 167 | ## Directory-based project format: 168 | .idea/ 169 | # if you remove the above rule, at least ignore the following: 170 | 171 | # User-specific stuff: 172 | # .idea/workspace.xml 173 | # .idea/tasks.xml 174 | # .idea/dictionaries 175 | 176 | # Sensitive or high-churn files: 177 | # .idea/dataSources.ids 178 | # .idea/dataSources.xml 179 | # .idea/sqlDataSources.xml 180 | # .idea/dynamic.xml 181 | # .idea/uiDesigner.xml 182 | 183 | # Gradle: 184 | # .idea/gradle.xml 185 | # .idea/libraries 186 | 187 | # Mongo Explorer plugin: 188 | # .idea/mongoSettings.xml 189 | 190 | ## File-based project format: 191 | *.ipr 192 | *.iws 193 | 194 | ## Plugin-specific files: 195 | 196 | # IntelliJ 197 | out/ 198 | 199 | # mpeltonen/sbt-idea plugin 200 | .idea_modules/ 201 | 202 | # JIRA plugin 203 | atlassian-ide-plugin.xml 204 | 205 | # Crashlytics plugin (for Android Studio and IntelliJ) 206 | com_crashlytics_export_strings.xml 207 | crashlytics.properties 208 | crashlytics-build.properties 209 | 210 | 211 | /.recommenders/ 212 | -------------------------------------------------------------------------------- /src/main/java/org/voltdb/exportclient/FirehoseExportLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License (MIT) 3 | * 4 | * Copyright (C) 2008-2018 VoltDB Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package org.voltdb.exportclient; 26 | 27 | import java.util.concurrent.TimeUnit; 28 | 29 | import org.voltcore.logging.Level; 30 | import org.voltcore.logging.VoltLogger; 31 | import org.voltcore.utils.EstTime; 32 | import org.voltcore.utils.RateLimitedLogger; 33 | 34 | public class FirehoseExportLogger { 35 | 36 | final static long SUPPRESS_INTERVAL = 10; 37 | final private VoltLogger m_logger = new VoltLogger("ExportClient"); 38 | 39 | public FirehoseExportLogger() { 40 | } 41 | 42 | private void log(Level level, Throwable cause, String format, Object...args) { 43 | RateLimitedLogger.tryLogForMessage( 44 | EstTime.currentTimeMillis(), 45 | SUPPRESS_INTERVAL, TimeUnit.SECONDS, 46 | m_logger, level, 47 | cause, format, args 48 | ); 49 | } 50 | 51 | public VoltLogger getLogger() { 52 | return m_logger; 53 | } 54 | 55 | public void trace(String format, Object...args) { 56 | if (m_logger.isTraceEnabled()) { 57 | log(Level.TRACE, null, format, args); 58 | } 59 | } 60 | 61 | public void debug(String format, Object...args) { 62 | if (m_logger.isDebugEnabled()) { 63 | log(Level.DEBUG, null, format, args); 64 | } 65 | } 66 | 67 | public void info(String format, Object...args) { 68 | if (m_logger.isInfoEnabled()) { 69 | log(Level.INFO, null, format, args); 70 | } 71 | } 72 | 73 | public void warn(String format, Object...args) { 74 | log(Level.WARN, null, format, args); 75 | } 76 | 77 | public void error(String format, Object...args) { 78 | log(Level.ERROR, null, format, args); 79 | } 80 | 81 | public void fatal(String format, Object...args) { 82 | log(Level.FATAL, null, format, args); 83 | } 84 | 85 | public void trace(String format, Throwable cause, Object...args) { 86 | if (m_logger.isTraceEnabled()) { 87 | log(Level.TRACE, cause, format, args); 88 | } 89 | } 90 | 91 | public void debug(String format, Throwable cause, Object...args) { 92 | if (m_logger.isDebugEnabled()) { 93 | log(Level.DEBUG, cause, format, args); 94 | } 95 | } 96 | 97 | public void info(String format, Throwable cause, Object...args) { 98 | if (m_logger.isInfoEnabled()) { 99 | log(Level.INFO, cause, format, args); 100 | } 101 | } 102 | 103 | public void warn(String format, Throwable cause, Object...args) { 104 | log(Level.WARN, cause, format, args); 105 | } 106 | 107 | public void error(String format, Throwable cause, Object...args) { 108 | log(Level.ERROR, cause, format, args); 109 | } 110 | 111 | public void fatal(String format, Throwable cause, Object...args) { 112 | log(Level.FATAL, cause, format, args); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/org/voltdb/exportclient/FirehoseSink.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License (MIT) 3 | * 4 | * Copyright (C) 2008-2018 VoltDB Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package org.voltdb.exportclient; 26 | 27 | import java.util.ArrayList; 28 | import java.util.List; 29 | import java.util.Queue; 30 | import java.util.concurrent.Callable; 31 | import java.util.concurrent.ExecutionException; 32 | import java.util.concurrent.ThreadLocalRandom; 33 | import java.util.concurrent.TimeUnit; 34 | import java.util.concurrent.atomic.AtomicInteger; 35 | 36 | import org.voltcore.utils.CoreUtils; 37 | 38 | import com.amazonaws.services.kinesisfirehose.AmazonKinesisFirehoseClient; 39 | import com.amazonaws.services.kinesisfirehose.model.PutRecordBatchRequest; 40 | import com.amazonaws.services.kinesisfirehose.model.PutRecordBatchResult; 41 | import com.amazonaws.services.kinesisfirehose.model.PutRecordRequest; 42 | import com.amazonaws.services.kinesisfirehose.model.Record; 43 | import com.amazonaws.services.kinesisfirehose.model.ServiceUnavailableException; 44 | import com.google_voltpatches.common.base.Throwables; 45 | import com.google_voltpatches.common.collect.ImmutableList; 46 | import com.google_voltpatches.common.util.concurrent.Futures; 47 | import com.google_voltpatches.common.util.concurrent.ListenableFuture; 48 | import com.google_voltpatches.common.util.concurrent.ListeningExecutorService; 49 | 50 | public class FirehoseSink { 51 | private final static FirehoseExportLogger LOG = new FirehoseExportLogger(); 52 | private final static int MAX_RETRY = 9; 53 | private final List m_executors; 54 | 55 | private final String m_streamName; 56 | private AmazonKinesisFirehoseClient m_client; 57 | private final int m_concurrentWriters; 58 | private final AtomicInteger m_backpressureIndication = new AtomicInteger(0); 59 | private BackOff m_backOff; 60 | public FirehoseSink(String streamName, AmazonKinesisFirehoseClient client, int concurrentWriters, BackOff backOff) { 61 | ImmutableList.Builder lbldr = ImmutableList.builder(); 62 | for (int i = 0; i < concurrentWriters; ++i) { 63 | String threadName = "Firehose Deliver Stream " + streamName + " Sink Writer " + i; 64 | lbldr.add(CoreUtils.getListeningSingleThreadExecutor(threadName, CoreUtils.MEDIUM_STACK_SIZE)); 65 | } 66 | m_executors = lbldr.build(); 67 | m_streamName = streamName; 68 | m_client = client; 69 | m_concurrentWriters = concurrentWriters; 70 | m_backOff = backOff; 71 | } 72 | 73 | ListenableFuture asWriteTask(List recordsList) { 74 | final int hashed = ThreadLocalRandom.current().nextInt(m_concurrentWriters); 75 | if (m_executors.get(hashed).isShutdown()) { 76 | return Futures.immediateFailedFuture(new FirehoseExportException("Firehose sink executor is shut down")); 77 | } 78 | return m_executors.get(hashed).submit(new Callable() { 79 | @Override 80 | public Void call() throws Exception { 81 | PutRecordBatchRequest batchRequest = new PutRecordBatchRequest().withDeliveryStreamName(m_streamName) 82 | .withRecords(recordsList); 83 | applyBackPressure(); 84 | PutRecordBatchResult res = m_client.putRecordBatch(batchRequest); 85 | if (res.getFailedPutCount() > 0) { 86 | setBackPressure(true); 87 | String msg = "%d Firehose records failed"; 88 | LOG.warn(msg, res.getFailedPutCount()); 89 | throw new FirehoseExportException(msg, res.getFailedPutCount()); 90 | } 91 | setBackPressure(false); 92 | return null; 93 | } 94 | }); 95 | } 96 | 97 | public void write(Queue> records) { 98 | List> tasks = new ArrayList<>(); 99 | for (List recordsList : records) { 100 | tasks.add(asWriteTask(recordsList)); 101 | } 102 | try { 103 | Futures.allAsList(tasks).get(); 104 | } catch (InterruptedException e) { 105 | String msg = "Interrupted write for message %s"; 106 | LOG.error(msg, e, records); 107 | throw new FirehoseExportException(msg, e, records); 108 | } catch (ExecutionException e) { 109 | if (e.getCause() instanceof FirehoseExportException) { 110 | throw (FirehoseExportException) e.getCause(); 111 | } 112 | String msg = "Fault on write for message %s"; 113 | LOG.error(msg, e, records); 114 | throw new FirehoseExportException(msg, e, records); 115 | } 116 | } 117 | 118 | public void writeRow(Record record){ 119 | int retry = MAX_RETRY; 120 | while (retry > 0){ 121 | try { 122 | PutRecordRequest putRecordRequest = new PutRecordRequest(); 123 | putRecordRequest.setDeliveryStreamName(m_streamName); 124 | putRecordRequest.setRecord(record); 125 | m_client.putRecord(putRecordRequest); 126 | } catch (ServiceUnavailableException e){ 127 | if(retry == 1){ 128 | throw new FirehoseExportException("Failed to send record", e, true); 129 | }else{ 130 | LOG.warn("Failed to send record: %s. Retry #%d", e.getErrorMessage(), (MAX_RETRY-retry + 1)); 131 | backoffSleep(retry); 132 | } 133 | } 134 | retry--; 135 | } 136 | } 137 | 138 | public void syncWrite(Queue> records) { 139 | 140 | for (List recordsList : records) { 141 | int retry = MAX_RETRY; 142 | while (retry > 0){ 143 | try { 144 | PutRecordBatchRequest batchRequest = new PutRecordBatchRequest().withDeliveryStreamName(m_streamName).withRecords(recordsList); 145 | PutRecordBatchResult res = m_client.putRecordBatch(batchRequest); 146 | if (res.getFailedPutCount() > 0) { 147 | String msg = "Records failed with the batch: %d, retry: #%d"; 148 | if(retry == 1){ 149 | throw new FirehoseExportException(msg, res.getFailedPutCount(), (MAX_RETRY-retry + 1)); 150 | }else{ 151 | LOG.warn(msg, res.getFailedPutCount(), (MAX_RETRY-retry + 1)); 152 | backoffSleep(retry); 153 | } 154 | }else{ 155 | recordsList.clear(); 156 | break; 157 | } 158 | } catch (ServiceUnavailableException e){ 159 | if(retry == 1){ 160 | throw new FirehoseExportException("Failed to send record batch", e, true); 161 | }else{ 162 | LOG.warn("Failed to send record batch: %s. Retry #%d", e.getErrorMessage(), (MAX_RETRY-retry + 1)); 163 | backoffSleep(retry); 164 | } 165 | } 166 | retry--; 167 | } 168 | } 169 | } 170 | 171 | private void backoffSleep(int seed) { 172 | try { 173 | int sleep = m_backOff.backoff(seed); 174 | Thread.sleep(sleep); 175 | LOG.warn("Sleep for back pressure for %d ms", sleep); 176 | } catch (InterruptedException e) { 177 | LOG.warn("Interrupted sleep: %s", e.getMessage()); 178 | } 179 | } 180 | 181 | private boolean setBackPressure(boolean b) { 182 | int prev = m_backpressureIndication.get(); 183 | int delta = b ? 1 : -(prev > 1 ? prev >> 1 : 1); 184 | int next = prev + delta; 185 | while (next >= 0 && !m_backpressureIndication.compareAndSet(prev, next)) { 186 | prev = m_backpressureIndication.get(); 187 | delta = b ? 1 : -(prev > 1 ? prev >> 1 : 1); 188 | next = prev + delta; 189 | } 190 | return b; 191 | } 192 | 193 | private void applyBackPressure() { 194 | int sleep = m_backOff.backoff(m_backpressureIndication.get()); 195 | LOG.warn("Sleep for back pressure for %d ms", sleep); 196 | try { 197 | Thread.sleep(sleep); 198 | } catch (InterruptedException e) { 199 | LOG.debug("Sleep for back pressure interrupted", e); 200 | } 201 | } 202 | 203 | public void shutDown(){ 204 | if(m_executors != null){ 205 | for(ListeningExecutorService srv : m_executors){ 206 | srv.shutdown(); 207 | try { 208 | srv.awaitTermination(365, TimeUnit.DAYS); 209 | } catch (InterruptedException e) { 210 | Throwables.propagate(e); 211 | } 212 | } 213 | } 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /src/main/java/org/voltdb/exportclient/KinesisFirehoseExportClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License (MIT) 3 | * 4 | * Copyright (C) 2008-2018 VoltDB Inc. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package org.voltdb.exportclient; 26 | 27 | import java.io.IOException; 28 | import java.nio.ByteBuffer; 29 | import java.nio.charset.StandardCharsets; 30 | import java.util.LinkedList; 31 | import java.util.List; 32 | import java.util.Properties; 33 | import java.util.Queue; 34 | import java.util.TimeZone; 35 | import java.util.concurrent.TimeUnit; 36 | 37 | import org.voltcore.utils.CoreUtils; 38 | import org.voltdb.VoltDB; 39 | import org.voltdb.common.Constants; 40 | import org.voltdb.export.AdvertisedDataSource; 41 | import org.voltdb.exportclient.decode.CSVStringDecoder; 42 | 43 | import com.amazonaws.AmazonServiceException; 44 | import com.amazonaws.auth.BasicAWSCredentials; 45 | import com.amazonaws.regions.Region; 46 | import com.amazonaws.regions.RegionUtils; 47 | import com.amazonaws.services.kinesisfirehose.AmazonKinesisFirehoseClient; 48 | import com.amazonaws.services.kinesisfirehose.model.DescribeDeliveryStreamRequest; 49 | import com.amazonaws.services.kinesisfirehose.model.DescribeDeliveryStreamResult; 50 | import com.amazonaws.services.kinesisfirehose.model.InvalidArgumentException; 51 | import com.amazonaws.services.kinesisfirehose.model.Record; 52 | import com.amazonaws.services.kinesisfirehose.model.ResourceNotFoundException; 53 | import com.amazonaws.services.kinesisfirehose.model.ServiceUnavailableException; 54 | import com.google_voltpatches.common.base.Throwables; 55 | import com.google_voltpatches.common.util.concurrent.ListeningExecutorService; 56 | 57 | public class KinesisFirehoseExportClient extends ExportClientBase { 58 | private static final FirehoseExportLogger LOG = new FirehoseExportLogger(); 59 | 60 | private Region m_region; 61 | private String m_streamName; 62 | private String m_accessKey; 63 | private String m_secretKey; 64 | private TimeZone m_timeZone; 65 | private AmazonKinesisFirehoseClient m_firehoseClient; 66 | private FirehoseSink m_sink; 67 | private String m_recordSeparator; 68 | 69 | 70 | private int m_backOffCap; 71 | private int m_backOffBase; 72 | private int m_streamLimit; 73 | private int m_concurrentWriter; 74 | private String m_backOffStrategy; 75 | private BackOff m_backOff; 76 | private boolean m_batchMode; 77 | private int m_batchSize; 78 | 79 | public static final String ROW_LENGTH_LIMIT = "row.length.limit"; 80 | public static final String RECORD_SEPARATOR = "record.separator"; 81 | 82 | public static final String BACKOFF_CAP = "backoff.cap"; 83 | public static final String STREAM_LIMIT = "stream.limit"; 84 | public static final String BACKOFF_TYPE = "backoff.type"; 85 | public static final String CONCURRENT_WRITER = "concurrent.writers"; 86 | public static final String BATCH_MODE = "batch.mode"; 87 | public static final String BATCH_SIZE = "batch.size"; 88 | 89 | public static final int BATCH_NUMBER_LIMIT = 500; 90 | public static final int BATCH_SIZE_LIMIT = 4*1024*1024; 91 | 92 | @Override 93 | public void configure(Properties config) throws Exception 94 | { 95 | String regionName = config.getProperty("region","").trim(); 96 | if (regionName.isEmpty()) { 97 | throw new IllegalArgumentException("KinesisFirehoseExportClient: must provide a region"); 98 | } 99 | m_region = RegionUtils.getRegion(regionName); 100 | 101 | m_streamName = config.getProperty("stream.name","").trim(); 102 | if (m_streamName.isEmpty()) { 103 | throw new IllegalArgumentException("KinesisFirehoseExportClient: must provide a stream.name"); 104 | } 105 | 106 | m_accessKey = config.getProperty("access.key","").trim(); 107 | if (m_accessKey.isEmpty()) { 108 | throw new IllegalArgumentException("KinesisFirehoseExportClient: must provide an access.key"); 109 | } 110 | m_secretKey = config.getProperty("secret.key","").trim(); 111 | if (m_secretKey.isEmpty()) { 112 | throw new IllegalArgumentException("KinesisFirehoseExportClient: must provide a secret.key"); 113 | } 114 | 115 | m_timeZone = TimeZone.getTimeZone(config.getProperty("timezone", VoltDB.REAL_DEFAULT_TIMEZONE.getID())); 116 | 117 | 118 | m_recordSeparator = config.getProperty(RECORD_SEPARATOR,"\n"); 119 | 120 | config.setProperty(ROW_LENGTH_LIMIT, 121 | config.getProperty(ROW_LENGTH_LIMIT,Integer.toString(1024000 - m_recordSeparator.length()))); 122 | 123 | m_backOffCap = Integer.parseInt(config.getProperty(BACKOFF_CAP,"1000")); 124 | // minimal interval between each putRecordsBatch api call; 125 | // for small records (row length < 1KB): records/s is the bottleneck 126 | // for large records (row length > 1KB): data throughput is the bottleneck 127 | // for orignal limit, (5000 records/s divie by 500 records per call = 10 calls) 128 | // interval is 1000 ms / 10 = 100 ms 129 | m_streamLimit = Integer.parseInt(config.getProperty(STREAM_LIMIT,"5000")); 130 | m_backOffBase = Math.max(2, 1000 / (m_streamLimit/BATCH_NUMBER_LIMIT)); 131 | 132 | // concurrent aws client = number of export table to this stream * number of voltdb partition 133 | m_concurrentWriter = Integer.parseInt(config.getProperty(CONCURRENT_WRITER,"0")); 134 | m_backOffStrategy = config.getProperty(BACKOFF_TYPE,"equal"); 135 | 136 | m_firehoseClient = new AmazonKinesisFirehoseClient(new BasicAWSCredentials(m_accessKey, m_secretKey)); 137 | m_firehoseClient.setRegion(m_region); 138 | m_backOff = BackOffFactory.getBackOff(m_backOffStrategy, m_backOffBase, m_backOffCap); 139 | m_sink = new FirehoseSink(m_streamName,m_firehoseClient, m_concurrentWriter, m_backOff); 140 | m_batchMode = Boolean.parseBoolean(config.getProperty(BATCH_MODE, "true")); 141 | m_batchSize = Math.min(BATCH_NUMBER_LIMIT, Integer.parseInt(config.getProperty(BATCH_SIZE,"200"))); 142 | } 143 | 144 | @Override 145 | public ExportDecoderBase constructExportDecoder(AdvertisedDataSource source) 146 | { 147 | return new KinesisFirehoseExportDecoder(source); 148 | } 149 | 150 | class KinesisFirehoseExportDecoder extends ExportDecoderBase { 151 | private final ListeningExecutorService m_es; 152 | private final CSVStringDecoder m_decoder; 153 | 154 | private boolean m_primed = false; 155 | private Queue> m_records; 156 | private List currentBatch; 157 | private int m_currentBatchSize; 158 | 159 | @Override 160 | public ListeningExecutorService getExecutor() { 161 | return m_es; 162 | } 163 | 164 | public KinesisFirehoseExportDecoder(AdvertisedDataSource source) 165 | { 166 | super(source); 167 | 168 | CSVStringDecoder.Builder builder = CSVStringDecoder.builder(); 169 | builder 170 | .dateFormatter(Constants.ODBC_DATE_FORMAT_STRING) 171 | .timeZone(m_timeZone) 172 | .columnNames(source.columnNames) 173 | .columnTypes(source.columnTypes) 174 | ; 175 | m_es = CoreUtils.getListeningSingleThreadExecutor( 176 | "Kinesis Firehose Export decoder for partition " + source.partitionId 177 | + " table " + source.tableName 178 | + " generation " + source.m_generation, CoreUtils.MEDIUM_STACK_SIZE); 179 | m_decoder = builder.build(); 180 | } 181 | 182 | private void validateStream() throws RestartBlockException, InterruptedException { 183 | DescribeDeliveryStreamRequest describeHoseRequest = new DescribeDeliveryStreamRequest(). 184 | withDeliveryStreamName(m_streamName); 185 | DescribeDeliveryStreamResult describeHoseResult = null; 186 | String status = "UNDEFINED"; 187 | describeHoseResult = m_firehoseClient.describeDeliveryStream(describeHoseRequest); 188 | status = describeHoseResult.getDeliveryStreamDescription().getDeliveryStreamStatus(); 189 | if("ACTIVE".equalsIgnoreCase(status)){ 190 | return; 191 | } 192 | else if("CREATING".equalsIgnoreCase(status)){ 193 | Thread.sleep(5000); 194 | validateStream(); 195 | } 196 | else { 197 | LOG.error("Cannot use stream %s, responded with %s", m_streamName, status); 198 | throw new RestartBlockException(true); 199 | } 200 | } 201 | 202 | final void checkOnFirstRow() throws RestartBlockException { 203 | if (!m_primed) try { 204 | validateStream(); 205 | } catch (AmazonServiceException | InterruptedException e) { 206 | LOG.error("Unable to instantiate a Amazon Kinesis Firehose client", e); 207 | throw new RestartBlockException("Unable to instantiate a Amazon Kinesis Firehose client", e, true); 208 | } 209 | m_primed = true; 210 | } 211 | 212 | @Override 213 | public boolean processRow(int rowSize, byte[] rowData) throws RestartBlockException { 214 | if (!m_primed) checkOnFirstRow(); 215 | Record record = new Record(); 216 | try { 217 | final ExportRowData rd = decodeRow(rowData); 218 | String decoded = m_decoder.decode(null, rd.values) + m_recordSeparator; // add a record separator ; 219 | record.withData(ByteBuffer.wrap(decoded.getBytes(StandardCharsets.UTF_8))); 220 | } catch(IOException e) { 221 | LOG.error("Failed to build record", e); 222 | throw new RestartBlockException("Failed to build record", e, true); 223 | } 224 | if(m_batchMode){ 225 | // PutRecordBatchRequest can not contain more than 500 records 226 | // And up to a limit of 4 MB for the entire request 227 | if ((m_currentBatchSize + rowSize) > BATCH_SIZE_LIMIT || currentBatch.size() >= m_batchSize) { 228 | // roll to next batch 229 | m_records.add(currentBatch); 230 | m_currentBatchSize = 0; 231 | currentBatch = new LinkedList(); 232 | } 233 | currentBatch.add(record); 234 | m_currentBatchSize += rowSize; 235 | }else{ 236 | try { 237 | m_sink.writeRow(record); 238 | } catch (FirehoseExportException e) { 239 | throw new RestartBlockException("firehose write fault", e, true); 240 | } catch (ResourceNotFoundException | InvalidArgumentException | ServiceUnavailableException e) { 241 | LOG.error("Failed to send record batch", e); 242 | throw new RestartBlockException("Failed to send record batch", e, true); 243 | } 244 | } 245 | return true; 246 | } 247 | 248 | @Override 249 | public void sourceNoLongerAdvertised(AdvertisedDataSource source) 250 | { 251 | if(m_sink != null){ 252 | m_sink.shutDown(); 253 | } 254 | if (m_firehoseClient != null) m_firehoseClient.shutdown(); 255 | m_es.shutdown(); 256 | try { 257 | m_es.awaitTermination(365, TimeUnit.DAYS); 258 | } catch (InterruptedException e) { 259 | Throwables.propagate(e); 260 | } 261 | } 262 | 263 | @Override 264 | public void onBlockStart() throws RestartBlockException 265 | { 266 | if (!m_primed) checkOnFirstRow(); 267 | m_records = new LinkedList>(); 268 | m_currentBatchSize = 0; 269 | currentBatch = new LinkedList(); 270 | } 271 | 272 | @Override 273 | public void onBlockCompletion() throws RestartBlockException { 274 | 275 | if(m_batchMode){ 276 | // add last batch 277 | if (!currentBatch.isEmpty()) { 278 | // roll to next batch 279 | m_records.add(currentBatch); 280 | m_currentBatchSize = 0; 281 | currentBatch = new LinkedList(); 282 | } 283 | 284 | try { 285 | if(m_concurrentWriter > 0){ 286 | m_sink.write(m_records); 287 | }else{ 288 | m_sink.syncWrite(m_records); 289 | } 290 | } catch (FirehoseExportException e) { 291 | throw new RestartBlockException("firehose write fault", e, true); 292 | } catch (ResourceNotFoundException | InvalidArgumentException | ServiceUnavailableException e) { 293 | LOG.error("Failed to send record batch", e); 294 | throw new RestartBlockException("Failed to send record batch", e, true); 295 | } 296 | } 297 | } 298 | } 299 | } 300 | --------------------------------------------------------------------------------