├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── pom.xml └── src ├── main ├── java │ └── net │ │ └── apnic │ │ └── example │ │ └── jdbcstream │ │ ├── EnableJdbcStream.java │ │ ├── JdbcStream.java │ │ └── JdbcStreamApplication.java └── resources │ └── application.properties └── test ├── java └── net │ └── apnic │ └── example │ └── jdbcstream │ ├── JdbcPerformanceTest.java │ └── JdbcStreamApplicationTests.java └── resources ├── data.sql └── schema.sql /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | *.iml 3 | .idea/ 4 | 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: java 3 | jdk: 4 | - oraclejdk8 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The BSD License 2 | 3 | Copyright (c) 2015 APNIC Pty Ltd 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | - Redistributions of source code must retain the above copyright notice, 10 | this list of conditions and the following disclaimer. 11 | 12 | - Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | - Neither the name of APNIC nor the names of its contributors may be 17 | used to endorse or promote products derived from this software without 18 | specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Spring JdbcTemplate with Java 8 Streams 2 | 3 | [![Build Status](https://travis-ci.org/APNIC-net/spring-jdbctemplate-streams.svg?branch=master)](https://travis-ci.org/APNIC-net/spring-jdbctemplate-streams) 4 | 5 | This repository contains example code for how one might use Spring's 6 | [JdbcTemplate] with the Java 8 [Stream] API. 7 | 8 | To integrate it in your own project clone this repository in a subdirectory 9 | (this way you can always pull for updates) and integrate it in your build. 10 | If you use maven use the `include` tag of the maven-compiler-plugin. To run 11 | it in your IDE ensure to set the additional Sources Root. 12 | 13 | Sample code 14 | 15 | - `JdbcStream.streamableQuery()`: an extension of [JdbcTemplate] to make a 16 | `Closeable` streamable query. This gets into the protected guts of 17 | `JdbcTemplate` to manage a connection's lifetime, and becomes a resource 18 | which must be closed. 19 | - `JdbcStream.streamQuery()`: a callback-style query interface allowing 20 | stream processing inside `JdbcTemplate`'s own resource management system. 21 | This version will not work for empty result sets. 22 | - `JdbcStreamApplication.streamer`: a bean implementing `streamQuery()` as a 23 | consumer of a `JdbcTemplate`. This one correctly handles empty result sets, 24 | and is the basis of the [gist] and [blog post] which this code fed. 25 | 26 | There's a performance test suite which will go out of memory for query methods 27 | which inadvertently cache the entire result set in memory, and a basic test 28 | suite demonstrating the streaming code performs as expected. 29 | 30 | The test cases show what not to do with this interface: they reimplement trivial 31 | SQL queries in Java code. 32 | 33 | [JdbcTemplate]: https://docs.spring.io/spring/docs/current/javadoc-api/org/springframework/jdbc/core/JdbcTemplate.html 34 | [Stream]: https://docs.oracle.com/javase/8/docs/api/java/util/stream/package-summary.html 35 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | net.apnic.example 7 | jdbcstream 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | JDBC Stream 12 | Demo project for making a JdbcTemplate stream 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 1.2.5.RELEASE 18 | 19 | 20 | 21 | 22 | UTF-8 23 | 1.8 24 | 25 | 26 | 27 | 28 | org.springframework.boot 29 | spring-boot-starter-jdbc 30 | 31 | 32 | com.h2database 33 | h2 34 | runtime 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-test 39 | test 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.springframework.boot 47 | spring-boot-maven-plugin 48 | 49 | 50 | 51 | org.apache.maven.plugins 52 | maven-surefire-plugin 53 | 2.18.1 54 | 55 | -Xmx256m 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /src/main/java/net/apnic/example/jdbcstream/EnableJdbcStream.java: -------------------------------------------------------------------------------- 1 | package net.apnic.example.jdbcstream; 2 | 3 | /** 4 | * Created by Jannik on 15.01.16. 5 | */ 6 | public @interface EnableJdbcStream { 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/net/apnic/example/jdbcstream/JdbcStream.java: -------------------------------------------------------------------------------- 1 | package net.apnic.example.jdbcstream; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.jdbc.core.JdbcTemplate; 5 | import org.springframework.jdbc.datasource.DataSourceUtils; 6 | import org.springframework.jdbc.support.rowset.ResultSetWrappingSqlRowSet; 7 | import org.springframework.jdbc.support.rowset.SqlRowSet; 8 | import org.springframework.stereotype.Component; 9 | 10 | import javax.sql.DataSource; 11 | import java.io.Closeable; 12 | import java.io.IOException; 13 | import java.sql.Connection; 14 | import java.sql.PreparedStatement; 15 | import java.sql.SQLException; 16 | import java.sql.Timestamp; 17 | import java.util.Iterator; 18 | import java.util.NoSuchElementException; 19 | import java.util.Spliterator; 20 | import java.util.Spliterators; 21 | import java.util.function.Function; 22 | import java.util.function.Supplier; 23 | import java.util.stream.Stream; 24 | import java.util.stream.StreamSupport; 25 | 26 | @Component 27 | public class JdbcStream extends JdbcTemplate { 28 | 29 | @Autowired 30 | public JdbcStream(DataSource dataSource) { 31 | super(dataSource); 32 | } 33 | 34 | public T streamQuery(String sql, Function, ? extends T> streamer, Object... args) { 35 | return query(sql, resultSet -> { 36 | final SqlRowSet rowSet = new ResultSetWrappingSqlRowSet(resultSet); 37 | final SqlRow sqlRow = new SqlRowAdapter(rowSet); 38 | 39 | Supplier> supplier = () -> Spliterators.spliteratorUnknownSize(new Iterator() { 40 | @Override 41 | public boolean hasNext() { 42 | return !rowSet.isLast(); 43 | } 44 | 45 | @Override 46 | public SqlRow next() { 47 | if (!rowSet.next()) { 48 | throw new NoSuchElementException(); 49 | } 50 | return sqlRow; 51 | } 52 | }, Spliterator.IMMUTABLE); 53 | return streamer.apply(StreamSupport.stream(supplier, Spliterator.IMMUTABLE, false)); 54 | 55 | }, args); 56 | } 57 | 58 | public StreamableQuery streamableQuery(String sql, Object... args) throws SQLException { 59 | Connection connection = DataSourceUtils.getConnection(getDataSource()); 60 | PreparedStatement preparedStatement = connection.prepareStatement(sql); 61 | newArgPreparedStatementSetter(args).setValues(preparedStatement); 62 | return new StreamableQuery(connection, preparedStatement); 63 | } 64 | 65 | public class StreamableQuery implements Closeable { 66 | private final Connection connection; 67 | private final PreparedStatement preparedStatement; 68 | 69 | private StreamableQuery(Connection connection, PreparedStatement preparedStatement) { 70 | this.connection = connection; 71 | this.preparedStatement = preparedStatement; 72 | } 73 | 74 | public Stream stream() throws SQLException { 75 | final SqlRowSet rowSet = new ResultSetWrappingSqlRowSet(preparedStatement.executeQuery()); 76 | final SqlRow sqlRow = new SqlRowAdapter(rowSet); 77 | 78 | Supplier> supplier = () -> Spliterators.spliteratorUnknownSize(new Iterator() { 79 | @Override 80 | public boolean hasNext() { 81 | return !rowSet.isLast(); 82 | } 83 | 84 | @Override 85 | public SqlRow next() { 86 | if (!rowSet.next()) { 87 | throw new NoSuchElementException(); 88 | } 89 | return sqlRow; 90 | } 91 | }, Spliterator.IMMUTABLE); 92 | return StreamSupport.stream(supplier, Spliterator.IMMUTABLE, false); 93 | } 94 | 95 | ; 96 | 97 | @Override 98 | public void close() throws IOException { 99 | DataSourceUtils.releaseConnection(connection, getDataSource()); 100 | } 101 | } 102 | 103 | /** 104 | * Facade to hide the cursor movement methods of an SqlRowSet 105 | */ 106 | public interface SqlRow { 107 | //TODO - implement remaining getters 108 | Long getLong(String columnLabel); 109 | 110 | String getString(String columnLabel); 111 | 112 | Timestamp getTimestamp(String columnLabel); 113 | } 114 | 115 | public class SqlRowAdapter implements SqlRow { 116 | private final SqlRowSet sqlRowSet; 117 | 118 | public SqlRowAdapter(SqlRowSet sqlRowSet) { 119 | this.sqlRowSet = sqlRowSet; 120 | } 121 | 122 | @Override 123 | public Long getLong(String columnLabel) { 124 | return sqlRowSet.getLong(columnLabel); 125 | } 126 | 127 | @Override 128 | public String getString(String columnLabel) { 129 | return sqlRowSet.getString(columnLabel); 130 | } 131 | 132 | @Override 133 | public Timestamp getTimestamp(String columnLabel) { 134 | return sqlRowSet.getTimestamp(columnLabel); 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/net/apnic/example/jdbcstream/JdbcStreamApplication.java: -------------------------------------------------------------------------------- 1 | package net.apnic.example.jdbcstream; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.jdbc.core.JdbcTemplate; 7 | import org.springframework.jdbc.datasource.embedded.ConnectionProperties; 8 | import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseConfigurer; 9 | import org.springframework.jdbc.support.rowset.ResultSetWrappingSqlRowSet; 10 | import org.springframework.jdbc.support.rowset.SqlRowSet; 11 | 12 | import javax.sql.DataSource; 13 | import java.util.Iterator; 14 | import java.util.NoSuchElementException; 15 | import java.util.Spliterator; 16 | import java.util.Spliterators; 17 | import java.util.function.Function; 18 | import java.util.stream.Stream; 19 | import java.util.stream.StreamSupport; 20 | 21 | @SpringBootApplication 22 | public class JdbcStreamApplication implements EmbeddedDatabaseConfigurer { 23 | 24 | public static void main(String[] args) { 25 | SpringApplication.run(JdbcStreamApplication.class, args); 26 | } 27 | 28 | public interface QueryStream { 29 | public T streamQuery(String sql, Function, ? extends T> streamer, Object... args); 30 | } 31 | 32 | private enum IsParallel { 33 | SEQUENTIAL(false), 34 | PARALLEL(true); 35 | 36 | private final boolean flag; 37 | 38 | IsParallel(boolean flag) { 39 | this.flag = flag; 40 | } 41 | 42 | boolean getFlag() { return flag; } 43 | } 44 | 45 | /** 46 | * The gist code at 47 | * https://gist.github.com/codebje/58d1b12e7a2d0ed31b3a#file-jdbcstreams-java 48 | * 49 | * @param jdbcTemplate the JdbcTemplate to use 50 | * @return a QueryStream instance 51 | */ 52 | @Bean 53 | public QueryStream streamer(JdbcTemplate jdbcTemplate) { 54 | return new QueryStream() { 55 | @Override 56 | public T streamQuery(String sql, Function, ? extends T> streamer, Object... args) { 57 | return jdbcTemplate.query(sql, resultSet -> { 58 | final SqlRowSet rowSet = new ResultSetWrappingSqlRowSet(resultSet); 59 | 60 | if (!rowSet.next()) { 61 | return streamer.apply(StreamSupport.stream(Spliterators.emptySpliterator(), 62 | IsParallel.PARALLEL.getFlag())); 63 | } 64 | 65 | Spliterator spliterator = Spliterators.spliteratorUnknownSize(new Iterator() { 66 | private boolean first = true; 67 | @Override 68 | public boolean hasNext() { 69 | return !rowSet.isLast(); 70 | } 71 | 72 | @Override 73 | public SqlRowSet next() { 74 | if (!first || !rowSet.next()) { 75 | throw new NoSuchElementException(); 76 | } 77 | first = false; 78 | return rowSet; 79 | } 80 | }, Spliterator.IMMUTABLE); 81 | return streamer.apply(StreamSupport.stream(spliterator, 82 | IsParallel.SEQUENTIAL.getFlag())); 83 | }, args); 84 | } 85 | }; 86 | } 87 | 88 | @Override 89 | public void configureConnectionProperties(ConnectionProperties connectionProperties, String s) { 90 | 91 | } 92 | 93 | @Override 94 | public void shutdown(DataSource dataSource, String s) { 95 | 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.datasource.url=jdbc:h2:./h2 -------------------------------------------------------------------------------- /src/test/java/net/apnic/example/jdbcstream/JdbcPerformanceTest.java: -------------------------------------------------------------------------------- 1 | package net.apnic.example.jdbcstream; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.junit.After; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.boot.test.SpringApplicationConfiguration; 11 | import org.springframework.jdbc.core.JdbcTemplate; 12 | import org.springframework.jdbc.core.ParameterizedPreparedStatementSetter; 13 | import org.springframework.test.context.TestExecutionListeners; 14 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 15 | import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; 16 | import org.springframework.test.context.support.DirtiesContextTestExecutionListener; 17 | import org.springframework.test.context.transaction.TransactionalTestExecutionListener; 18 | 19 | import java.io.IOException; 20 | import java.sql.PreparedStatement; 21 | import java.sql.SQLException; 22 | import java.util.stream.Collectors; 23 | import java.util.stream.Stream; 24 | 25 | @RunWith(SpringJUnit4ClassRunner.class) 26 | @SpringApplicationConfiguration(classes = JdbcStreamApplication.class) 27 | @TestExecutionListeners(listeners = { DependencyInjectionTestExecutionListener.class, 28 | DirtiesContextTestExecutionListener.class, TransactionalTestExecutionListener.class }) 29 | public class JdbcPerformanceTest { 30 | protected final Log logger = LogFactory.getLog(JdbcPerformanceTest.class); 31 | 32 | @Autowired 33 | JdbcStream jdbcStream; 34 | 35 | @Autowired 36 | JdbcTemplate jdbcTemplate; 37 | 38 | @Before 39 | public void setUp(){ 40 | logger.info("populating database"); 41 | 42 | ParameterizedPreparedStatementSetter ppss = new ParameterizedPreparedStatementSetter() { 43 | @Override 44 | public void setValues(PreparedStatement preparedStatement, String s) throws SQLException { 45 | preparedStatement.setString(1, s); 46 | } 47 | }; 48 | 49 | jdbcTemplate.batchUpdate("DELETE FROM test_data"); 50 | 51 | int batchSize = 10000; 52 | for(int i=0; i< 5; i++) { 53 | jdbcTemplate.batchUpdate("INSERT INTO test_data (entry) VALUES (?)", 54 | Stream.generate(Math::random).map(String::valueOf).limit(1000000).collect(Collectors.toList()), 55 | batchSize, ppss); 56 | } 57 | 58 | logger.info("test data populated"); 59 | } 60 | 61 | @After 62 | public void cleanUp() { 63 | logger.info("cleaning up database"); 64 | jdbcTemplate.batchUpdate("DELETE FROM test_data"); 65 | } 66 | 67 | @Test 68 | public void streamsData() throws SQLException, IOException { 69 | try (JdbcStream.StreamableQuery query = jdbcStream.streamableQuery("SELECT * FROM test_data")) { 70 | logger.info("Queried streaming records: " + query.stream() 71 | .map(row -> row.getString("entry")) 72 | .collect(Collectors.counting())); 73 | } 74 | } 75 | 76 | @Test 77 | public void callbackData(){ 78 | final Counter counter = new Counter(); 79 | jdbcTemplate.query("SELECT * FROM test_data", resultSet -> { 80 | String s = resultSet.getString("entry"); 81 | counter.value++; 82 | }); 83 | logger.info("Queried callback records: " + counter.value); 84 | } 85 | 86 | static class Counter{ 87 | public long value; 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/net/apnic/example/jdbcstream/JdbcStreamApplicationTests.java: -------------------------------------------------------------------------------- 1 | package net.apnic.example.jdbcstream; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.boot.test.SpringApplicationConfiguration; 8 | import org.springframework.context.ApplicationContext; 9 | import org.springframework.jdbc.datasource.init.ScriptUtils; 10 | import org.springframework.jdbc.support.rowset.SqlRowSet; 11 | import org.springframework.test.context.TestExecutionListeners; 12 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 13 | import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; 14 | import org.springframework.test.context.support.DirtiesContextTestExecutionListener; 15 | import org.springframework.test.context.transaction.TransactionalTestExecutionListener; 16 | 17 | import java.io.IOException; 18 | import java.sql.Connection; 19 | import java.sql.SQLException; 20 | import java.util.Set; 21 | import java.util.stream.Collectors; 22 | import java.util.stream.Stream; 23 | 24 | import static org.junit.Assert.*; 25 | import static org.hamcrest.Matchers.*; 26 | 27 | @RunWith(SpringJUnit4ClassRunner.class) 28 | @SpringApplicationConfiguration(classes = JdbcStreamApplication.class) 29 | @TestExecutionListeners(listeners = { DependencyInjectionTestExecutionListener.class, 30 | DirtiesContextTestExecutionListener.class, TransactionalTestExecutionListener.class }) 31 | public class JdbcStreamApplicationTests { 32 | @Autowired 33 | JdbcStream jdbcStream; 34 | 35 | @Autowired 36 | JdbcStreamApplication.QueryStream queryStream; 37 | 38 | @Autowired 39 | ApplicationContext context; 40 | 41 | @Before 42 | public void setUp() { 43 | jdbcStream.execute((Connection conn) -> { 44 | ScriptUtils.executeSqlScript(conn, context.getResource("data.sql")); 45 | return null; 46 | }); 47 | } 48 | 49 | @Test 50 | public void contextLoads() { 51 | } 52 | 53 | private Set streamData(Stream stream) { 54 | return stream.map(row -> row.getString("entry")) 55 | .filter(s -> Character.isAlphabetic(s.charAt(0))) 56 | .collect(Collectors.toSet()); 57 | } 58 | 59 | @Test 60 | public void streamsData() throws SQLException, IOException { 61 | try (JdbcStream.StreamableQuery query = jdbcStream.streamableQuery("SELECT * FROM test_data")) { 62 | Set results = query.stream() 63 | .map(row -> row.getString("entry")) 64 | .filter(s -> Character.isAlphabetic(s.charAt(0))) 65 | .collect(Collectors.toSet()); 66 | 67 | assertThat("3 results start with an alphabetic character", results.size(), is(equalTo(3))); 68 | } 69 | } 70 | 71 | @Test 72 | public void streamsEmptyData() { 73 | Set results = queryStream.streamQuery("SELECT * FROM test_data WHERE entry IS NULL", 74 | this::streamData); 75 | assertThat("A query with no results produces an empty set", results, is(empty())); 76 | } 77 | 78 | @Test 79 | public void callbackStreaming() { 80 | Set results = jdbcStream.streamQuery("SELECT * FROM test_data", stream -> stream 81 | .map(row -> row.getString("entry")) 82 | .filter(s -> Character.isAlphabetic(s.charAt(0))) 83 | .collect(Collectors.toSet())); 84 | 85 | assertThat("3 results start with an alphabetic character", results.size(), is(equalTo(3))); 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/test/resources/data.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM test_data; 2 | INSERT INTO test_data(entry) VALUES 3 | ('hello'), 4 | ('world'), 5 | ('0 starts with number'), 6 | ('- starts with hyphen'), 7 | (' starts with space'), 8 | ('starts with character'); -------------------------------------------------------------------------------- /src/test/resources/schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS test_data; 2 | CREATE TABLE test_data( 3 | entry varchar(200) not null 4 | ); --------------------------------------------------------------------------------