├── LICENSE ├── README ├── src └── com │ └── zoominfo │ └── util │ └── yieldreturn │ ├── CollectionAbortedException.java │ ├── Collector.java │ ├── Generator.java │ ├── ReferenceYieldAdapter.java │ ├── ResultHandler.java │ ├── ThreadedYieldAdapter.java │ ├── YieldAdapter.java │ ├── YieldAdapterIterable.java │ └── YieldAdapterIterator.java └── test └── com └── zoominfo └── util └── yieldreturn └── GeneratorTest.java /LICENSE: -------------------------------------------------------------------------------- 1 | All the code here is released into the public domain. 2 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | If you've ever used more than one programming language, you've probably found that - as you switch between them - there's usually some feature that you've left behind that you really miss. It could be a particular class, syntactic sugar, or language construct that you just can't do without. When I went from doing development primarily in Python and C# to doing development primarily in Java, one of the little things I missed from those languages was generators. 2 | 3 | Generators are a simple but powerful tool for creating iterators. They are written like regular functions but use a "yield" statement whenever they want to return data. Each time next() is called in your for-each loop, control passes to the generator and it resumes right where it left off - its local variables and execution state are automatically saved between calls. The generator returns control back to your loop when it "yields" the next value in the iteration. 4 | 5 | Using generators - like iterators - can confer some nice performance benefits. This performance boost is the result of the lazy (on-demand) generation of values, which translates to lower memory usage. Furthermore, we do not need to wait until all the elements have been generated before we start to use them. 6 | 7 | It's important to note that anything that can be done with generators can also be done with iterators. What makes generators so nice & compact is that the iterator(), hasNext() and next() methods are all created automatically for you. This helps make generators easier to write and much clearer than an iterator-based approach. You don't have to write a ton of boilerplate and a mini state machine just to keep track of your progress. 8 | 9 | By now, you've probably picked up that Java doesn't have generators. Worse still, Java (the language, not the JVM) doesn't have built-in support for continuations - a useful building block for implementing generators (though there are some 3rd-party add-ons like Apache Javaflow that implement them via bytecode manipulation). Fortunately, not all is lost. A bit of Googling turned up an excellent blog post by Jim Blackler who had implemented a Yield/Return framework in Java. After ironing out a few bugs in the framework (and passing those patches back to Jim), we at Zoom adopted it for use in our production environment. 10 | 11 | Jim's framework is based on a traditional producer/consumer model. In it, two threads effectively do the work of one. Control passes between the "worker" thread (which computes your iteration's values) and the managing thread (which implements the java.util.Iterator logic). Each invocation of "Iterator.next()" causes the worker thread to wake up and compute the next result, which it puts into a Java SynchronousQueue. The worker thread goes back to sleep and the manager thread pops the queue, returning the newly-computed value to your "for" loop. 12 | 13 | By itself, Jim's framework got us lazy-computation and a more natural programming model than straight Java Iterators, which was a huge improvement. But much like iterators, it required writing a lot of boilerblate code. Writing anything in Java requires writing a lot of boilerplate, but we strive to keep that to a minimum. To solve this, we wrote a really simple "Generator" base class that is easily extensible. It implements the Iterable interface, so you can use Generators wherever you would use a Java 1.5-style "for-each" loop. All it requires is that you implement your business logic inside of a "run()" method, return values via its "yield" method, and it figures out the rest for you. 14 | -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/CollectionAbortedException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * A "yield return" implementation for Java 3 | * By Jim Blackler (jimblackler@gmail.com) 4 | * 5 | * http://jimblackler.net/blog/?p=61 6 | * http://svn.jimblackler.net/jimblackler/trunk/IdeaProjects/YieldAdapter/ 7 | */ 8 | package com.zoominfo.util.yieldreturn; 9 | 10 | /** 11 | * An exception class that can be thrown by collectors or results handlers in order to abort or 12 | * signal abortion of the collecting process, for any reason. 13 | */ 14 | public class CollectionAbortedException extends Exception { 15 | 16 | public CollectionAbortedException() { 17 | } 18 | 19 | public CollectionAbortedException(String message) { 20 | super(message); 21 | } 22 | 23 | public CollectionAbortedException(String message, Throwable cause) { 24 | super(message, cause); 25 | } 26 | 27 | public CollectionAbortedException(Throwable cause) { 28 | super(cause); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/Collector.java: -------------------------------------------------------------------------------- 1 | /** 2 | * A "yield return" implementation for Java 3 | * By Jim Blackler (jimblackler@gmail.com) 4 | * 5 | * http://jimblackler.net/blog/?p=61 6 | * http://svn.jimblackler.net/jimblackler/trunk/IdeaProjects/YieldAdapter/ 7 | */ 8 | package com.zoominfo.util.yieldreturn; 9 | 10 | /** 11 | * Defines a class that collects values of type T and submits each value to a ResultHandler<> 12 | * object immediately on collection. 13 | */ 14 | public interface Collector { 15 | 16 | /** 17 | * Perform the collection operation. 18 | * 19 | * @param handler The processor object to return results to. 20 | * @throws CollectionAbortedException The collection operation was aborted part way through. 21 | */ 22 | void collect(ResultHandler handler) throws CollectionAbortedException; 23 | } -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/Generator.java: -------------------------------------------------------------------------------- 1 | package com.zoominfo.util.yieldreturn; 2 | 3 | import java.util.Iterator; 4 | 5 | /** 6 | * This class implements something akin to Python's "Generators". 7 | * 8 | *

A Generator is a function which returns an iterator. It looks like a normal 9 | * function except that it contains yield statements for producing a series a 10 | * values usable in a for-loop or that can be retrieved one at a time with the 11 | * next() function. Each "yield" temporarily suspends processing, remembering the 12 | * location execution state (including local variables and pending try-statements). 13 | * When the generator resumes, it picks-up where it left-off (in contrast to functions 14 | * which start fresh on every invocation).

15 | * 16 | * @author dom 17 | */ 18 | public abstract class Generator implements Iterable { 19 | 20 | private ResultHandler resultHandler; 21 | 22 | /** 23 | * Yield a single result. To be called from within your "run" method. 24 | * 25 | * @param t 26 | */ 27 | protected final void yield(final T t) { 28 | try { 29 | resultHandler.handleResult(t); 30 | } catch (CollectionAbortedException ex) { 31 | // suppress 32 | } 33 | } 34 | 35 | /** 36 | * The method that will generate your results. Call yield() in here. 37 | */ 38 | protected abstract void run(); 39 | 40 | @Override 41 | public final Iterator iterator() { 42 | return new ThreadedYieldAdapter().adapt(new Collector() { 43 | 44 | @Override 45 | public void collect(final ResultHandler handler) throws CollectionAbortedException { 46 | resultHandler = handler; 47 | run(); 48 | } 49 | }).iterator(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/ReferenceYieldAdapter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * A "yield return" implementation for Java 3 | * By Jim Blackler (jimblackler@gmail.com) 4 | * 5 | * http://jimblackler.net/blog/?p=61 6 | * http://svn.jimblackler.net/jimblackler/trunk/IdeaProjects/TestYieldAdapter 7 | */ 8 | package com.zoominfo.util.yieldreturn; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Iterator; 12 | 13 | /** 14 | * This reference adapter simply invokes the Collector<>, first gathering the results into a list. 15 | * It is provided to illustrate the simplicity of the function of the adapter, and to aid debugging 16 | * if threading issues are suspected as the cause of problems in the calling code. 17 | * 18 | * @author Jim Blackler (jimblackler@gmail.com) 19 | */ 20 | public class ReferenceYieldAdapter implements YieldAdapter { 21 | 22 | /** 23 | * Convert a method that implements the Collector<> class with a standard Iterable<>, by 24 | * collecting the results in a list, and returning an iterator to that list. 25 | */ 26 | public YieldAdapterIterable adapt(Collector client) { 27 | 28 | final ArrayList results = new ArrayList(); 29 | 30 | try { 31 | client.collect(new ResultHandler() { 32 | public void handleResult(T value) { 33 | results.add(value); 34 | } 35 | }); 36 | } catch (CollectionAbortedException e) { 37 | // The process was aborted by calling code. 38 | } 39 | 40 | // Wrap container's iterator with yield adapter interface for compatibility 41 | return new YieldAdapterIterable() { 42 | public YieldAdapterIterator iterator() { 43 | final Iterator iterator = results.iterator(); 44 | return new YieldAdapterIterator() { 45 | public boolean hasNext() { 46 | return iterator.hasNext(); 47 | } 48 | 49 | public T next() { 50 | return iterator.next(); 51 | } 52 | 53 | public void remove() { 54 | iterator.remove(); 55 | } 56 | 57 | public void close() { 58 | // Does nothing in this implementation 59 | } 60 | }; 61 | } 62 | }; 63 | 64 | } 65 | } -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/ResultHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * A "yield return" implementation for Java 3 | * By Jim Blackler (jimblackler@gmail.com) 4 | * 5 | * http://jimblackler.net/blog/?p=61 6 | * http://svn.jimblackler.net/jimblackler/trunk/IdeaProjects/YieldAdapter/ 7 | */ 8 | package com.zoominfo.util.yieldreturn; 9 | 10 | /** 11 | * Defines objects that handle results from a Collector<>, with a function called immediately as 12 | * each value is gathered. 13 | */ 14 | public interface ResultHandler { 15 | 16 | /** 17 | * This method is called by collectors whenever a result is collected. 18 | * 19 | * @param value The collected result 20 | * @throws CollectionAbortedException The client code requests that the collection is aborted 21 | */ 22 | void handleResult(T value) throws CollectionAbortedException; 23 | } -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/ThreadedYieldAdapter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * A "yield return" implementation for Java 3 | * By Jim Blackler (jimblackler@gmail.com) 4 | * 5 | * http://jimblackler.net/blog/?p=61 6 | * http://svn.jimblackler.net/jimblackler/trunk/IdeaProjects/YieldAdapter/ 7 | */ 8 | package com.zoominfo.util.yieldreturn; 9 | 10 | import java.util.NoSuchElementException; 11 | import java.util.concurrent.SynchronousQueue; 12 | 13 | /** 14 | * A class to convert methods that implement the Collector<> class into a standard Iterable<>, using 15 | * a new thread created for the collection process, and a SynchronousQueue<> object. 16 | */ 17 | public class ThreadedYieldAdapter implements YieldAdapter { 18 | 19 | /** 20 | * Message structure to pass values between threads. 21 | */ 22 | class Message { 23 | 24 | } 25 | 26 | abstract private class StopMessage extends Message { 27 | 28 | } 29 | 30 | private class EndMessage extends StopMessage { 31 | 32 | } 33 | 34 | private class AbortedMessage extends StopMessage { 35 | 36 | } 37 | 38 | /** 39 | * The vehicle to pass the actual values. 40 | */ 41 | class ValueMessage extends Message { 42 | 43 | ValueMessage(T value) { 44 | this.value = value; 45 | } 46 | 47 | final T value; 48 | } 49 | 50 | /** 51 | * Convert a method that implements the Collector<> class with a standard Iterable<>. This means 52 | * that the collecting method can use complex recursive logic, but still allows the calling code 53 | * to handle the results with a standard iterator. Results are returned immediately and do not 54 | * incur overhead of being stored in a list. Calculation overhead is only performed for the 55 | * results that are requested through the iterator. 56 | * 57 | * This is implemented using a new thread created for the collection process, and a 58 | * SynchronousQueue<> object. 59 | */ 60 | public YieldAdapterIterable adapt(final Collector client) { 61 | 62 | return new YieldAdapterIterable() { 63 | public YieldAdapterIterator iterator() { 64 | 65 | final SynchronousQueue synchronousQueue = new SynchronousQueue(); 66 | 67 | // Mechanism to ensure both threads don't run at the same time 68 | final SynchronousQueue returnQueue = new SynchronousQueue(); 69 | 70 | // This thread is where the collecting logic is executed. 71 | final Thread collectThread = new Thread() { 72 | @Override 73 | public void run() { 74 | 75 | // Important .. handling thread (main thread) gets to run first. 76 | // This is because the collecting process should be run on demand in response to 77 | // iterator access. Each result should be dealt with by the handling process before 78 | // the collecting process is able to modify any resources that may be requred by 79 | // results. 80 | 81 | try { 82 | returnQueue.take(); 83 | } catch (final InterruptedException e) { 84 | //throw new RuntimeException("Error with yield adapter", e); 85 | return; 86 | } 87 | try { 88 | try { 89 | 90 | client.collect(new ResultHandler() { 91 | public void handleResult(final T value) 92 | throws CollectionAbortedException { 93 | try { 94 | synchronousQueue.put(new ValueMessage(value)); 95 | returnQueue.take(); // wait for permission to continue 96 | } catch (final InterruptedException e) { 97 | // this thread has been aborted 98 | throw new CollectionAbortedException(e); 99 | } 100 | } 101 | }); 102 | 103 | synchronousQueue.put(new EndMessage()); 104 | // Signal no more results to come 105 | 106 | } catch (final CollectionAbortedException collectionAborted) { 107 | if (!(collectionAborted 108 | .getCause() instanceof InterruptedException)) { 109 | // Collect was aborted by client 110 | // This is not sent on thread abort as there is nothing waiting 111 | // to receive it, and the thread will block. 112 | synchronousQueue.put(new AbortedMessage()); 113 | } 114 | } 115 | 116 | } catch (final InterruptedException e) { 117 | // Operation was aborted internally (e.g. iterator out of scope) 118 | } 119 | } 120 | }; 121 | collectThread.setDaemon(true); 122 | collectThread.start(); 123 | 124 | return new YieldAdapterIterator() { 125 | private Message messageWaiting = null; 126 | 127 | public boolean hasNext() { 128 | 129 | readNextMessage(); 130 | return !StopMessage.class.isAssignableFrom(messageWaiting.getClass()); 131 | // instanceof cannot be used because of generics restriction 132 | } 133 | 134 | public T next() { 135 | readNextMessage(); 136 | 137 | if (StopMessage.class.isAssignableFrom(messageWaiting.getClass())) { 138 | // instanceof cannot be used because of generics restriction 139 | throw new NoSuchElementException(); 140 | } 141 | 142 | final T value = ((ValueMessage) messageWaiting).value; 143 | messageWaiting = null; // for next time 144 | return value; 145 | } 146 | 147 | private void readNextMessage() { 148 | if (messageWaiting == null) { // do not run if value waiting to be put 149 | try { 150 | returnQueue.put(new Object()); // allow other thread to gather result 151 | messageWaiting = synchronousQueue.take(); 152 | 153 | } catch (final InterruptedException e) { 154 | messageWaiting = new EndMessage(); 155 | } 156 | } 157 | } 158 | 159 | public void remove() { 160 | throw new UnsupportedOperationException("Generators don't support remove()"); 161 | } 162 | 163 | @Override 164 | /** 165 | * Iterator's finalize() can be used to tell when it is out of scope, and the 166 | * collecting thread can be terminated. 167 | */ 168 | protected void finalize() throws Throwable { 169 | close(); 170 | super.finalize(); 171 | } 172 | 173 | /** 174 | * This can be manually called by the calling code to force release of 175 | * resources at the earliest opportunity. 176 | */ 177 | @Override 178 | public void close() { 179 | collectThread.interrupt(); 180 | } 181 | }; 182 | } 183 | }; 184 | } 185 | } 186 | 187 | -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/YieldAdapter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * A "yield return" implementation for Java 3 | * By Jim Blackler (jimblackler@gmail.com) 4 | * 5 | * http://jimblackler.net/blog/?p=61 6 | * http://svn.jimblackler.net/jimblackler/trunk/IdeaProjects/YieldAdapter/ 7 | */ 8 | package com.zoominfo.util.yieldreturn; 9 | 10 | /** 11 | * A class to convert methods that implement the Collector<> class into a standard Iterable<>. 12 | */ 13 | public interface YieldAdapter { 14 | 15 | YieldAdapterIterable adapt(Collector client); 16 | } 17 | -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/YieldAdapterIterable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * A "yield return" implementation for Java 3 | * By Jim Blackler (jimblackler@gmail.com) 4 | * 5 | * http://jimblackler.net/blog/?p=61 6 | * http://svn.jimblackler.net/jimblackler/trunk/IdeaProjects/YieldAdapter/ 7 | */ 8 | package com.zoominfo.util.yieldreturn; 9 | 10 | /** 11 | * A special version of Iterable<> that returns YieldAdapterIterators<>. 12 | */ 13 | public interface YieldAdapterIterable extends Iterable { 14 | 15 | /** 16 | * Returns an iterator over the results. 17 | */ 18 | YieldAdapterIterator iterator(); 19 | } 20 | -------------------------------------------------------------------------------- /src/com/zoominfo/util/yieldreturn/YieldAdapterIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * A "yield return" implementation for Java 3 | * By Jim Blackler (jimblackler@gmail.com) 4 | * 5 | * http://jimblackler.net/blog/?p=61 6 | * http://svn.jimblackler.net/jimblackler/trunk/IdeaProjects/YieldAdapter/ 7 | */ 8 | package com.zoominfo.util.yieldreturn; 9 | 10 | import java.util.Iterator; 11 | import java.io.Closeable; 12 | 13 | /** 14 | * A version of a standard Iterator<> used by the yield adapter. The only addition is a close() 15 | * function to clear resources manually when required. 16 | */ 17 | public interface YieldAdapterIterator extends Iterator, Closeable { 18 | 19 | /** 20 | * Because the Yield Adapter starts a separate thread for duration of the collection, this can 21 | * be left open if the calling code only reads part of the collection. If the iterator goes out 22 | * of scope, when it is GCed its finalize() will close the collection thread. However garbage 23 | * collection is sporadic and the VM will not trigger it simply because there is a lack of 24 | * available threads. So, if a lot of partial reads are happening, it will be wise to manually 25 | * close the iterator (which will clear the resources immediately). 26 | */ 27 | } -------------------------------------------------------------------------------- /test/com/zoominfo/util/yieldreturn/GeneratorTest.java: -------------------------------------------------------------------------------- 1 | package com.zoominfo.util.yieldreturn; 2 | 3 | import org.junit.Test; 4 | import static org.junit.Assert.*; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.Arrays; 8 | 9 | public class GeneratorTest { 10 | 11 | static abstract class GeneratorTestBase extends Generator { 12 | 13 | // keep track of which items have been generated, for bookkeeping purposes. 14 | List generatedItems = new ArrayList(); 15 | } 16 | 17 | static Iterable range(int hi) { 18 | return range(0, hi); 19 | } 20 | 21 | static Iterable range(int lo, int hi) { 22 | return range(lo, hi, 1); 23 | } 24 | 25 | static Iterable range(final int lo, final int hi, final int step) { 26 | return new GeneratorTestBase() { 27 | 28 | @Override 29 | protected void run() { 30 | for (int i = lo; i != hi; i += step) { 31 | generatedItems.add(i); 32 | yield(i); 33 | } 34 | } 35 | }; 36 | } 37 | 38 | static Iterable firstn(final int n) { 39 | return new GeneratorTestBase() { 40 | 41 | @Override 42 | protected void run() { 43 | int num = 0; 44 | while (num < n) { 45 | generatedItems.add(num); 46 | yield(num); 47 | num += 1; 48 | } 49 | } 50 | }; 51 | } 52 | 53 | static Iterable reverse(final CharSequence string) { 54 | return new GeneratorTestBase() { 55 | 56 | @Override 57 | protected void run() { 58 | for (int i : range(string.length() - 1, -1, -1)) { 59 | generatedItems.add(string.charAt(i)); 60 | yield(string.charAt(i)); 61 | } 62 | } 63 | }; 64 | } 65 | 66 | private static Iterable Power(final int number, final int exponent) { 67 | return new GeneratorTestBase() { 68 | 69 | @Override 70 | protected void run() { 71 | int counter = 0; 72 | 73 | 74 | int result = 1; 75 | 76 | 77 | while (counter++ < exponent) { 78 | result = result * number; 79 | generatedItems.add(result); 80 | yield(result); 81 | } 82 | } 83 | }; 84 | } 85 | 86 | private static List runTest(Iterable iterable) { 87 | GeneratorTestBase generator = (GeneratorTestBase) iterable; 88 | 89 | List list = new ArrayList(); 90 | 91 | for (T item : generator) { 92 | list.add(item); 93 | } 94 | 95 | // assert that no more - and no fewer - items were generated than were returned 96 | assertEquals(generator.generatedItems.size(), list.size()); 97 | 98 | // assert that the lists have the same items in the same order 99 | for (int i = 0; i 100 | < list.size(); i++) { 101 | T item1 = list.get(i); 102 | T item2 = generator.generatedItems.get(i); 103 | 104 | assertEquals(item1, item2); 105 | } 106 | 107 | return list; 108 | } 109 | 110 | @Test 111 | public void testPower() { 112 | List list = runTest(Power(2, 8)); 113 | 114 | assertEquals(8, list.size()); 115 | assertTrue(list.containsAll(Arrays.asList(2, 4, 8, 16, 32, 64, 128, 256))); 116 | } 117 | 118 | @Test 119 | public void testRange() { 120 | runTest(range(0, 1000)); 121 | 122 | GeneratorTestBase generator = (GeneratorTestBase) range(0, 1000); 123 | 124 | for (int i : generator) { 125 | if (i == 499) { 126 | break; // stop short 127 | } 128 | } 129 | 130 | // assert that we generated only 500 items - 0 through 499, inclusive 131 | assertEquals(500, generator.generatedItems.size()); 132 | assertEquals(0, (int) generator.generatedItems.get(0)); 133 | assertEquals(499, (int) generator.generatedItems.get(499)); 134 | } 135 | 136 | @Test 137 | public void testReverse() { 138 | StringBuilder testString = new StringBuilder("hello, world!"); 139 | StringBuilder sb = new StringBuilder(); 140 | 141 | for (Character c : reverse(testString)) { 142 | sb.append(c); 143 | } 144 | 145 | assertEquals(testString.reverse().toString(), sb.toString()); 146 | } 147 | 148 | @Test 149 | public void testFirstN() { 150 | List first_n = runTest(firstn(500)); 151 | 152 | assertEquals(500, first_n.size()); 153 | assertEquals(0, (int) first_n.get(0)); 154 | assertEquals(499, (int) first_n.get(499)); 155 | } 156 | } 157 | --------------------------------------------------------------------------------