├── .gitignore ├── README.md ├── create_threads_until_crash ├── Main.java ├── README.md └── run.sh ├── deadlock_lock ├── Main.java ├── README.md └── run.sh ├── deadlock_synchronized ├── Main.java ├── README.md └── run.sh ├── experimental └── scheduled_thread_pool_executor_100_cpu │ ├── .idea │ ├── .gitignore │ ├── misc.xml │ ├── modules.xml │ └── vcs.xml │ ├── Dockerfile │ ├── Main.java │ ├── out │ └── production │ │ └── scheduled_thread_pool_executor_100_cpu │ │ ├── .idea │ │ ├── .gitignore │ │ ├── misc.xml │ │ ├── modules.xml │ │ └── vcs.xml │ │ ├── Dockerfile │ │ ├── run.sh │ │ └── scheduled_thread_pool_executor_100_cpu.iml │ ├── run.sh │ └── scheduled_thread_pool_executor_100_cpu.iml ├── large_array ├── Main.java ├── README.md └── run.sh ├── max_connections ├── README.md ├── data │ ├── collapsed.wall.1.txt.gz │ ├── collapsed.wall.840000.txt.gz │ ├── flamegraph.1.svg │ ├── flamegraph.840000.svg │ ├── out.1.sar.gz │ ├── out.1.svg │ ├── out.1.txt.gz │ ├── out.10000.sar.gz │ ├── out.10000.txt.gz │ ├── out.100000.sar.gz │ ├── out.100000.txt.gz │ ├── out.1000000.sar.gz │ ├── out.1000000.txt.gz │ ├── out.640000.sar.gz │ ├── out.640000.txt.gz │ ├── out.800000.sar.gz │ ├── out.800000.svg │ ├── out.800000.txt.gz │ ├── out.840000.sar.gz │ ├── out.840000.svg │ ├── out.840000.txt.gz │ ├── out.880000.sar.gz │ ├── out.880000.txt.gz │ ├── out.900000.sar.gz │ └── out.900000.txt.gz ├── run.sh └── src │ └── Main.java ├── millis_nanos_repeat_or_decrease ├── Main.class ├── Main.java ├── README.md └── run.sh ├── native_memory_impact ├── Main.java ├── README.md └── run.sh ├── prepared_statement_limits ├── README.md ├── compile.txt ├── err.txt ├── out.txt ├── pom.xml ├── run.sh └── src │ └── main │ └── java │ └── Main.java ├── read_write_lock_promotion ├── Main.java ├── README.md └── run.sh ├── resource_leaks ├── .idea │ ├── .gitignore │ ├── compiler.xml │ ├── jarRepositories.xml │ ├── libraries │ │ ├── Maven__args4j_args4j_2_33.xml │ │ └── Maven__org_mariadb_jdbc_mariadb_java_client_2_7_4.xml │ ├── misc.xml │ ├── modules.xml │ └── vcs.xml ├── README.md ├── compile.txt ├── err.txt ├── histo.txt ├── out.txt ├── pom.xml ├── resource.leaks.iml ├── run.sh └── src │ └── main │ └── java │ └── Main.java ├── spark_protobuf ├── .gitignore ├── 01_run_cluster.sh ├── 02_run_word_count.sh ├── 03_run_word_count_java_rdd.sh ├── 04_run_char_count.sh ├── README.md ├── pom.xml └── src │ └── main │ └── java │ ├── CharacterCount.java │ ├── WordCount.java │ └── WordCountJavaRdd.java ├── stack_trace_super_sub_classes ├── Main.java ├── README.md └── run.sh └── tree_map_corruption ├── cpp ├── .gitignore ├── Makefile ├── README.md ├── SimpleRepro_2025-01-18_155758_VeXO.sample.txt └── main.cpp ├── csharp ├── .gitignore ├── README.md └── TreeMapCorruption │ ├── TreeMapCorruption.sln │ └── TreeMapCorruption │ ├── Program.cs │ └── TreeMapCorruption.csproj ├── golang ├── README.md ├── go.mod ├── go.sum └── simple_repro.go ├── java ├── README.md ├── pom.xml └── src │ └── main │ ├── java │ ├── ExecutorUncaughtRepro.java │ ├── ExploringTreeMap.java │ ├── GrpcRepro.java │ ├── GrpcThrowNPE.java │ ├── ProtectedAbstractMap.java │ ├── ProtectedSimpleRepro.java │ ├── ProtectedTreeMap.java │ ├── ReproDetectDump.java │ ├── SimpleRepro.java │ └── TreeMapExplorer.java │ └── protobuf │ └── ReceiptProcessorService.proto ├── ruby ├── README.md └── simple_repo.rb └── rust ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md └── src └── main.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | *.class 3 | *.jar 4 | *.tmp 5 | *.swp 6 | *.swo 7 | .idea/ 8 | *.sar 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | If you have a question about Java, you can probably Google it and find Stackoverflow question with the answer in a few minutes. 2 | Or you could spend hours writing an experimental program that also gives you the answer! 3 | 4 | Learn about java fundementals by writing experimental programs. 5 | Each article will cover as small of a concept as possible. 6 | The article will start with a question, and hide the answer in an expand block. 7 | 8 | For each article you should: 9 | 10 | 1. Try to think about the question and guess it. 11 | 2. Try to write a program that will help you explore the answer. How does it 12 | compare with your guess? 13 | 3. Expand the answer block, and compare with your what you learned from your 14 | experiment. 15 | 4. Inspect the code included with the article and compare the approach. 16 | 17 | 18 | Topics: 19 | 20 | 1. Concurrency 21 | 1. [What happens when you get a deadlock with synchronized in java?](deadlock_synchronized/README.md) 22 | 1. [What happens when you get a deadlock with locks in java? Is it any different from synchronized?](deadlock_lock/README.md) 23 | 1. [Can a thread promote their `ReentrantReadWriteLock.readLock()` to a `ReentrantReadWriteLock.writeLock()`?](read_write_lock_promotion/README.md) 24 | 5. Performance 25 | 1. [What's the max num of connections a server and client can concurrently keep open?](max_connections/README.md) 26 | 1. [What happens if you don't close something properly?](resource_leaks/README.md) 27 | 1. [How many threads until your JVM crashes?](native_memory_impact/README.md) 28 | 1. [How does native memory impact heap? OS memory?](create_threads_until_crash/README.md) 29 | 1. [Does ArrayList have a limit? LinkedList?](large_array/README.md) 30 | 1. [Does Spark work well with Protobuf?](spark_protobuf/README.md) 31 | 2. Date and time 32 | 1. [Does `System.currentTimeMillis()` or `System.nanoTime()` ever decrease? ever repeat?](millis_nanos_repeat_or_decrease/README.md) 33 | 3. Exceptions 34 | 1. [What does stack trace of the overrided method look like compared to super class's method?](stack_trace_super_sub_classes/README.md) 35 | 4. JDBC 36 | 1. [Where can you use '?' in prepared statements?](prepared_statement_limits/README.md) 37 | 38 | Future topics: 39 | 40 | 1. What happens when you get a deadlock in jdbc? 41 | 1. Does synchronized use fair locking? 42 | 1. What line throws NPE when you try to autobox a null? 43 | 1. What happens to your thread when you gracefully kill the java process? 44 | 1. What happens to the runnables in an ExecutorService when you gracefully kill the java process? 45 | 1. What happens when an Runnable in an ExecutorService throws an exception? 46 | 1. What hapepns when a Callable in an Executor Service throws an exception and 47 | you call .get() on the future? 48 | 1. Can in inner class use the privates of the holding class? 49 | 1. How do you chain method calls when Non-null, nullable, primitive, and 50 | optional? 51 | 1. What happens when you print an array? Does toString() call toString on each 52 | element? What about an ArrayList? 53 | 1. Converting bytes to string 54 | -------------------------------------------------------------------------------- /create_threads_until_crash/Main.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | 3 | public final class Main { 4 | 5 | public static void main(String [] args) throws Exception { 6 | int numOfThreads = Integer.parseInt(args[0]); 7 | for (int i = 1; i <= numOfThreads; i++) { 8 | Thread thread = new Thread(() -> { 9 | while (true) { 10 | try { 11 | Thread.sleep(60*1000); 12 | } catch (InterruptedException e) { 13 | } 14 | } 15 | }); 16 | thread.start(); 17 | System.out.println(i); 18 | } 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /create_threads_until_crash/README.md: -------------------------------------------------------------------------------- 1 | How many threads until your JVM crashes? 2 | How much does each thread impact heap? 3 | How can you increase the number of threads? 4 | What does the JVM report? 5 | What does the OS report? 6 | 7 | 1. Think about guess the answer. You will learn more if you guess first then 8 | check compared to immediately checking the answer. 9 | 2. Try to write some experimental code 10 | 3. Compare with my experiments 11 | 12 |
13 | Click to reveal the results of my experiment. 14 | 15 | I put together a program that creates threads and starts them, printing out the number of threads so far to standard out forever. 16 | Eventually it will hit some bottleneck and fail. 17 | I wrapped a bash script around it to let me modify the stack and heap size. 18 | 19 | **WARNING:** if you run this script, sometimes your OS will crash because it uses too much memory. 20 | 21 | ``` 22 | ./run.sh 1M 10M 10000 23 | ... 24 | 4072 25 | 4073 26 | 4074 27 | [0.533s][warning][os,thread] Failed to start thread - pthread_create failed (EAGAIN) for attributes: stacksize: 1024k, guardsize: 4k, detached. 28 | Exception in thread "main" java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached 29 | at java.base/java.lang.Thread.start0(Native Method) 30 | at java.base/java.lang.Thread.start(Thread.java:800) 31 | at Main.main(Main.java:16) 32 | ``` 33 | 34 | Looks like I can create 4,000 threads before my JVM crashes! 35 | It looks like heap is unrelated. 36 | I set the heap to 10MB but gave the stack 1MB of memory. 37 | If the resources consumed threads were allocated on the heap, then I expect to only be able to create about 100 threads. 38 | As a result, we know that Java creates the threads off heap and the heap has no effect. 39 | 40 | How can you increase the number of threads? 41 | Each thread comes with its own stack to keep track of the call stack. 42 | Lets check if increase the stack size reduces the number of threads we can create. 43 | ``` 44 | ./run.sh 2M 10M 10000 45 | ... 46 | 4072 47 | 4073 48 | 4074 49 | [0.566s][warning][os,thread] Failed to start thread - pthread_create failed (EAGAIN) for attributes: stacksize: 2048k, guardsize: 4k, detached. 50 | Exception in thread "main" java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached 51 | at java.base/java.lang.Thread.start0(Native Method) 52 | at java.base/java.lang.Thread.start(Thread.java:800) 53 | at Main.main(Main.java:16) 54 | ``` 55 | Again 4000? 56 | I did not expect this. 57 | 58 | ``` 59 | ./run.sh 256M 10M 10000 60 | 4072 61 | 4073 62 | 4074 63 | [0.566s][warning][os,thread] Failed to start thread - pthread_create failed (EAGAIN) for attributes: stacksize: 2048k, guardsize: 4k, detached. 64 | Exception in thread "main" java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached 65 | at java.base/java.lang.Thread.start0(Native Method) 66 | at java.base/java.lang.Thread.start(Thread.java:800) 67 | at Main.main(Main.java:16) 68 | ``` 69 | Still 4000? 70 | How am I allocating 1 TB of memory? 71 | My laptop only has 16GB. 72 | 73 | Lets allocate less and see what the JVM reports and the OS reports to help 74 | figure it out. 75 | ``` 76 | ./run.sh 256M 10M 1000 77 | jcmd $java_process_pid VM.native_memory 78 | ... 79 | - Thread (reserved=264777398KB, committed=264777398KB) 80 | (thread #1020) 81 | (stack: reserved=264774656KB, committed=264774656KB) 82 | (malloc=1549KB #6116) 83 | (arena=1192KB #2036) 84 | ... 85 | - Native Memory Tracking (reserved=581KB, committed=581KB) 86 | (malloc=98KB #1058) 87 | (tracking overhead=483KB) 88 | 89 | jcmd $java_process_pid GC.heap_info 90 | 1035: 91 | garbage-first heap total 10240K, used 2107K [0x00000007ff600000, 0x0000000800000000) 92 | region size 1024K, 2 young (2048K), 0 survivors (0K) 93 | Metaspace used 265K, committed 448K, reserved 1056768K 94 | class space used 5K, committed 128K, reserved 1048576K 95 | 96 | ps -l -p $java_process_pid 97 | UID PID PPID F CPU PRI NI SZ RSS WCHAN S ADDR TTY TIME CMD 98 | 501 1035 1033 4006 0 31 0 270756616 95428 - S+ 0 ttys001 0:02.26 /usr/bin/java -XX:NativeMemoryTracking=summary -Xss256M -Xmx10M Main 1000 99 | 100 | ``` 101 | 102 | The `jcmd` command tell us that the thread stacks have asked the OS for \~250GB of memory! 103 | The `ps` command The OS has granted \~250GB virtual memory. 104 | However only 93MB is actually being used (reported as RSS). 105 | This makes sense since each of the stacks would only be two call stacks deep: 106 | ```java.lang.Exception 107 | at Main.lambda$main$0(Main.java:10) 108 | at java.base/java.lang.Thread.run(Thread.java:831) 109 | ``` 110 | 111 | In summary you can create about ~4000 threads before the JVM or your OS crashes. 112 | This is variable depending on your OS and resources available. 113 | Increasing or decreasing the stack size doesn't seem to have an impact since it 99% virtual memory. 114 | We confirmed this by comparing virtual memory to resident set memory. 115 | It made sense since the entire stack is allocated, 116 | but only two call stacks were consumed by our program. 117 | 118 |
119 | 120 | -------------------------------------------------------------------------------- /create_threads_until_crash/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -ne 3 ]; then 4 | echo "./run.sh [stacksize] [heapsize] [numThreads]" 5 | exit -1 6 | fi 7 | 8 | javac Main.java 9 | retVal=$? 10 | if [ $retVal -ne 0 ]; then 11 | echo "compile error" 12 | exit $retVal 13 | fi 14 | 15 | java -XX:NativeMemoryTracking=summary -Xss$1 -Xmx$2 Main $3 16 | 17 | -------------------------------------------------------------------------------- /deadlock_lock/Main.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import java.util.concurrent.CyclicBarrier; 3 | import java.util.concurrent.locks.ReentrantLock; 4 | 5 | public final class Main { 6 | 7 | public static void main(String [] args) throws Exception { 8 | ReentrantLock lockA = new ReentrantLock(); 9 | ReentrantLock lockB = new ReentrantLock(); 10 | final CyclicBarrier barrier = new CyclicBarrier(2); 11 | 12 | Thread thread1 = new Thread(() -> { 13 | lockA.lock(); 14 | try { 15 | barrier.await(); 16 | } catch (Exception e) { // don't do this in production code :) 17 | } 18 | lockB.lock(); 19 | System.out.println("Got A then B"); 20 | lockB.unlock(); 21 | lockA.unlock(); 22 | }); 23 | thread1.start(); 24 | Thread thread2 = new Thread(() -> { 25 | lockB.lock(); 26 | try { 27 | barrier.await(); 28 | } catch (Exception e) { // don't do this in production code :) 29 | } 30 | lockA.lock(); 31 | System.out.println("Got B then A"); 32 | lockA.unlock(); 33 | lockB.unlock(); 34 | }); 35 | thread2.start(); 36 | 37 | thread1.join(); 38 | thread2.join(); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /deadlock_lock/README.md: -------------------------------------------------------------------------------- 1 | What happens when you get a deadlock with a lock in java? 2 | 3 | 1. Think about guess the answer. You will learn more if you guess first then 4 | check compared to immediately checking the answer. 5 | 2. Try to write some experimental code 6 | 3. Compare with my experiments 7 | 8 |
9 | Click to reveal the results of my experiment. 10 | 11 | To create a deadlock, we need to create a cycle in the wait for resource graph. 12 | The easiest way to do that is have one thread grab synchronized lock A then B. 13 | The other thread need to grab B then A. 14 | 15 | However, there's a catch! 16 | That won't reliably reproduce the problem 100% of the time. 17 | Sometimes, the thread will storm through and grab A then B before the other 18 | thread even starts to grab B. 19 | As a result, we introduce a barrier that will wait until thread 1 grab lock a and thread 2 grabs lock b. 20 | 21 | ``` 22 | Found one Java-level deadlock: 23 | ============================= 24 | "Thread-0": 25 | waiting for ownable synchronizer 0x000000070fe1a608, (a java.util.concurrent.locks.ReentrantLock$NonfairSync), 26 | which is held by "Thread-1" 27 | 28 | "Thread-1": 29 | waiting for ownable synchronizer 0x000000070fe1a5d8, (a java.util.concurrent.locks.ReentrantLock$NonfairSync), 30 | which is held by "Thread-0" 31 | 32 | Java stack information for the threads listed above: 33 | =================================================== 34 | "Thread-0": 35 | at jdk.internal.misc.Unsafe.park(java.base@16.0.1/Native Method) 36 | - parking to wait for <0x000000070fe1a608> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) 37 | at java.util.concurrent.locks.LockSupport.park(java.base@16.0.1/LockSupport.java:211) 38 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@16.0.1/AbstractQueuedSynchronizer.java:714) 39 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@16.0.1/AbstractQueuedSynchronizer.java:937) 40 | at java.util.concurrent.locks.ReentrantLock$Sync.lock(java.base@16.0.1/ReentrantLock.java:153) 41 | at java.util.concurrent.locks.ReentrantLock.lock(java.base@16.0.1/ReentrantLock.java:322) 42 | at Main.lambda$main$0(Main.java:18) 43 | at Main$$Lambda$1/0x0000000800c00a08.run(Unknown Source) 44 | at java.lang.Thread.run(java.base@16.0.1/Thread.java:831) 45 | "Thread-1": 46 | at jdk.internal.misc.Unsafe.park(java.base@16.0.1/Native Method) 47 | - parking to wait for <0x000000070fe1a5d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) 48 | at java.util.concurrent.locks.LockSupport.park(java.base@16.0.1/LockSupport.java:211) 49 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@16.0.1/AbstractQueuedSynchronizer.java:714) 50 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@16.0.1/AbstractQueuedSynchronizer.java:937) 51 | at java.util.concurrent.locks.ReentrantLock$Sync.lock(java.base@16.0.1/ReentrantLock.java:153) 52 | at java.util.concurrent.locks.ReentrantLock.lock(java.base@16.0.1/ReentrantLock.java:322) 53 | at Main.lambda$main$1(Main.java:30) 54 | at Main$$Lambda$2/0x0000000800c00c30.run(Unknown Source) 55 | at java.lang.Thread.run(java.base@16.0.1/Thread.java:831) 56 | 57 | Found 1 deadlock. 58 | ``` 59 | 60 | Just like with synchronized, sending `kill -s QUIT` will detect the deadlock and print it to standard out. 61 | You can use the stack traces to help debug and figure out what is going on. 62 | 63 |
64 | 65 | -------------------------------------------------------------------------------- /deadlock_lock/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -ne 0 ]; then 4 | echo "./run.sh" 5 | exit -1 6 | fi 7 | 8 | javac Main.java 9 | retVal=$? 10 | if [ $retVal -ne 0 ]; then 11 | echo "compile error" 12 | exit $retVal 13 | fi 14 | 15 | java -XX:NativeMemoryTracking=summary Main 16 | 17 | -------------------------------------------------------------------------------- /deadlock_synchronized/Main.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import java.util.concurrent.CyclicBarrier; 3 | 4 | public final class Main { 5 | 6 | public static void main(String [] args) throws Exception { 7 | final Object lockA = new Object(); 8 | final Object lockB = new Object(); 9 | final CyclicBarrier barrier = new CyclicBarrier(2); 10 | 11 | Thread thread1 = new Thread(() -> { 12 | synchronized(lockA) { 13 | try { 14 | barrier.await(); 15 | } catch (Exception e) { // don't do this in production code :) 16 | } 17 | synchronized(lockB) { 18 | System.out.println("Got A then B"); 19 | } 20 | } 21 | }); 22 | thread1.start(); 23 | Thread thread2 = new Thread(() -> { 24 | synchronized(lockB) { 25 | try { 26 | barrier.await(); 27 | } catch (Exception e) { // don't do this in production code :) 28 | } 29 | synchronized(lockA) { 30 | System.out.println("Got B then A"); 31 | } 32 | } 33 | }); 34 | thread2.start(); 35 | 36 | thread1.join(); 37 | thread2.join(); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /deadlock_synchronized/README.md: -------------------------------------------------------------------------------- 1 | What happens when you get a deadlock with synchronized in java? 2 | 3 | 1. Think about guess the answer. You will learn more if you guess first then 4 | check compared to immediately checking the answer. 5 | 2. Try to write some experimental code 6 | 3. Compare with my experiments 7 | 8 |
9 | Click to reveal the results of my experiment. 10 | 11 | To create a deadlock, we need to create a cycle in the wait for resource graph. 12 | The easiest way to do that is have one thread grab synchronized lock A then B. 13 | The other thread need to grab B then A. 14 | 15 | However, there's a catch! 16 | That won't reliably reproduce the problem 100% of the time. 17 | Sometimes, the thread will storm through and grab A then B before the other 18 | thread even starts to grab B. 19 | As a result, we introduce a barrier that will wait until thread 1 grab lock a and thread 2 grabs lock b. 20 | 21 | ``` 22 | ./run.sh 23 | 24 | ``` 25 | As expected, the process never ends because thread 1 and 2 are deadlocked, wait for the other to give up their resource. 26 | Java does not detect this deadlock for you unless you ask it to. 27 | We can send `kill -s QUIT` to tell the JVM to send a status report to standard out. 28 | 29 | ``` 30 | ./run.sh > out.txt & 31 | kill -s QUIT $java_pid 32 | less out.txt 33 | ... 34 | Found one Java-level deadlock: 35 | ============================= 36 | "Thread-0": 37 | waiting to lock monitor 0x00007fc8b1f1cb00 (object 0x000000070fe1a5b8, a java.lang.Object), 38 | which is held by "Thread-1" 39 | 40 | "Thread-1": 41 | waiting to lock monitor 0x00007fc8b5306eb0 (object 0x000000070fe1a5a8, a java.lang.Object), 42 | which is held by "Thread-0" 43 | 44 | Java stack information for the threads listed above: 45 | =================================================== 46 | "Thread-0": 47 | at Main.lambda$main$0(Main.java:18) 48 | - waiting to lock <0x000000070fe1a5b8> (a java.lang.Object) 49 | - locked <0x000000070fe1a5a8> (a java.lang.Object) 50 | at Main$$Lambda$1/0x0000000800c00a08.run(Unknown Source) 51 | at java.lang.Thread.run(java.base@16.0.1/Thread.java:831) 52 | "Thread-1": 53 | at Main.lambda$main$1(Main.java:30) 54 | - waiting to lock <0x000000070fe1a5a8> (a java.lang.Object) 55 | - locked <0x000000070fe1a5b8> (a java.lang.Object) 56 | at Main$$Lambda$2/0x0000000800c00c30.run(Unknown Source) 57 | at java.lang.Thread.run(java.base@16.0.1/Thread.java:831) 58 | 59 | Found 1 deadlock. 60 | ... 61 | ``` 62 | 63 | It found the deadlock but it does not do anything about it. 64 | It even says the threads, locks, and stacktraces involved in the deadlock! 65 | That's very useful for debugging. 66 | At this point there is nothing we can do to recover, unless your application exposes a way to kill or interrupt threads. 67 | 68 |
69 | 70 | -------------------------------------------------------------------------------- /deadlock_synchronized/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ "$#" -ne 0 ]; then 4 | echo "./run.sh" 5 | exit -1 6 | fi 7 | 8 | javac Main.java 9 | retVal=$? 10 | if [ $retVal -ne 0 ]; then 11 | echo "compile error" 12 | exit $retVal 13 | fi 14 | 15 | java -XX:NativeMemoryTracking=summary Main 16 | 17 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM adoptopenjdk/openjdk9:alpine 2 | #FROM openjdk:8-jdk-alpine 3 | MAINTAINER Joseph Mate 4 | 5 | RUN apk add procps 6 | 7 | ADD Main.class / 8 | 9 | 10 | ENTRYPOINT ["java", "Main"] 11 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/Main.java: -------------------------------------------------------------------------------- 1 | import java.util.concurrent.Executors; 2 | import java.util.concurrent.ScheduledExecutorService; 3 | import java.util.concurrent.TimeUnit; 4 | 5 | public final class Main { 6 | 7 | public static void main(String [] args) throws Exception { 8 | ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); 9 | executorService.scheduleWithFixedDelay( 10 | () -> System.out.println("Hello World!"), 11 | 0, // Initial delay 12 | 10, // delay 13 | TimeUnit.MINUTES 14 | ); 15 | while (true) { 16 | Thread.sleep(10000); 17 | } 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/out/production/scheduled_thread_pool_executor_100_cpu/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/out/production/scheduled_thread_pool_executor_100_cpu/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/out/production/scheduled_thread_pool_executor_100_cpu/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/out/production/scheduled_thread_pool_executor_100_cpu/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/out/production/scheduled_thread_pool_executor_100_cpu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | MAINTAINER Joseph Mate 3 | 4 | RUN apk add procps 5 | 6 | ADD Main.class / 7 | 8 | 9 | ENTRYPOINT ["java", "Main"] 10 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/out/production/scheduled_thread_pool_executor_100_cpu/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # install an old JDK from like u92 4 | # https://www.oracle.com/ca-en/java/technologies/javase/javase8-archive-downloads.html 5 | # /usr/libexec/java_home -V 6 | # to list java versions 7 | 8 | 9 | if [ "$#" -ne 0 ]; then 10 | echo "./run.sh" 11 | exit -1 12 | fi 13 | 14 | old_java_home=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home 15 | #old_java_home=/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home 16 | 17 | $old_java_home/bin/javac Main.java 18 | retVal=$? 19 | if [ $retVal -ne 0 ]; then 20 | echo "compile error" 21 | exit $retVal 22 | fi 23 | 24 | #$old_java_home/bin/java Main 25 | 26 | docker build . --tag josephmate/scheduled-executor-service-100-cpu:1.0.0 27 | docker run --name scheduled_executor_service josephmate/scheduled-executor-service-100-cpu:1.0.0 28 | 29 | # in another terminal 30 | # docker exec --tty --interactive scheduled_executor_service sh 31 | # top 32 | # top -H -p 1 33 | 34 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/out/production/scheduled_thread_pool_executor_100_cpu/scheduled_thread_pool_executor_100_cpu.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # install an old JDK from like u92 4 | # https://www.oracle.com/ca-en/java/technologies/javase/javase8-archive-downloads.html 5 | # /usr/libexec/java_home -V 6 | # to list java versions 7 | 8 | 9 | if [ "$#" -ne 0 ]; then 10 | echo "./run.sh" 11 | exit -1 12 | fi 13 | 14 | old_java_home=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home 15 | #old_java_home=/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home 16 | 17 | $old_java_home/bin/javac Main.java 18 | retVal=$? 19 | if [ $retVal -ne 0 ]; then 20 | echo "compile error" 21 | exit $retVal 22 | fi 23 | 24 | #$old_java_home/bin/java Main 25 | 26 | docker build . --tag josephmate/scheduled-executor-service-100-cpu:1.0.0 27 | docker run --name scheduled_executor_service josephmate/scheduled-executor-service-100-cpu:1.0.0 28 | 29 | # in another terminal 30 | # docker exec --tty --interactive scheduled_executor_service sh 31 | # top 32 | # top -H -p 1 33 | 34 | -------------------------------------------------------------------------------- /experimental/scheduled_thread_pool_executor_100_cpu/scheduled_thread_pool_executor_100_cpu.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /large_array/Main.java: -------------------------------------------------------------------------------- 1 | import java.util.ArrayList; 2 | import java.util.LinkedList; 3 | import java.util.List; 4 | 5 | public class Main { 6 | public static void main(String[] args) { 7 | final List big; 8 | if ("array".equals(args[0])) { 9 | big = new ArrayList<>(); 10 | } else { 11 | big = new LinkedList<>(); 12 | } 13 | 14 | long bigSize = (long)Integer.MAX_VALUE+1; 15 | for (long i = 0; i < bigSize; i++) { 16 | try { 17 | big.add(i); 18 | } catch (Throwable t) { 19 | System.out.println("Died at: " + i); 20 | throw t; 21 | } 22 | } 23 | 24 | System.out.println(big.size()); 25 | } 26 | } -------------------------------------------------------------------------------- /large_array/README.md: -------------------------------------------------------------------------------- 1 | Does ArrayList have a limit? LinkedList? 2 | 3 |
4 | Take a guess without an experiment. 5 | The List interface uses an integer for the size. 6 | As a result, the limit must be 2^31-1. 7 | But what happens if you tried to go beyond? 8 | Would the size overflow and ArrayList tries to allocate an array with negative 9 | size? 10 | Would LinkedList let you keep adding beyond the limit since it's not backed by 11 | an array? 12 |
13 | 14 |
15 | Click to reveal the solution by experiment. 16 | 17 | 18 | new ArrayList<>(Integer.MAX_VALUE-1): 19 | ``` 20 | ./run.sh array 21 | Exception in thread "main" java.lang.OutOfMemoryError: Requested array size 22 | exceeds VM limit 23 | at java.base/java.util.ArrayList.(ArrayList.java:156) 24 | at Main.main(Main.java:9) 25 | Error 26 | ``` 27 | JVM doesn't even let us allocate an array of that size. 28 | 29 | The exception probably comes from native code because this is the line that's 30 | failing: 31 | ```java 32 | 147 /** 33 | 148 * Constructs an empty list with the specified initial capacity. 34 | 149 * 35 | 150 * @param initialCapacity the initial capacity of the list 36 | 151 * @throws IllegalArgumentException if the specified initial capacity 37 | 152 * is negative 38 | 153 */ 39 | 154 public ArrayList(int initialCapacity) { 40 | 155 if (initialCapacity > 0) { 41 | 42 | ////////////////////////////// THIS LINE ////////////////////////////// 43 | 156 this.elementData = new Object[initialCapacity]; 44 | /////////////////////////////////////////////////////////////////////// 45 | 46 | 157 } else if (initialCapacity == 0) { 47 | 158 this.elementData = EMPTY_ELEMENTDATA; 48 | 159 } else { 49 | 160 throw new IllegalArgumentException("Illegal Capacity: "+ 50 | 161 initialCapacity); 51 | 162 } 52 | 163 } 53 | ``` 54 | 55 | ArrayList<>() and -Xmx10g: 56 | ``` 57 | ./run.sh array 58 | Died at: 354836040 59 | Exception in thread "main" java.lang.OutOfMemoryError: Java heap space 60 | at java.base/java.util.Arrays.copyOf(Arrays.java:3511) 61 | at java.base/java.util.Arrays.copyOf(Arrays.java:3480) 62 | at java.base/java.util.ArrayList.grow(ArrayList.java:237) 63 | at java.base/java.util.ArrayList.grow(ArrayList.java:244) 64 | at java.base/java.util.ArrayList.add(ArrayList.java:454) 65 | at java.base/java.util.ArrayList.add(ArrayList.java:467) 66 | at Main.main(Main.java:17) 67 | Error 68 | ``` 69 | My laptop might not have enought memory for this experiment :( 70 | 71 | 72 | ArrayList<>() and -Xmx16g: 73 | ``` 74 | ./run.sh array 75 | Died at: 532254060 76 | Exception in thread "main" java.lang.OutOfMemoryError: Java heap space 77 | at java.base/java.util.Arrays.copyOf(Arrays.java:3511) 78 | at java.base/java.util.Arrays.copyOf(Arrays.java:3480) 79 | at java.base/java.util.ArrayList.grow(ArrayList.java:237) 80 | at java.base/java.util.ArrayList.grow(ArrayList.java:244) 81 | at java.base/java.util.ArrayList.add(ArrayList.java:454) 82 | at java.base/java.util.ArrayList.add(ArrayList.java:467) 83 | at Main.main(Main.java:17) 84 | Error 85 | ``` 86 | 87 | LinkedList<>() 88 | ``` 89 | ./run.sh link 90 | Exception in thread "main" java.lang.OutOfMemoryError: Java heap space 91 | at Main.main(Main.java:17) 92 | Error 93 | ``` 94 | 95 |
96 | -------------------------------------------------------------------------------- /large_array/run.sh: -------------------------------------------------------------------------------- 1 | 2 | javac Main.java 3 | retVal=$? 4 | if [ $retVal -ne 0 ]; then 5 | echo "Error" 6 | exit $retVal 7 | fi 8 | 9 | 10 | java -Xmx16g Main $1 11 | retVal=$? 12 | if [ $retVal -ne 0 ]; then 13 | echo "Error" 14 | exit $retVal 15 | fi 16 | -------------------------------------------------------------------------------- /max_connections/README.md: -------------------------------------------------------------------------------- 1 | What's the max num of connections a server and client can concurrently keep open? 2 | 3 | 4 | 5 |
6 | Click to reveal the short summary. 7 | 8 | The theoretical limit is 1 quadrillion because: 9 | 10 | 1. [number of IP addresses]x[num of ports] 11 | 2. because the server multiplexes the connections from the client using that. 12 | 3. 32 bits for the address and 16 bits for the port 13 | 4. In total is 2^48. 14 | 5. Which is about a billion connects (log(2^48)/log(10)=14.449)! 15 | 16 | ## Mac 17 | 18 | But the practical limit is much lower on my Mac. 19 | ``` 20 | 2.5 GHz Quad-Core Intel Core i7 21 | 16 GB 1600 MHz DDR3 22 | ``` 23 | 24 | To get this to work, you need to jump through a lot of hoops. 25 | 26 | First, setup sysctl so that we can have as many file descriptors as we want. 27 | ``` 28 | sudo sysctl kern.maxfiles=2000000 kern.maxfilesperproc=1000000 29 | kern.maxfiles: 49152 -> 2000000 30 | kern.maxfilesperproc: 24576 -> 1000000 31 | sysctl -a | grep maxfiles 32 | kern.maxfiles: 2000000 33 | kern.maxfilesperproc: 1000000 34 | ``` 35 | 36 | Setup loop back addresses on 10.0.0.X to overcome the src tcp port limit: 37 | ``` 38 | for i in `seq 0 200`; do sudo ifconfig lo0 alias 10.0.0.$i/8 up ; done 39 | ``` 40 | 41 | For some reason we need to set the soft limit too even though the hard limit is 42 | unlimited. I cannot explain why this step was necessary. Without it, it wouldn't 43 | work. 44 | ``` 45 | ulimit -Sn 1000000 46 | ``` 47 | 48 | Run java with `-XX:-MaxFDLimit` to prevent the JVM from limiting file 49 | descriptors: 50 | ``` 51 | java -XX:-MaxFDLimit Main 80000 52 | ... 53 | SERVER: 79998 from /10.0.0.15:64069 to /127.0.0.1:9999 msg 79998 54 | SERVER: 79999 from /10.0.0.15:64070 to /127.0.0.1:9999 msg 79999 55 | Shutting down sockets from client 56 | Shutting down sockets to server 57 | ``` 58 | What's strange is that after running this, even though the above program 59 | completed and released all resources, my Mac becomes less responsive and then 60 | eventually crashes. As a result, I declare this the pratical limit for my Mac. 61 | 62 | At this point I lack the skills to keep digging to figure out why my Mac is 63 | crashing; however, I can try to see if it works better on my Linux machine. 64 | 65 | 66 |
67 | 68 |
69 | Click to reveal the journey on Mac. 70 | 71 | At about 5k my Mac runs out of file descriptors. 72 | ``` 73 | SERVER: 5115 from /127.0.0.1:51940 to /127.0.0.1:9999 74 | Exception in thread "main" java.lang.ExceptionInInitializerError 75 | at java.base/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70) 76 | at java.base/sun.nio.ch.NioSocketImpl.lambda$closerFor$0(NioSocketImpl.java:1203) 77 | at java.base/jdk.internal.ref.CleanerImpl$PhantomCleanableRef.performCleanup(CleanerImpl.java:178) 78 | at java.base/jdk.internal.ref.PhantomCleanable.clean(PhantomCleanable.java:133) 79 | at java.base/sun.nio.ch.NioSocketImpl.tryClose(NioSocketImpl.java:854) 80 | at java.base/sun.nio.ch.NioSocketImpl.close(NioSocketImpl.java:906) 81 | at java.base/java.net.SocksSocketImpl.close(SocksSocketImpl.java:562) 82 | at java.base/java.net.Socket.close(Socket.java:1585) 83 | at Main.main(Main.java:123) 84 | Caused by: java.io.IOException: Too many open files 85 | at java.base/sun.nio.ch.FileDispatcherImpl.init(Native Method) 86 | at java.base/sun.nio.ch.FileDispatcherImpl.(FileDispatcherImpl.java:38) 87 | ... 9 more 88 | ``` 89 | 90 | Some sources say I need to adjust `ulimit -n`: 91 | ``` 92 | ulimit -n 93 | 256 94 | ``` 95 | 96 | I updated it: 97 | ``` 98 | ulimit -S -n 1000000 99 | ulimit -n 100 | 1000000 101 | ``` 102 | and still get the problem. 103 | 104 | 105 | Others say `launchctl limit maxfiles`, but that is also already `unlimited`: 106 | ``` 107 | launchctl limit maxfiles 108 | maxfiles 256 unlimited 109 | ``` 110 | 111 | This could be it: 112 | ``` 113 | sysctl -a | grep maxfiles 114 | kern.maxfiles: 49152 115 | kern.maxfilesperproc: 24576 116 | ``` 117 | Makes sense because I would use about 4 file descriptions per process: 118 | 1. server, incoming 119 | 2. server, outgoing 120 | 3. client, incoming 121 | 4. client, outgoing 122 | 123 | You can set it according to this [stackoverflow answer](https://superuser.com/a/1644788) 124 | ``` 125 | sudo sysctl kern.maxfiles=2000000 kern.maxfilesperproc=1000000 126 | kern.maxfiles: 49152 -> 2000000 127 | kern.maxfilesperproc: 24576 -> 1000000 128 | sysctl -a | grep maxfiles 129 | kern.maxfiles: 2000000 130 | kern.maxfilesperproc: 1000000 131 | ``` 132 | 133 | But that didn't help, I still get the error. I think I might have mis-used the 134 | utlimit command because with -n I get: 135 | ``` 136 | TODO 137 | ``` 138 | 139 | 140 | https://superuser.com/a/1171028 141 | ``` 142 | sudo vim /Library/LaunchDaemons/limit.maxfiles.plist 143 | ``` 144 | with contents: 145 | ``` 146 | 147 | 149 | 150 | 151 | Label 152 | limit.maxfiles 153 | ProgramArguments 154 | 155 | launchctl 156 | limit 157 | maxfiles 158 | 64000 159 | 524288 160 | 161 | RunAtLoad 162 | 163 | ServiceIPC 164 | 165 | 166 | 167 | ``` 168 | 169 | Load the configs (this will load on restarts too): 170 | ``` 171 | sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist 172 | sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist 173 | launchctl limit maxfiles 174 | ``` 175 | 176 | NOPE! Still getting stuck at 5000... 177 | /Library/LaunchDaemons/com.startup.sysctl.plist 178 | ``` 179 | sudo vim /Library/LaunchDaemons/com.startup.sysctl.plist 180 | ``` 181 | with contents: 182 | ``` 183 | 184 | 186 | 187 | 188 | Label 189 | com.startup.sysctl 190 | LaunchOnlyOnce 191 | 192 | ProgramArguments 193 | 194 | /usr/sbin/sysctl 195 | kern.maxfiles=40480 196 | kern.maxfilesperproc=28000 197 | 198 | RunAtLoad 199 | 200 | 201 | 202 | ``` 203 | install: 204 | ``` 205 | chown root:wheel /Library/LaunchDaemons/com.startup.sysctl.plist 206 | launchctl load /Library/LaunchDaemons/com.startup.sysctl.plist 207 | ``` 208 | 209 | AHHHHH STILL 5000. Let me restart and see. 210 | 211 | Fixed with: https://superuser.com/a/1398360 212 | https://docs.oracle.com/en/java/javase/16/docs/specs/man/java.html 213 | 214 | > `-XX:-MaxFDLimit` : Disables the attempt to set the soft limit for the 215 | > number of open file descriptors to the hard limit. By default, this option is 216 | > enabled on all platforms, but is ignored on Windows. The only time that you 217 | > may need to disable this is on Mac OS, where its use imposes a maximum of 218 | > 10240, which is lower than the actual system maximum. 219 | 220 | ``` 221 | java -XX:-MaxFDLimit -cp out/production/max_connections Main 6000 222 | ``` 223 | 224 | Looks like I am running out of client ports: 225 | ``` 226 | Exception in thread "main" java.net.BindException: Can't assign requested address 227 | at java.base/sun.nio.ch.Net.bind0(Native Method) 228 | at java.base/sun.nio.ch.Net.bind(Net.java:555) 229 | at java.base/sun.nio.ch.Net.bind(Net.java:544) 230 | at java.base/sun.nio.ch.NioSocketImpl.bind(NioSocketImpl.java:643) 231 | at java.base/java.net.DelegatingSocketImpl.bind(DelegatingSocketImpl.java:94) 232 | at java.base/java.net.Socket.bind(Socket.java:682) 233 | at java.base/java.net.Socket.(Socket.java:506) 234 | at java.base/java.net.Socket.(Socket.java:403) 235 | at Main.main(Main.java:137) 236 | ``` 237 | 238 | https://phoenixframework.org/blog/the-road-to-2-million-websocket-connections 239 | encountered a similar issue. 240 | 241 | I can workaround this by creating 242 | fake addresses for each client and those addresses to /etc/hosts. 243 | 244 | Before my /etc/hosts was: 245 | ``` 246 | ## 247 | # Host Database 248 | # 249 | # localhost is used to configure the loopback interface 250 | # when the system is booting. Do not change this entry. 251 | ## 252 | 127.0.0.1 localhost 253 | 255.255.255.255 broadcasthost 254 | ::1 localhost 255 | # Added by Docker Desktop 256 | # To allow the same kube context to work on the host and the container: 257 | 127.0.0.1 kubernetes.docker.internal 258 | # End of section 259 | ``` 260 | Using private address range `10.*`, I can add 256^3 addresses which is ~16 261 | million which should be more than enough. I actually only need about 1000000/5000=200. 262 | 263 | ``` 264 | for i in `seq 0 200`; do echo "10.0.0.$i localhost" ; done > out.txt 265 | ``` 266 | then paste out onto /etc/hosts 267 | 268 | then 269 | ``` 270 | sudo killall -HUP mDNSResponder 271 | ``` 272 | to restart DNS 273 | 274 | 275 | https://serverfault.com/questions/402744/assigning-multiple-ip-addresses-to-localhost-os-x-10-6 276 | ``` 277 | sudo ifconfig lo0 alias 10.0.0.0/8 up 278 | ping 10.0.0.0 279 | ``` 280 | 281 | To add: 282 | ``` 283 | for i in `seq 0 200`; do sudo ifconfig lo0 alias 10.0.0.$i/8 up ; done 284 | ``` 285 | 286 | To test: 287 | ``` 288 | for i in `seq 0 200`; do ping -c 1 10.0.0.$i ; done 289 | ``` 290 | 291 | To remove: 292 | ``` 293 | for i in `seq 0 200`; do sudo ifconfig lo0 alias 10.0.0.$i ; done 294 | ``` 295 | 296 | Now re-running with the above mac configs, plus the modified java program to use 297 | 10.0.0.X as the src address for the clients: 298 | ```Exception in thread "main" java.io.IOException: Too many open files 299 | at java.base/sun.nio.ch.Net.accept(Native Method) 300 | at java.base/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:755) 301 | at java.base/java.net.ServerSocket.implAccept(ServerSocket.java:681) 302 | at 303 | java.base/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647) 304 | at java.base/java.net.ServerSocket.implAccept(ServerSocket.java:623) 305 | at java.base/java.net.ServerSocket.implAccept(ServerSocket.java:580) 306 | at java.base/java.net.ServerSocket.accept(ServerSocket.java:538) 307 | at Main.lambda$main$1(Main.java:47) 308 | at java.base/java.lang.Thread.run(Thread.java:831) 309 | Suppressed: java.lang.NoClassDefFoundError: Could not initialize class 310 | sun.nio.ch.FileDispatcherImpl 311 | at 312 | java.base/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70) 313 | at 314 | java.base/sun.nio.ch.NioSocketImpl.lambda$closerFor$0(NioSocketImpl.java:1203) 315 | at 316 | java.base/jdk.internal.ref.CleanerImpl$PhantomCleanableRef.performCleanup(CleanerImpl.java:178) 317 | at 318 | java.base/jdk.internal.ref.PhantomCleanable.clean(PhantomCleanable.java:133) 319 | at 320 | java.base/sun.nio.ch.NioSocketImpl.tryClose(NioSocketImpl.java:854) 321 | at 322 | java.base/sun.nio.ch.NioSocketImpl.close(NioSocketImpl.java:906) 323 | at java.base/java.net.ServerSocket.close(ServerSocket.java:723) 324 | at Main.lambda$main$1(Main.java:86) 325 | ... 1 more 326 | java.lang.ExceptionInInitializerError 327 | at java.base/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70) 328 | at 329 | java.base/sun.nio.ch.NioSocketImpl.lambda$closerFor$0(NioSocketImpl.java:1203) 330 | at 331 | java.base/jdk.internal.ref.CleanerImpl$PhantomCleanableRef.performCleanup(CleanerImpl.java:178) 332 | at 333 | java.base/jdk.internal.ref.PhantomCleanable.clean(PhantomCleanable.java:133) 334 | at java.base/sun.nio.ch.NioSocketImpl.tryClose(NioSocketImpl.java:854) 335 | at java.base/sun.nio.ch.NioSocketImpl.close(NioSocketImpl.java:906) 336 | at java.base/java.net.SocksSocketImpl.close(SocksSocketImpl.java:562) 337 | at java.base/java.net.Socket.close(Socket.java:1585) 338 | at Main.main(Main.java:174) 339 | ``` 340 | out of file descriptors again. 341 | 342 | For some reason my settings got reset so I had to rerun: 343 | ``` 344 | sudo sysctl kern.maxfiles=2000000 kern.maxfilesperproc=1000000 345 | kern.maxfiles: 49152 -> 2000000 346 | kern.maxfilesperproc: 24576 -> 1000000 347 | ``` 348 | 349 | Now it's working again: 350 | ``` 351 | java -XX:-MaxFDLimit Main 20000 352 | ... 353 | SERVER: 19998 from /10.0.0.3:50637 to /127.0.0.1:9999 msg 19998 354 | SERVER: 19999 from /10.0.0.3:50638 to /127.0.0.1:9999 msg 19999 355 | Shutting down sockets from client 356 | Shutting down sockets to server 357 | 358 | java -XX:-MaxFDLimit Main 40000 359 | SERVER: 39998 from /10.0.0.7:57875 to /127.0.0.1:9999 msg 39998 360 | SERVER: 39999 from /10.0.0.7:57876 to /127.0.0.1:9999 msg 39999 361 | Shutting down sockets from client 362 | Shutting down sockets to server 363 | ``` 364 | 365 | At 80,000 my computer crashed. Let me try again. 366 | ``` 367 | java -XX:-MaxFDLimit Main 80000 368 | ... 369 | SERVER: 79998 from /10.0.0.15:64069 to /127.0.0.1:9999 msg 79998 370 | SERVER: 79999 from /10.0.0.15:64070 to /127.0.0.1:9999 msg 79999 371 | Shutting down sockets from client 372 | Shutting down sockets to server 373 | ``` 374 | however, a few moments after it completed the experiment my laptop died again. I 375 | can safely say that that's the limit for my Mac. 376 | 377 |
378 | 379 |
380 | Click to reveal the journey on Linux. 381 | 382 | ``` 383 | AMD FX(tm)-6300 Six-Core Processor 384 | 8GiB 1600 MHz 385 | 386 | ``` 387 | 388 | 389 | https://superuser.com/a/1089140 390 | ``` 391 | for i in `seq 0 200`; do sudo ip addr add 10.0.0.$i/8 dev lo; done 392 | ``` 393 | 394 | To remove: 395 | ``` 396 | for i in `seq 0 200`; do sudo ip addr del 10.0.0.$i/8 dev lo; done 397 | ``` 398 | 399 | I didn't need to adjust limits because by default it's already above 1,000,000: 400 | ``` 401 | ulimit -Hn 402 | 1048576 403 | ``` 404 | 405 | Nevermind there's something wrong with my limits: 406 | ``` 407 | java -XX:-MaxFDLimit Main 80000 408 | . 409 | . 410 | . 411 | SERVER: 472 from /10.0.0.0:47375 to /127.0.0.1:9999 412 | Exception in thread "main" java.net.SocketException: Too many open files (Accept 413 | failed) 414 | at java.net.PlainSocketImpl.socketAccept(Native Method) 415 | atjava.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) 416 | at java.net.ServerSocket.implAccept(ServerSocket.java:560) 417 | at java.net.ServerSocket.accept(ServerSocket.java:528) 418 | at Main.lambda$main$1(Main.java:47) 419 | at java.lang.Thread.run(Thread.java:748) 420 | java.net.SocketException: Too many open files 421 | at java.net.Socket.createImpl(Socket.java:478) 422 | at java.net.Socket.(Socket.java:449) 423 | at java.net.Socket.(Socket.java:346) 424 | at Main.main(Main.java:125) 425 | 426 | ``` 427 | 428 | Maybe it's the soft limit? 429 | ``` 430 | ulimit -Sn 431 | 1024 432 | ``` 433 | 434 | ``` 435 | ulimit -Sn 1000000 436 | ulimit -Sn 437 | 1000000 438 | ``` 439 | 440 | ``` 441 | java -XX:-MaxFDLimit Main 80000 442 | . 443 | . 444 | . 445 | SERVER: 79998 from /10.0.0.15:57459 to /127.0.0.1:9999 msg 79998 446 | SERVER: 79999 from /10.0.0.15:48259 to /127.0.0.1:9999 msg 79999 447 | Shutting down sockets to server 448 | Shutting down sockets from client 449 | ``` 450 | 451 | ``` 452 | java -XX:-MaxFDLimit Main 160000 453 | . 454 | . 455 | . 456 | SERVER: 159998 from /10.0.0.31:41043 to /127.0.0.1:9999 msg 159998 457 | SERVER: 159999 from /10.0.0.31:41513 to /127.0.0.1:9999 msg 159999 458 | Shutting down sockets to server 459 | Shutting down sockets from client 460 | ``` 461 | # second 160000 indicates to use 160,000 connections per client ip address 462 | java -XX:-MaxFDLimit Main 160000 160000 463 | If you try to use all the ports from a single client IP you run out after 28,000: 464 | ``` 465 | SERVER: 28230 from /10.0.0.0:39902 to /127.0.0.1:9999 466 | SERVER: 28231 from /10.0.0.0:39906 to /127.0.0.1:9999 467 | Exception in thread "main" java.net.BindException: Address already in use (Bind failed) 468 | at java.net.PlainSocketImpl.socketBind(Native Method) 469 | at java.net.AbstractPlainSocketImpl.bind(AbstractPlainSocketImpl.java:387) 470 | at java.net.Socket.bind(Socket.java:662) 471 | at java.net.Socket.(Socket.java:451) 472 | at java.net.Socket.(Socket.java:346) 473 | at Main.main(Main.java:125) 474 | . 475 | . 476 | . 477 | ``` 478 | 479 | 480 | ``` 481 | java -XX:-MaxFDLimit Main 320000 482 | ... 483 | SERVER: 319998 from /10.0.0.63:39921 to /127.0.0.1:9999 msg 319998 484 | SERVER: 319999 from /10.0.0.63:36455 to /127.0.0.1:9999 msg 319999 485 | Shutting down sockets to server 486 | Shutting down sockets from client 487 | ``` 488 | 489 | Need increase ulimit 490 | ``` 491 | java -XX:-MaxFDLimit Main 640000 492 | ... 493 | SERVER: 499988 from /10.0.0.99:45337 to /127.0.0.1:9999 494 | SERVER: 499989 from /10.0.0.99:56891 to /127.0.0.1:9999 495 | java.net.SocketException: Too many open files (Accept failed) 496 | at java.net.PlainSocketImpl.socketAccept(Native Method) 497 | at 498 | java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) 499 | at java.net.ServerSocket.implAccept(ServerSocket.java:560) 500 | at java.net.ServerSocket.accept(ServerSocket.java:528) 501 | at Main.lambda$main$1(Main.java:47) 502 | at java.lang.Thread.run(Thread.java:748) 503 | ``` 504 | 505 | At that point linux complains if I try to increase the soft or hard limit beyond 506 | ``` 507 | ulimit -Sn 2000000 508 | bash: ulimit: open files: cannot modify limit: Invalid argument 509 | ulimit -Sn 1048576 510 | 511 | 512 | 513 | ulimit -Hn 514 | 1048576 515 | joseph@Joseph:~/projects/java-by-experiments/max_connections/src$ ulimit -Hn 516 | 1048577 517 | bash: ulimit: open files: cannot modify limit: Operation not permitted 518 | 519 | ulimit -Hn 1048576 520 | 521 | ulimit -Hn 1048575 522 | 523 | ``` 524 | 525 | https://serverfault.com/a/1029846 526 | 527 | ``` 528 | cat /proc/sys/fs/file-max 529 | 9223372036854775807 530 | cat /proc/sys/fs/file-nr 531 | 8864 0 9223372036854775807 532 | 533 | ``` 534 | 535 | ``` 536 | sudo su 537 | # 2^25 538 | sysctl -w fs.nr_open=33554432 539 | fs.nr_open = 33554432 540 | ulimit -Hn 33554432 541 | ulimit -Sn 33554432 542 | 543 | ``` 544 | 545 | It worked! 546 | ``` 547 | java -XX:-MaxFDLimit Main 640000 548 | ... 549 | SERVER: 639998 from /10.0.0.127:47321 to /127.0.0.1:9999 msg 639998 550 | SERVER: 639999 from /10.0.0.127:48501 to /127.0.0.1:9999 msg 639999 551 | Shutting down sockets to server 552 | Shutting down sockets from client 553 | ``` 554 | 555 | I tried `1,000,000` but my computer became unresponsive. If I moved my mouse, it 556 | moved a minute later. 557 | 558 |
559 | -------------------------------------------------------------------------------- /max_connections/data/collapsed.wall.1.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/collapsed.wall.1.txt.gz -------------------------------------------------------------------------------- /max_connections/data/collapsed.wall.840000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/collapsed.wall.840000.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.1.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.1.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.1.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.1.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.10000.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.10000.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.10000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.10000.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.100000.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.100000.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.100000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.100000.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.1000000.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.1000000.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.1000000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.1000000.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.640000.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.640000.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.640000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.640000.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.800000.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.800000.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.800000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.800000.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.840000.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.840000.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.840000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.840000.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.880000.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.880000.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.880000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.880000.txt.gz -------------------------------------------------------------------------------- /max_connections/data/out.900000.sar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.900000.sar.gz -------------------------------------------------------------------------------- /max_connections/data/out.900000.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/max_connections/data/out.900000.txt.gz -------------------------------------------------------------------------------- /max_connections/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | NUM_CONNECTIONS=$1 4 | 5 | javac src/Main.java 6 | 7 | sar -o out.$NUM_CONNECTIONS.sar -A 1 3600 2>&1 > /dev/null & 8 | SAR_PID=$! 9 | 10 | echo running: 11 | echo java \ 12 | -agentpath:/home/joseph/lib/async-profiler-2.7-linux-x64/build/libasyncProfiler.so=start,collapsed,event=wall,file=collapsed.wall.$NUM_CONNECTIONS.txt \ 13 | -cp src/ \ 14 | Main $NUM_CONNECTIONS 15 | 16 | java \ 17 | -agentpath:/home/joseph/lib/async-profiler-2.7-linux-x64/build/libasyncProfiler.so=start,collapsed,event=wall,file=collapsed.wall.$NUM_CONNECTIONS.txt \ 18 | -cp src/ \ 19 | Main $NUM_CONNECTIONS \ 20 | | tee out.$NUM_CONNECTIONS.txt 21 | 22 | kill $SAR_PID 23 | -------------------------------------------------------------------------------- /max_connections/src/Main.java: -------------------------------------------------------------------------------- 1 | import java.io.IOException; 2 | import java.net.InetAddress; 3 | import java.net.InetSocketAddress; 4 | import java.net.ServerSocket; 5 | import java.net.Socket; 6 | import java.nio.ByteBuffer; 7 | import java.util.ArrayList; 8 | import java.util.Collections; 9 | import java.util.List; 10 | import java.util.concurrent.Semaphore; 11 | import java.util.*; 12 | import java.text.*; 13 | 14 | public class Main { 15 | 16 | private static final int SERVER_PORT = 9999; 17 | private static final TimeZone tz = TimeZone.getTimeZone("UTC"); 18 | private static final DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss:SSS'Z'"); // Quoted "Z" to indicate UTC, no timezone offset 19 | static { 20 | df.setTimeZone(tz); 21 | } 22 | 23 | public static void print(String msg) { 24 | System.out.print(df.format(new Date())); 25 | System.out.print("\t"); 26 | System.out.println(msg); 27 | } 28 | 29 | public static void main(String[] args) throws Exception { 30 | 31 | int numberOfConnections = Integer.parseInt(args[0]); 32 | int connectionsPerAddress = 5000; 33 | if (args.length == 2) { 34 | connectionsPerAddress = Integer.parseInt(args[1]); 35 | } 36 | 37 | Semaphore waitForServerStartup = new Semaphore(0); 38 | 39 | Thread serverThread = new Thread( 40 | () -> { 41 | List tcpSocketsFromClient = Collections.synchronizedList( 42 | new ArrayList<>(numberOfConnections)); 43 | 44 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 45 | print("Shutting down sockets from client"); 46 | try { 47 | for(Socket socket : tcpSocketsFromClient) { 48 | socket.close();; 49 | } 50 | } catch (IOException e) { 51 | 52 | } 53 | })); 54 | 55 | try (ServerSocket serverSocket = new ServerSocket(SERVER_PORT, numberOfConnections, null)) { 56 | waitForServerStartup.release(); 57 | 58 | // accumulate all the connections 59 | for (int i = 0; i < numberOfConnections; i++) { 60 | Socket tcpSocketFromClient = serverSocket.accept(); 61 | tcpSocketsFromClient.add(tcpSocketFromClient); 62 | print( 63 | "SERVER: " + i 64 | + " from " + tcpSocketFromClient.getInetAddress() 65 | + ":" + tcpSocketFromClient.getPort() 66 | + " to " + tcpSocketFromClient.getLocalAddress() 67 | + ":" + tcpSocketFromClient.getLocalPort() 68 | ); 69 | } 70 | 71 | // Send a message to make sure all the connections still work 72 | // by send and then receiving something. 73 | ByteBuffer buffer = ByteBuffer.allocate(4); 74 | for (int i = 0; i < numberOfConnections; i++) { 75 | buffer.putInt(i); 76 | tcpSocketsFromClient.get(i).getOutputStream().write(buffer.array()); 77 | buffer.clear(); 78 | } 79 | 80 | for (int i = 0; i < numberOfConnections; i++) { 81 | Socket tcpSocketFromClient = tcpSocketsFromClient.get(i); 82 | tcpSocketFromClient.getInputStream().read(buffer.array()); 83 | int msgFromClient = buffer.getInt(); 84 | if (i != msgFromClient) { 85 | print("ERROR: got " + msgFromClient + " but expected i"); 86 | } 87 | print( 88 | "SERVER: " + i 89 | + " from " + tcpSocketFromClient.getInetAddress() 90 | + ":" + tcpSocketFromClient.getPort() 91 | + " to " + tcpSocketFromClient.getLocalAddress() 92 | + ":" + tcpSocketFromClient.getLocalPort() 93 | + " msg " + msgFromClient 94 | ); 95 | buffer.clear(); 96 | } 97 | 98 | } catch (IOException e) { 99 | e.printStackTrace(); 100 | } finally { 101 | for (Socket socket : tcpSocketsFromClient) { 102 | try { 103 | socket.close(); 104 | } catch (IOException e) { 105 | e.printStackTrace(); 106 | } 107 | } 108 | } 109 | }); 110 | serverThread.start(); 111 | 112 | // once the server is accepting connections clients can start connecting. 113 | waitForServerStartup.acquire(); 114 | 115 | List tcpConnectionsToServer = Collections.synchronizedList(new ArrayList<>(numberOfConnections)); 116 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 117 | print("Shutting down sockets to server"); 118 | try { 119 | for(Socket socket : tcpConnectionsToServer) { 120 | socket.close();; 121 | } 122 | } catch (IOException e) { 123 | 124 | } 125 | })); 126 | 127 | try { 128 | // establish all connections 129 | int idx = 0; 130 | int address = 0; 131 | label: 132 | while (true) { 133 | for (int j = 0; j < connectionsPerAddress; j++) { 134 | Socket tcpConnectionToServer = new Socket( 135 | InetAddress.getByName("127.0.0.1"), 136 | SERVER_PORT, 137 | InetAddress.getByName("10.0.0." + address), 138 | 0 // next available port 139 | ); 140 | tcpConnectionsToServer.add(tcpConnectionToServer); 141 | idx++; 142 | if (idx >= numberOfConnections) { 143 | break label; 144 | } 145 | } 146 | address++; 147 | } 148 | 149 | // Receive a message to make sure all the connections still work 150 | // by send and then receiving something. 151 | ByteBuffer buffer = ByteBuffer.allocate(4); 152 | for (int i = 0; i < numberOfConnections; i++) { 153 | Socket tcpConnectionToServer = tcpConnectionsToServer.get(i); 154 | tcpConnectionToServer.getInputStream().read(buffer.array()); 155 | int msgFromServer = buffer.getInt(); 156 | if (i != msgFromServer) { 157 | print("ERROR: got " + msgFromServer + " but expected i"); 158 | } 159 | print( 160 | "CLIENT: " + i 161 | + " from " + tcpConnectionToServer.getInetAddress() 162 | + ":" + tcpConnectionToServer.getPort() 163 | + " to " + tcpConnectionToServer.getLocalAddress() 164 | + ":" + tcpConnectionToServer.getLocalPort() 165 | + " msg " + msgFromServer 166 | ); 167 | buffer.clear(); 168 | } 169 | 170 | // Send a message to make sure all the connections still work 171 | // by send and then receiving something. 172 | for (int i = 0; i < numberOfConnections; i++) { 173 | buffer.putInt(i); 174 | tcpConnectionsToServer.get(i).getOutputStream().write(buffer.array()); 175 | buffer.clear(); 176 | } 177 | 178 | } finally { 179 | for (Socket socket : tcpConnectionsToServer) { 180 | try { 181 | socket.close(); 182 | } catch (IOException e) { 183 | e.printStackTrace(); 184 | } 185 | } 186 | } 187 | 188 | 189 | serverThread.join(); 190 | } 191 | 192 | } 193 | 194 | -------------------------------------------------------------------------------- /millis_nanos_repeat_or_decrease/Main.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/millis_nanos_repeat_or_decrease/Main.class -------------------------------------------------------------------------------- /millis_nanos_repeat_or_decrease/Main.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | 3 | public final class Main { 4 | 5 | public static void main(String [] args) throws Exception { 6 | Thread currentTimeMillisRepeatThread = new Thread(Main::doesCurrentTimeMillisRepeat); 7 | Thread nanoTimeRepeatThread = new Thread(Main::doesNanoTimeRepeat); 8 | 9 | currentTimeMillisRepeatThread.start(); 10 | nanoTimeRepeatThread.start(); 11 | currentTimeMillisRepeatThread.join(); 12 | nanoTimeRepeatThread.join(); 13 | System.out.println("All threads complete"); 14 | } 15 | 16 | private static void doesCurrentTimeMillisRepeat() { 17 | long prev = System.currentTimeMillis(); 18 | while (true) { 19 | final long curr = System.currentTimeMillis(); 20 | if (prev == curr) { 21 | System.out.println("System.currentTimeMillis() repeated prev=" + prev + ", curr=" + curr); 22 | break; 23 | } 24 | prev = curr; 25 | } 26 | } 27 | 28 | private static void doesCurrentTimeMillisDecrease() { 29 | long prev = System.currentTimeMillis(); 30 | while (true) { 31 | final long curr = System.currentTimeMillis(); 32 | if (prev > curr) { 33 | System.out.println("System.currentTimeMillis() decreased prev=" + prev + ", curr=" + curr); 34 | break; 35 | } 36 | prev = curr; 37 | } 38 | } 39 | 40 | private static void doesNanoTimeRepeat() { 41 | long prev = System.nanoTime(); 42 | while (true) { 43 | final long curr = System.nanoTime(); 44 | if (prev == curr) { 45 | System.out.println("System.nanoTime() repeated prev=" + prev + ", curr=" + curr); 46 | break; 47 | } 48 | prev = curr; 49 | } 50 | } 51 | 52 | private static void doesNanoTimeDecrease() { 53 | long prev = System.nanoTime(); 54 | while (true) { 55 | final long curr = System.nanoTime(); 56 | if (prev > curr) { 57 | System.out.println("System.nanoTime() decreased prev=" + prev + ", curr=" + curr); 58 | break; 59 | } 60 | prev = curr; 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /millis_nanos_repeat_or_decrease/README.md: -------------------------------------------------------------------------------- 1 | Do `System.currentTimeMillis()` or `System.nanoTime()` ever decrease between 2 | successive calls? Do they every repeat? 3 | 4 | This question is important because I have maintained code where the author 5 | expected that `currentTimeMillis()` always created a new value. The code would 6 | fail if there was ever a repeat. 7 | 8 | 1. Think about the answer. 9 | 2. Try to write some experimental code to determine the answer. 10 | 3. Read through resources online. 11 | 12 |
13 | Click to reveal the solution by experiment. 14 | My experiment creates 4 threads that run forever. Each thread compares the 15 | current value to the previous, prints, and stops if a violation is found. 16 | 17 | ``` 18 | long prev = System.currentTimeMillis(); 19 | while (true) { 20 | final long curr = System.currentTimeMillis(); 21 | if (prev == curr) { 22 | System.out.println("System.currentTimeMillis() repeated prev=" + prev + ", curr=" + curr); 23 | break; 24 | } 25 | prev = curr; 26 | } 27 | ``` 28 | Only currentTimeMillis() repeated. nanoTime() did not. currentTimeMillis and 29 | nanoTime never decreased. 30 | 31 | This results make sense since between successive currentTimeMillis() calls could 32 | be less than 1 millisecond. However, it would be more difficult depending on the 33 | accuracy of nanoTime() implemented on your system. 34 | 35 | However, in this case experimentation is not 36 | sufficient. However, what happens if you set the system time to something in the 37 | past? 38 | 39 | Using `sudo date -u 0829120021` I was able to do that, and the values returned 40 | never decreased. However, online sources claim otherwise. 41 |
42 | 43 |
44 | Click to reveal the solution by reading sources online. 45 | Depending on what version of the JDK you use, you might see the time decrease 46 | when the system time changes! 47 | 48 | https://stackoverflow.com/questions/2978598/will-system-currenttimemillis-always-return-a-value-previous-calls 49 | 50 | https://bugs.java.com/bugdatabase/view_bug.do?bug_id=6458294 51 |
52 | 53 |
54 | Click to reveal the overall lesson of this exercise. 55 | NEVER use currentTimeMillis() nor nanoTime() to create a unique identifier. 56 |
57 | -------------------------------------------------------------------------------- /millis_nanos_repeat_or_decrease/run.sh: -------------------------------------------------------------------------------- 1 | javac Main.java \ 2 | && java Main 3 | 4 | # UTC 5 | # sudo date -u {month}{day}{hour}{minute}{year} 6 | # sudo date -u 0829120021 7 | 8 | -------------------------------------------------------------------------------- /native_memory_impact/Main.java: -------------------------------------------------------------------------------- 1 | import java.io.IOException; 2 | import java.nio.ByteBuffer; 3 | import java.nio.file.Files; 4 | import java.nio.file.Paths; 5 | 6 | public final class Main { 7 | 8 | public static void main(String [] args) throws IOException { 9 | // add shutdown hook to clean up file on exit 10 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 11 | try { 12 | Files.delete(Paths.get("ready.tmp")); 13 | } catch (IOException e) { 14 | e.printStackTrace(); 15 | } 16 | })); 17 | 18 | // Allocate 1GB of native memory (method accepts bytes) 19 | // KB MB GB 20 | ByteBuffer nativeMemoryBuffer = ByteBuffer.allocateDirect(1024*1024*1024); 21 | while(true) { 22 | Files.write(Paths.get("ready.tmp"), "I'm ready!".getBytes()); 23 | // sleep until we're interrupted 24 | try { 25 | Thread.sleep(1000); // sleep for one second 26 | } catch (InterruptedException e) { 27 | break; 28 | } 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /native_memory_impact/README.md: -------------------------------------------------------------------------------- 1 | How does native memory impact heap? OS memory? 2 | 3 | 4 | 1. Think about the answer. 5 | 2. Try to write some experimental code to determine the answer. 6 | 7 |
8 | Click to reveal the result from my experiment. 9 | 10 | To use native memory, I used the direct byte buffer. 11 | I allocated 1 gigabyte. 12 | I used `jcmd` `VM.native_memory` and `GC.heap_info` with `-XX:NativeMemoryTracking=summary` to get memory stats from the JVM. 13 | Here are the results: 14 | 15 | ``` 16 | Native Memory Tracking: 17 | 18 | Total: reserved=6787775KB, committed=1417475KB 19 | - Java Heap (reserved=4194304KB, committed=266240KB) 20 | (mmap: reserved=4194304KB, committed=266240KB) 21 | 22 | ... 23 | 24 | - Other (reserved=1048578KB, committed=1048578KB) 25 | (malloc=1048578KB #3) 26 | ... 27 | 28 | 54416: 29 | garbage-first heap total 266240K, used 3375K [0x0000000700000000, 0x0000000800000000) 30 | region size 2048K, 1 young (2048K), 0 survivors (0K) 31 | Metaspace used 348K, committed 512K, reserved 1056768K 32 | class space used 25K, committed 128K, reserved 1048576K 33 | ``` 34 | 35 | The heap has only committed 260MB while the Other category has committed just over 1024MB. 36 | That's the direct byte buffer! 37 | From this you can see that native is not allocated on the heap. 38 | So make sure to always monitor both the OS memory usage and the heap usage. 39 | If you notice heap is fine, but the OS memory is not, then you've got a native memory issue! 40 | 41 |
42 | 43 | -------------------------------------------------------------------------------- /native_memory_impact/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | javac Main.java 4 | retVal=$? 5 | if [ $retVal -ne 0 ]; then 6 | echo "compile error" 7 | exit $retVal 8 | fi 9 | 10 | 11 | java -XX:NativeMemoryTracking=summary Main & 12 | process_pid=$! 13 | echo process_pid=$process_pid 14 | # wait for temp file 15 | while [ ! -f "ready.tmp" ] 16 | do 17 | sleep 1 18 | done 19 | 20 | line=$(cat ready.tmp) 21 | while [ "I'm ready!" != "$line" ] 22 | do 23 | sleep 1 24 | done 25 | 26 | jcmd $process_pid VM.native_memory 27 | jcmd $process_pid GC.heap_info 28 | 29 | kill $process_pid 30 | 31 | -------------------------------------------------------------------------------- /prepared_statement_limits/README.md: -------------------------------------------------------------------------------- 1 | Where can you use `?` in a PreparedStatement? 2 | 3 | Consider these queries: 4 | 1. `CREATE DATABASE ?` 5 | 1. `DROP DATABASE ?` 6 | 1. `CREATE USER ? IDENTIFIED BY ?` 7 | 1. `DROP USER ?` 8 | 1. `CREATE TABLE ? (sample_str VARCHAR(256), sample_int INT)` 9 | 1. `CREATE TABLE sample_table (? VARCHAR(256), sample_int INT)` 10 | 1. `DROP TABLE ?` 11 | 1. `INSERT INTO ? (sample_str, sample_int) values ('sample_value', 1)` 12 | 1. `INSERT INTO sample_table (?, sample_int) values ('sample_value', 1)` 13 | 1. `INSERT INTO sample_table (sample_str, sample_int) values (?, 1)` 14 | 1. `DELETE FROM ?` 15 | 1. `DELETE FROM sample_table WHERE ?='sample_value'` 16 | 1. `DELETE FROM sample_table WHERE sample_str=?` 17 | 1. `SELECT * FROM ?` 18 | 1. `SELECT ? FROM sample_table` 19 | 1. `SELECT * FROM sample_table WHERE ?='sample_value'` 20 | 1. `SELECT * FROM sample_table WHERE sample_str=?` 21 | 22 | 23 |
24 | Click to reveal the solution by experiment. 25 | You cannot use ? as a reference (like a table name or column name). 26 | 27 | 28 | Use can use ? where ever a value makes sense, even in `CREATE USER ?` 29 | and `DROP USER ?`, since the username is a value. 30 | ``` 31 | ✖ CREATE DATABASE ? 32 | ✖ DROP DATABASE ? 33 | ✔ CREATE USER ? IDENTIFIED BY ? 34 | ✔ DROP USER ? 35 | ✖ CREATE TABLE ? (sample_str VARCHAR(256), sample_int INT) 36 | ✖ CREATE TABLE sample_table (? VARCHAR(256), sample_int INT) 37 | ✖ DROP TABLE ? 38 | ✖ INSERT INTO ? (sample_str, sample_int) values ('sample_value', 1) 39 | ✖ INSERT INTO sample_table (?, sample_int) values ('sample_value', 1) 40 | ✔ INSERT INTO sample_table (sample_str, sample_int) values (?, 1) 41 | ✖ DELETE FROM ? 42 | ✖ DELETE FROM sample_table WHERE ?='sample_value' 43 | ✖ DELETE FROM sample_table WHERE sample_str=? 44 | ✖ SELECT * FROM ? 45 | ✖ SELECT ? FROM sample_table 46 | ✖ SELECT * FROM sample_table WHERE ?='sample_value' 47 | ✔ SELECT * FROM sample_table WHERE sample_str=? 48 | ``` 49 |
50 | -------------------------------------------------------------------------------- /prepared_statement_limits/compile.txt: -------------------------------------------------------------------------------- 1 | [INFO] Scanning for projects... 2 | [INFO] 3 | [INFO] ---------< org.java.by.experiments:prepared.statements.limits >--------- 4 | [INFO] Building prepared.statements.limits 1.0-SNAPSHOT 5 | [INFO] --------------------------------[ jar ]--------------------------------- 6 | [INFO] 7 | [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ prepared.statements.limits --- 8 | [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! 9 | [INFO] skip non existing resourceDirectory /Users/joseph/projects/java-by-experiments/prepared_statement_limits/src/main/resources 10 | [INFO] 11 | [INFO] --- maven-compiler-plugin:3.1:compile (default-compile) @ prepared.statements.limits --- 12 | [INFO] Nothing to compile - all classes are up to date 13 | [INFO] 14 | [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ prepared.statements.limits --- 15 | [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! 16 | [INFO] skip non existing resourceDirectory /Users/joseph/projects/java-by-experiments/prepared_statement_limits/src/test/resources 17 | [INFO] 18 | [INFO] --- maven-compiler-plugin:3.1:testCompile (default-testCompile) @ prepared.statements.limits --- 19 | [INFO] No sources to compile 20 | [INFO] 21 | [INFO] --- maven-surefire-plugin:2.12.4:test (default-test) @ prepared.statements.limits --- 22 | [INFO] No tests to run. 23 | [INFO] 24 | [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ prepared.statements.limits --- 25 | [INFO] 26 | [INFO] --- maven-install-plugin:2.4:install (default-install) @ prepared.statements.limits --- 27 | [INFO] Installing /Users/joseph/projects/java-by-experiments/prepared_statement_limits/target/prepared.statements.limits-1.0-SNAPSHOT.jar to /Users/joseph/.m2/repository/org/java/by/experiments/prepared.statements.limits/1.0-SNAPSHOT/prepared.statements.limits-1.0-SNAPSHOT.jar 28 | [INFO] Installing /Users/joseph/projects/java-by-experiments/prepared_statement_limits/pom.xml to /Users/joseph/.m2/repository/org/java/by/experiments/prepared.statements.limits/1.0-SNAPSHOT/prepared.statements.limits-1.0-SNAPSHOT.pom 29 | [INFO] ------------------------------------------------------------------------ 30 | [INFO] BUILD SUCCESS 31 | [INFO] ------------------------------------------------------------------------ 32 | [INFO] Total time: 0.928 s 33 | [INFO] Finished at: 2021-08-29T19:10:53-04:00 34 | [INFO] ------------------------------------------------------------------------ 35 | -------------------------------------------------------------------------------- /prepared_statement_limits/err.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/josephmate/java-by-experiments/67a7c98bf3e318482daa0e09d07939d7587cfdf5/prepared_statement_limits/err.txt -------------------------------------------------------------------------------- /prepared_statement_limits/out.txt: -------------------------------------------------------------------------------- 1 | [INFO] Scanning for projects... 2 | [INFO] 3 | [INFO] ---------< org.java.by.experiments:prepared.statements.limits >--------- 4 | [INFO] Building prepared.statements.limits 1.0-SNAPSHOT 5 | [INFO] --------------------------------[ jar ]--------------------------------- 6 | [INFO] 7 | [INFO] --- exec-maven-plugin:3.0.0:java (default-cli) @ prepared.statements.limits --- 8 | Success? Query 9 | ✖ CREATE DATABASE ? 10 | ✖ DROP DATABASE ? 11 | ✔ CREATE USER ? IDENTIFIED BY ? 12 | ✔ DROP USER ? 13 | ✖ CREATE TABLE ? (sample_str VARCHAR(256), sample_int INT) 14 | ✖ CREATE TABLE sample_table (? VARCHAR(256), sample_int INT) 15 | ✖ DROP TABLE ? 16 | ✖ INSERT INTO ? (sample_str, sample_int) values ('sample_value', 1) 17 | ✖ INSERT INTO sample_table (?, sample_int) values ('sample_value', 1) 18 | ✔ INSERT INTO sample_table (sample_str, sample_int) values (?, 1) 19 | ✖ DELETE FROM ? 20 | ✖ DELETE FROM sample_table WHERE ?='sample_value' 21 | ✖ DELETE FROM sample_table WHERE sample_str=? 22 | ✖ SELECT * FROM ? 23 | ✖ SELECT ? FROM sample_table 24 | ✖ SELECT * FROM sample_table WHERE ?='sample_value' 25 | ✔ SELECT * FROM sample_table WHERE sample_str=? 26 | [INFO] ------------------------------------------------------------------------ 27 | [INFO] BUILD SUCCESS 28 | [INFO] ------------------------------------------------------------------------ 29 | [INFO] Total time: 3.528 s 30 | [INFO] Finished at: 2021-08-29T19:10:58-04:00 31 | [INFO] ------------------------------------------------------------------------ 32 | -------------------------------------------------------------------------------- /prepared_statement_limits/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | org.java.by.experiments 6 | prepared.statements.limits 7 | 1.0-SNAPSHOT 8 | 9 | 10 | 1.8 11 | 1.8 12 | 13 | 14 | 15 | 16 | org.mariadb.jdbc 17 | mariadb-java-client 18 | 2.7.4 19 | 20 | 21 | org.postgresql 22 | postgresql 23 | 42.2.23 24 | 25 | 26 | org.apache.derby 27 | derby 28 | 10.15.2.0 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /prepared_statement_limits/run.sh: -------------------------------------------------------------------------------- 1 | mvn --batch-mode install > compile.txt 2>&1 2 | 3 | mvn --batch-mode exec:java \ 4 | -Dexec.mainClass="Main" \ 5 | -Dexec.args="$1 $2 $3 $4 $5" \ 6 | 2> err.txt \ 7 | | tee out.txt 8 | -------------------------------------------------------------------------------- /prepared_statement_limits/src/main/java/Main.java: -------------------------------------------------------------------------------- 1 | import java.sql.Connection; 2 | import java.sql.DriverManager; 3 | import java.sql.PreparedStatement; 4 | import java.sql.ResultSet; 5 | import java.sql.Statement; 6 | import java.util.Collections; 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | public final class Main { 11 | 12 | public static void main(String [] args) throws Exception { 13 | if (args.length != 5) { 14 | System.err.println("Main [mysql|postgres|derby] "); 15 | System.exit(-1); 16 | } 17 | int i = 0; 18 | final String dbType = args[i++]; 19 | final String hostname = args[i++]; 20 | final String port = args[i++]; 21 | final String username = args[i++]; 22 | final String password = args[i++]; 23 | final String url = "jdbc:" + dbType + "://" + hostname + ":" + port + "/"; 24 | 25 | try(Connection conn = DriverManager.getConnection(url, username, password)) { 26 | Main main = new Main(conn); 27 | main.runExperiments(); 28 | } 29 | } 30 | 31 | private final Connection conn; 32 | 33 | private Main(Connection conn) { 34 | this.conn = conn; 35 | } 36 | 37 | private void runExperiments() throws Exception { 38 | System.out.println("Success? Query"); 39 | 40 | runExperiment( 41 | false, 42 | Collections.emptyList(), 43 | "CREATE DATABASE ?", 44 | (pstmt) -> { 45 | pstmt.setString(1, "sample_db"); 46 | pstmt.executeUpdate(); 47 | }, 48 | Collections.emptyList() 49 | ); 50 | 51 | runExperiment( 52 | "DROP DATABASE ?", 53 | (pstmt) -> { 54 | pstmt.setString(1, "sample_db"); 55 | pstmt.executeUpdate(); 56 | } 57 | ); 58 | 59 | runExperiment( 60 | "CREATE USER ? IDENTIFIED BY ?", 61 | (pstmt) -> { 62 | pstmt.setString(1, "sample_user"); 63 | pstmt.setString(2, "insecure password"); 64 | pstmt.executeUpdate(); 65 | }, 66 | "DROP USER sample_user" 67 | ); 68 | 69 | runExperiment( 70 | "CREATE USER sample_user IDENTIFIED BY 'insecure password'", 71 | "DROP USER ?", 72 | (pstmt) -> { 73 | pstmt.setString(1, "sample_user"); 74 | pstmt.executeUpdate(); 75 | } 76 | ); 77 | 78 | runExperiment( 79 | "CREATE TABLE ? (sample_str VARCHAR(256), sample_int INT)", 80 | (pstmt) -> { 81 | pstmt.setString(1, "sample_table"); 82 | pstmt.executeUpdate(); 83 | } 84 | ); 85 | 86 | runExperiment( 87 | "CREATE TABLE sample_table (? VARCHAR(256), sample_int INT)", 88 | (pstmt) -> { 89 | pstmt.setString(1, "sample_str"); 90 | pstmt.executeUpdate(); 91 | } 92 | ); 93 | 94 | runExperiment( 95 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 96 | "DROP TABLE ?", 97 | (pstmt) -> { 98 | pstmt.setString(1, "sample_table"); 99 | pstmt.executeUpdate(); 100 | } 101 | ); 102 | 103 | runExperiment( 104 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 105 | "INSERT INTO ? (sample_str, sample_int) values ('sample_value', 1)", 106 | (pstmt) -> { 107 | pstmt.setString(1, "sample_table"); 108 | pstmt.executeUpdate(); 109 | } 110 | ); 111 | 112 | runExperiment( 113 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 114 | "INSERT INTO sample_table (?, sample_int) values ('sample_value', 1)", 115 | (pstmt) -> { 116 | pstmt.setString(1, "sample_str"); 117 | pstmt.executeUpdate(); 118 | } 119 | ); 120 | 121 | runExperiment( 122 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 123 | "INSERT INTO sample_table (sample_str, sample_int) values (?, 1)", 124 | (pstmt) -> { 125 | pstmt.setString(1, "sample_value"); 126 | pstmt.executeUpdate(); 127 | } 128 | ); 129 | 130 | runExperiment( 131 | Arrays.asList( 132 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 133 | "INSERT INTO sample_table (sample_str, sample_int) values ('sample_value', 1)" 134 | ), 135 | "DELETE FROM ?", 136 | (pstmt) -> { 137 | pstmt.setString(1, "sample_table"); 138 | pstmt.executeUpdate(); 139 | } 140 | ); 141 | 142 | runExperiment( 143 | Arrays.asList( 144 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 145 | "INSERT INTO sample_table (sample_str, sample_int) values ('sample_value', 1)" 146 | ), 147 | "DELETE FROM sample_table WHERE ?='sample_value'", 148 | (pstmt) -> { 149 | pstmt.setString(1, "sample_str"); 150 | pstmt.executeUpdate(); 151 | }, 152 | () -> { 153 | try (Statement stmt = conn.createStatement()) { 154 | try (ResultSet resultSet = stmt.executeQuery("SELECT count(*) FROM sample_table")) { 155 | if (!resultSet.next()) { 156 | return false; 157 | } 158 | return resultSet.getInt(0) == 0; 159 | } 160 | } 161 | } 162 | ); 163 | 164 | runExperiment( 165 | Arrays.asList( 166 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 167 | "INSERT INTO sample_table (sample_str, sample_int) values ('sample_value', 1)" 168 | ), 169 | "DELETE FROM sample_table WHERE sample_str=?", 170 | (pstmt) -> { 171 | pstmt.setString(1, "sample_value"); 172 | pstmt.executeUpdate(); 173 | }, 174 | () -> { 175 | try (Statement stmt = conn.createStatement()) { 176 | try (ResultSet resultSet = stmt.executeQuery("SELECT count(*) FROM sample_table")) { 177 | if (!resultSet.next()) { 178 | return false; 179 | } 180 | return resultSet.getInt(0) == 0; 181 | } 182 | } 183 | } 184 | ); 185 | 186 | runExperiment( 187 | Arrays.asList( 188 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 189 | "INSERT INTO sample_table (sample_str, sample_int) values ('sample_value', 1)" 190 | ), 191 | "SELECT * FROM ?", 192 | (pstmt) -> { 193 | pstmt.setString(1, "sample_table"); 194 | pstmt.executeQuery(); 195 | } 196 | ); 197 | 198 | runExperiment( 199 | Arrays.asList( 200 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 201 | "INSERT INTO sample_table (sample_str, sample_int) values ('sample_value', 1)" 202 | ), 203 | "SELECT ? FROM sample_table", 204 | (pstmt) -> { 205 | pstmt.setString(1, "sample_str"); 206 | try (ResultSet resultSet = pstmt.executeQuery()) { 207 | if (!resultSet.next()) { 208 | throw new IllegalStateException("Was expecting a single row"); 209 | } 210 | if (!"sample_value".equals(resultSet.getString(1))) { 211 | throw new IllegalStateException("Was expecting a single row with 1st column equal to sample_value"); 212 | } 213 | } 214 | } 215 | ); 216 | 217 | runExperiment( 218 | Arrays.asList( 219 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 220 | "INSERT INTO sample_table (sample_str, sample_int) values ('sample_value', 1)" 221 | ), 222 | "SELECT * FROM sample_table WHERE ?='sample_value'", 223 | (pstmt) -> { 224 | pstmt.setString(1, "sample_str"); 225 | try (ResultSet resultSet = pstmt.executeQuery()) { 226 | if (!resultSet.next()) { 227 | throw new IllegalStateException("Was expecting a single row"); 228 | } 229 | if (!"sample_value".equals(resultSet.getString(1))) { 230 | throw new IllegalStateException("Was expecting a single row with 1st column equal to sample_value"); 231 | } 232 | if (resultSet.getInt(2) != 1) { 233 | throw new IllegalStateException("Was expecting a single row with 2nd column equal to 1"); 234 | } 235 | } 236 | } 237 | ); 238 | 239 | runExperiment( 240 | Arrays.asList( 241 | "CREATE TABLE sample_table (sample_str VARCHAR(256), sample_int INT)", 242 | "INSERT INTO sample_table (sample_str, sample_int) values ('sample_value', 1)" 243 | ), 244 | "SELECT * FROM sample_table WHERE sample_str=?", 245 | (pstmt) -> { 246 | pstmt.setString(1, "sample_value"); 247 | try (ResultSet resultSet = pstmt.executeQuery()) { 248 | if (!resultSet.next()) { 249 | throw new IllegalStateException("Was expecting a single row"); 250 | } 251 | if (!"sample_value".equals(resultSet.getString(1))) { 252 | throw new IllegalStateException("Was expecting a single row with 1st column equal to sample_value"); 253 | } 254 | if (resultSet.getInt(2) != 1) { 255 | throw new IllegalStateException("Was expecting a single row with 2nd column equal to 1"); 256 | } 257 | } 258 | } 259 | ); 260 | } 261 | 262 | @FunctionalInterface 263 | public interface CheckedConsumer { 264 | void apply(T t) throws Exception; 265 | } 266 | 267 | @FunctionalInterface 268 | public interface CheckedProducer { 269 | T apply() throws Exception; 270 | } 271 | 272 | private void runExperiment( 273 | String preparedSqlString, 274 | CheckedConsumer experiment 275 | ) throws Exception { 276 | runExperiment( 277 | true, 278 | Collections.emptyList(), 279 | preparedSqlString, 280 | experiment, 281 | () -> true, 282 | Collections.emptyList() 283 | ); 284 | } 285 | 286 | private void runExperiment( 287 | String preReq, 288 | String preparedSqlString, 289 | CheckedConsumer experiment 290 | ) throws Exception { 291 | runExperiment( 292 | true, 293 | Arrays.asList(preReq), 294 | preparedSqlString, 295 | experiment, 296 | () -> true, 297 | Collections.emptyList() 298 | ); 299 | } 300 | 301 | private void runExperiment( 302 | String preparedSqlString, 303 | CheckedConsumer experiment, 304 | String customCleanup 305 | ) throws Exception { 306 | runExperiment( 307 | true, 308 | Collections.emptyList(), 309 | preparedSqlString, 310 | experiment, 311 | () -> true, 312 | Arrays.asList(customCleanup) 313 | ); 314 | } 315 | 316 | private void runExperiment( 317 | String preReq, 318 | String preparedSqlString, 319 | CheckedConsumer experiment, 320 | String customCleanup 321 | ) throws Exception { 322 | runExperiment( 323 | true, 324 | Arrays.asList(preReq), 325 | preparedSqlString, 326 | experiment, 327 | () -> true, 328 | Arrays.asList(customCleanup) 329 | ); 330 | } 331 | 332 | private void runExperiment( 333 | List preReqs, 334 | String preparedSqlString, 335 | CheckedConsumer experiment 336 | ) throws Exception { 337 | runExperiment( 338 | true, 339 | preReqs, 340 | preparedSqlString, 341 | experiment, 342 | () -> true, 343 | Collections.emptyList() 344 | ); 345 | } 346 | 347 | private void runExperiment( 348 | List preReqs, 349 | String preparedSqlString, 350 | CheckedConsumer experiment, 351 | CheckedProducer verification 352 | ) throws Exception { 353 | runExperiment( 354 | true, 355 | preReqs, 356 | preparedSqlString, 357 | experiment, 358 | verification, 359 | Collections.emptyList() 360 | ); 361 | } 362 | 363 | private void runExperiment( 364 | boolean createSampleDB, 365 | List preReqs, 366 | String preparedSqlString, 367 | CheckedConsumer experiment, 368 | List customCleanup 369 | ) throws Exception { 370 | runExperiment( 371 | createSampleDB, 372 | preReqs, 373 | preparedSqlString, 374 | experiment, 375 | () -> true, 376 | customCleanup 377 | ); 378 | } 379 | 380 | private void runExperiment( 381 | boolean createSampleDB, 382 | List preReqs, 383 | String preparedSqlString, 384 | CheckedConsumer experiment, 385 | CheckedProducer verification, 386 | List customCleanup 387 | ) throws Exception { 388 | if (createSampleDB) { 389 | try (Statement statement = conn.createStatement()) { 390 | statement.executeUpdate("CREATE DATABASE sample_db"); 391 | } 392 | try (Statement statement = conn.createStatement()) { 393 | statement.executeUpdate("USE sample_db"); 394 | } 395 | } 396 | 397 | for (String preReqSql : preReqs) { 398 | try (Statement statement = conn.createStatement()) { 399 | statement.executeUpdate(preReqSql); 400 | } 401 | } 402 | 403 | try (PreparedStatement pstmt = conn.prepareStatement(preparedSqlString)) { 404 | experiment.apply(pstmt); 405 | if (verification.apply()) { 406 | System.out.print("✔"); 407 | } else { 408 | System.out.print("✖"); 409 | } 410 | } catch (Exception e) { 411 | System.out.print("✖"); 412 | } 413 | System.out.println(" " + preparedSqlString); 414 | 415 | if (createSampleDB) { 416 | try (Statement statement = conn.createStatement()) { 417 | statement.executeUpdate("DROP DATABASE sample_db"); 418 | } 419 | } 420 | 421 | for (String cleanupSql : customCleanup) { 422 | try (Statement statement = conn.createStatement()) { 423 | statement.executeUpdate(cleanupSql); 424 | } 425 | } 426 | } 427 | 428 | } 429 | -------------------------------------------------------------------------------- /read_write_lock_promotion/Main.java: -------------------------------------------------------------------------------- 1 | import java.util.concurrent.locks.ReentrantReadWriteLock; 2 | 3 | public final class Main { 4 | 5 | public static void main(String [] args) throws Exception { 6 | if (args.length != 1) { 7 | throw new IllegalArgumentException("Program expects one argument," 8 | + " the sequence or R's and W's" 9 | + " instructing the order in which to acquire the locks"); 10 | } 11 | 12 | System.out.println("Program starting"); 13 | ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); 14 | 15 | for (char c : args[0].toCharArray()) { 16 | if (c == 'r' || c == 'R') { 17 | System.out.println("Acquiring the read lock"); 18 | readWriteLock.readLock().lock(); 19 | System.out.println("Got the read lock"); 20 | } else if (c == 'w' || c == 'W') { 21 | System.out.println("Acquiring the write lock"); 22 | readWriteLock.writeLock().lock(); 23 | System.out.println("Got the write lock"); 24 | } else { 25 | throw new IllegalArgumentException("Character " + c + " is not a recognized action."); 26 | } 27 | } 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /read_write_lock_promotion/README.md: -------------------------------------------------------------------------------- 1 | Can a thread promote their `ReentrantReadWriteLock.readLock()` to a `ReentrantReadWriteLock.writeLock()`? 2 | 3 | 1. Think about the answer. 4 | 2. Try to write some experimental code to determine the answer. 5 | 3. Read through resources online. 6 | 7 |
8 | Click to reveal the solution by experiment. 9 | You cannot promote an acquired read lock to a write lock. 10 | You must release the read lock then acquire the write lock. 11 | 12 | ``` 13 | ./run.sh RRW 14 | Program starting 15 | Acquiring the read lock 16 | Got the read lock 17 | Acquiring the read lock 18 | Got the read lock 19 | Acquiring the write lock 20 | ``` 21 | 22 | However, you can acquire the write lock, then the read lock: 23 | ``` 24 | ./run.sh WR 25 | Program starting 26 | Acquiring the write lock 27 | Got the write lock 28 | Acquiring the read lock 29 | Got the read lock 30 | ``` 31 |
32 | 33 |
34 | Click to reveal the solution by reading sources online. 35 | 36 | Read through 37 | 38 | the ReentrantReadWriteLock javadoc. 39 | 40 | 41 | > Additionally, a writer can acquire the read lock, but not vice-versa. Among other applications, reentrancy can be useful when write locks are held during calls or callbacks to methods that perform reads under read locks. If a reader tries to acquire the write lock it will never succeed. 42 | 43 | > Reentrancy also allows downgrading from the write lock to a read lock, by acquiring the write lock, then the read lock and then releasing the write lock. However, upgrading from a read lock to the write lock is not possible. 44 | 45 | You cannot acquire the read lock then the write lock. 46 |
47 | 48 | -------------------------------------------------------------------------------- /read_write_lock_promotion/run.sh: -------------------------------------------------------------------------------- 1 | javac Main.java \ 2 | && java Main "$@" 3 | 4 | -------------------------------------------------------------------------------- /resource_leaks/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /resource_leaks/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /resource_leaks/.idea/jarRepositories.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | 14 | 15 | 19 | 20 | -------------------------------------------------------------------------------- /resource_leaks/.idea/libraries/Maven__args4j_args4j_2_33.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /resource_leaks/.idea/libraries/Maven__org_mariadb_jdbc_mariadb_java_client_2_7_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /resource_leaks/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /resource_leaks/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /resource_leaks/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /resource_leaks/README.md: -------------------------------------------------------------------------------- 1 | What happens if you forget to close a resource? 2 | 3 | Resources you can forget to close: 4 | 1. Files.lines 5 | 2. Files.list 6 | 3. java.sql.Statement 7 | 4. java.sql.PreparedStatement 8 | 5. java.sql.Connection 9 | 6. java.sql.ResultSet 10 | 7. Url 11 | 8. Apache closeable http 12 | 9. Funny business with jooq's auto cleanup 13 | 14 | 15 | 16 |
17 | Click to reveal the solution by experiment. 18 | 19 | Files.lines: 20 | ``` 21 | java.nio.file.FileSystemException: pom.xml: Too many open files 22 | at sun.nio.fs.UnixException.translateToIOException (UnixException.java:100) 23 | at sun.nio.fs.UnixException.rethrowAsIOException (UnixException.java:106) 24 | at sun.nio.fs.UnixException.rethrowAsIOException (UnixException.java:111) 25 | at sun.nio.fs.UnixFileSystemProvider.newFileChannel (UnixFileSystemProvider.java:182) 26 | at java.nio.channels.FileChannel.open (FileChannel.java:292) 27 | at java.nio.channels.FileChannel.open (FileChannel.java:345) 28 | at java.nio.file.Files.lines (Files.java:4104) 29 | at java.nio.file.Files.lines (Files.java:4196) 30 | at Main.filesLinesLeak (Main.java:72) 31 | at Main.run (Main.java:63) 32 | at Main.main (Main.java:85) 33 | at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:254) 34 | at java.lang.Thread.run (Thread.java:831) 35 | ``` 36 | 37 | Files.list 38 | ``` 39 | java.nio.file.FileSystemException: .: Too many open files 40 | at sun.nio.fs.UnixException.translateToIOException (UnixException.java:100) 41 | at sun.nio.fs.UnixException.rethrowAsIOException (UnixException.java:106) 42 | at sun.nio.fs.UnixException.rethrowAsIOException (UnixException.java:111) 43 | at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream (UnixFileSystemProvider.java:419) 44 | at java.nio.file.Files.newDirectoryStream (Files.java:476) 45 | at java.nio.file.FileTreeWalker.visit (FileTreeWalker.java:300) 46 | at java.nio.file.FileTreeWalker.walk (FileTreeWalker.java:322) 47 | at java.nio.file.FileTreeIterator. (FileTreeIterator.java:71) 48 | at java.nio.file.Files.walk (Files.java:3891) 49 | at java.nio.file.Files.walk (Files.java:3945) 50 | at Main.filesWalkLeak (Main.java:104) 51 | at Main.run (Main.java:66) 52 | at Main.main (Main.java:117) 53 | at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:254) 54 | at java.lang.Thread.run (Thread.java:831) 55 | ``` 56 | 57 | java.sql.Statement: Didn't see any issue. 58 | 59 | java.sql.PreparedStatement: Didn't see any issue. 60 | 61 | java.sql.Connection: 62 | ``` 63 | java.sql.SQLNonTransientConnectionException: Could not connect to HostAddress{host='localhost', port=3306, type='master'}. Too many connections 64 | at org.mariadb.jdbc.internal.util.exceptions.ExceptionFactory.createException (ExceptionFactory.java:73) 65 | at org.mariadb.jdbc.internal.util.exceptions.ExceptionFactory.create (ExceptionFactory.java:188) 66 | at org.mariadb.jdbc.internal.protocol.AbstractConnectProtocol.connectWithoutProxy (AbstractConnectProtocol.java:1402) 67 | at org.mariadb.jdbc.internal.util.Utils.retrieveProxy (Utils.java:635) 68 | at org.mariadb.jdbc.MariaDbConnection.newConnection (MariaDbConnection.java:150) 69 | at org.mariadb.jdbc.Driver.connect (Driver.java:89) 70 | at java.sql.DriverManager.getConnection (DriverManager.java:677) 71 | at java.sql.DriverManager.getConnection (DriverManager.java:228) 72 | at Main.javaSqlConnectionLeak (Main.java:181) 73 | at Main.run (Main.java:74) 74 | at Main.main (Main.java:194) 75 | at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:254) 76 | at java.lang.Thread.run (Thread.java:831) 77 | Caused by: java.sql.SQLException: Too many connections 78 | at org.mariadb.jdbc.internal.com.read.ReadInitialHandShakePacket. (ReadInitialHandShakePacket.java:92) 79 | at org.mariadb.jdbc.internal.protocol.AbstractConnectProtocol.createConnection (AbstractConnectProtocol.java:527) 80 | at org.mariadb.jdbc.internal.protocol.AbstractConnectProtocol.connectWithoutProxy (AbstractConnectProtocol.java:1389) 81 | at org.mariadb.jdbc.internal.util.Utils.retrieveProxy (Utils.java:635) 82 | at org.mariadb.jdbc.MariaDbConnection.newConnection (MariaDbConnection.java:150) 83 | at org.mariadb.jdbc.Driver.connect (Driver.java:89) 84 | at java.sql.DriverManager.getConnection (DriverManager.java:677) 85 | at java.sql.DriverManager.getConnection (DriverManager.java:228) 86 | at Main.javaSqlConnectionLeak (Main.java:181) 87 | at Main.run (Main.java:74) 88 | at Main.main (Main.java:194) 89 | at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:254) 90 | at java.lang.Thread.run (Thread.java:831) 91 | ``` 92 | 93 | java.sql.ResultSet: Didn't see any issue. 94 | 95 |
96 | -------------------------------------------------------------------------------- /resource_leaks/compile.txt: -------------------------------------------------------------------------------- 1 | [INFO] Scanning for projects... 2 | [INFO] 3 | [INFO] ---------------< org.java.by.experiments:resource.leaks >--------------- 4 | [INFO] Building resource.leaks 1.0-SNAPSHOT 5 | [INFO] --------------------------------[ jar ]--------------------------------- 6 | [INFO] 7 | [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ resource.leaks --- 8 | [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! 9 | [INFO] skip non existing resourceDirectory /Users/joseph/projects/java-by-experiments/resource_leaks/src/main/resources 10 | [INFO] 11 | [INFO] --- maven-compiler-plugin:3.1:compile (default-compile) @ resource.leaks --- 12 | [INFO] Changes detected - recompiling the module! 13 | [WARNING] File encoding has not been set, using platform encoding UTF-8, i.e. build is platform dependent! 14 | [INFO] Compiling 1 source file to /Users/joseph/projects/java-by-experiments/resource_leaks/target/classes 15 | [INFO] 16 | [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ resource.leaks --- 17 | [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! 18 | [INFO] skip non existing resourceDirectory /Users/joseph/projects/java-by-experiments/resource_leaks/src/test/resources 19 | [INFO] 20 | [INFO] --- maven-compiler-plugin:3.1:testCompile (default-testCompile) @ resource.leaks --- 21 | [INFO] No sources to compile 22 | [INFO] 23 | [INFO] --- maven-surefire-plugin:2.12.4:test (default-test) @ resource.leaks --- 24 | [INFO] No tests to run. 25 | [INFO] 26 | [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ resource.leaks --- 27 | [INFO] Building jar: /Users/joseph/projects/java-by-experiments/resource_leaks/target/resource.leaks-1.0-SNAPSHOT.jar 28 | [INFO] 29 | [INFO] --- maven-install-plugin:2.4:install (default-install) @ resource.leaks --- 30 | [INFO] Installing /Users/joseph/projects/java-by-experiments/resource_leaks/target/resource.leaks-1.0-SNAPSHOT.jar to /Users/joseph/.m2/repository/org/java/by/experiments/resource.leaks/1.0-SNAPSHOT/resource.leaks-1.0-SNAPSHOT.jar 31 | [INFO] Installing /Users/joseph/projects/java-by-experiments/resource_leaks/pom.xml to /Users/joseph/.m2/repository/org/java/by/experiments/resource.leaks/1.0-SNAPSHOT/resource.leaks-1.0-SNAPSHOT.pom 32 | [INFO] ------------------------------------------------------------------------ 33 | [INFO] BUILD SUCCESS 34 | [INFO] ------------------------------------------------------------------------ 35 | [INFO] Total time: 2.366 s 36 | [INFO] Finished at: 2022-01-30T20:57:46-05:00 37 | [INFO] ------------------------------------------------------------------------ 38 | -------------------------------------------------------------------------------- /resource_leaks/err.txt: -------------------------------------------------------------------------------- 1 | Option "--directory" is required 2 | --dbType VAL : Sets type of DB to connect to 3 | (default: mysql) 4 | --directory FILE : the directory to list 5 | --experiment [Files_lines | : the type of experiment 6 | Files_list | java_sql_Statement | 7 | java_sql_PreparedStatement | 8 | java_sql_Connection | java_sql_ResultS 9 | et] 10 | --file FILE : the file to open 11 | --hostname VAL : Sets hostname of the db to connect to 12 | (default: localhost) 13 | --pass VAL : The pass to connect to the db with 14 | (default: root) 15 | --port N : Sets port of the db to connect to 16 | (default: 3306) 17 | --user VAL : The user to connect to the db with 18 | (default: root) 19 | -------------------------------------------------------------------------------- /resource_leaks/out.txt: -------------------------------------------------------------------------------- 1 | [INFO] Scanning for projects... 2 | [INFO] 3 | [INFO] ---------------< org.java.by.experiments:resource.leaks >--------------- 4 | [INFO] Building resource.leaks 1.0-SNAPSHOT 5 | [INFO] --------------------------------[ jar ]--------------------------------- 6 | [INFO] 7 | [INFO] --- exec-maven-plugin:3.0.0:java (default-cli) @ resource.leaks --- 8 | -------------------------------------------------------------------------------- /resource_leaks/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | org.java.by.experiments 6 | resource.leaks 7 | 1.0-SNAPSHOT 8 | 9 | 10 | 1.8 11 | 1.8 12 | 13 | 14 | 15 | 16 | org.mariadb.jdbc 17 | mariadb-java-client 18 | 2.7.4 19 | 20 | 21 | args4j 22 | args4j 23 | 2.33 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /resource_leaks/resource.leaks.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /resource_leaks/run.sh: -------------------------------------------------------------------------------- 1 | mvn install 2 | 3 | retVal=$? 4 | if [ $retVal -ne 0 ]; then 5 | echo "Compile error" 6 | fi 7 | 8 | mvn exec:java \ 9 | -Dexec.mainClass="Main" \ 10 | -Dexec.args="$*" \ 11 | 2>&1 \ 12 | | tee out.txt 13 | -------------------------------------------------------------------------------- /resource_leaks/src/main/java/Main.java: -------------------------------------------------------------------------------- 1 | import org.kohsuke.args4j.CmdLineException; 2 | import org.kohsuke.args4j.CmdLineParser; 3 | import org.kohsuke.args4j.Option; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | import java.nio.file.Files; 8 | import java.nio.file.Path; 9 | import java.nio.file.Paths; 10 | import java.sql.Connection; 11 | import java.sql.DriverManager; 12 | import java.sql.PreparedStatement; 13 | import java.sql.ResultSet; 14 | import java.sql.Statement; 15 | import java.util.Collections; 16 | import java.util.Arrays; 17 | import java.util.List; 18 | import java.util.stream.Stream; 19 | 20 | public final class Main { 21 | 22 | @Option(name="--experiment", usage="the type of experiment", required = true) 23 | private Experiment experiment; 24 | 25 | /** 26 | * Options for Files_lines, 27 | */ 28 | @Option(name="--file", usage="the file to open", required = false) 29 | private File file; 30 | 31 | /** 32 | * Options for Files_list, 33 | */ 34 | @Option(name="--directory", usage="the directory to list", required = false) 35 | private File directory; 36 | 37 | /** 38 | * Options for DB: 39 | * java_sql_Statement, 40 | * java_sql_PreparedStatement, 41 | * java_sql_Connection, 42 | * java_sql_ResultSet 43 | */ 44 | @Option(name="--dbType",usage="Sets type of DB to connect to") 45 | public String dbType = "mysql"; 46 | 47 | @Option(name="--hostname",usage="Sets hostname of the db to connect to") 48 | public String hostname = "localhost"; 49 | 50 | @Option(name="--port",usage="Sets port of the db to connect to") 51 | public int port = 3306; 52 | 53 | @Option(name="--user",usage="The user to connect to the db with") 54 | public String username = "root"; 55 | 56 | @Option(name="--pass",usage="The pass to connect to the db with") 57 | public String password = "password"; 58 | 59 | public void run() throws Exception { 60 | switch (experiment) { 61 | case Files_lines: 62 | filesLinesLeak(); 63 | break; 64 | case Files_walk: 65 | filesWalkLeak(); 66 | break; 67 | case java_sql_Statement: 68 | javaSqlStatementLeak(); 69 | break; 70 | case java_sql_PreparedStatement: 71 | javaSqlPreparedStatementLeak(); 72 | break; 73 | case java_sql_Connection: 74 | javaSqlConnectionLeak(); 75 | break; 76 | case java_sql_ResultSet: 77 | javaSqlResultSetLeak(); 78 | break; 79 | default: 80 | throw new UnsupportedOperationException("Experiment " + experiment + " not implemented yet!"); 81 | } 82 | } 83 | 84 | /** 85 | * java.nio.file.FileSystemException: pom.xml: Too many open files 86 | * at sun.nio.fs.UnixException.translateToIOException (UnixException.java:100) 87 | * at sun.nio.fs.UnixException.rethrowAsIOException (UnixException.java:106) 88 | * at sun.nio.fs.UnixException.rethrowAsIOException (UnixException.java:111) 89 | * at sun.nio.fs.UnixFileSystemProvider.newFileChannel (UnixFileSystemProvider.java:182) 90 | * at java.nio.channels.FileChannel.open (FileChannel.java:292) 91 | * at java.nio.channels.FileChannel.open (FileChannel.java:345) 92 | * at java.nio.file.Files.lines (Files.java:4104) 93 | * at java.nio.file.Files.lines (Files.java:4196) 94 | * at Main.filesLinesLeak (Main.java:72) 95 | * at Main.run (Main.java:63) 96 | * at Main.main (Main.java:85) 97 | * at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:254) 98 | * at java.lang.Thread.run (Thread.java:831) 99 | * 100 | * @throws IOException 101 | */ 102 | private void filesLinesLeak() throws IOException { 103 | while (true) { 104 | try { 105 | Stream lines = Files.lines(file.toPath()); 106 | throw new IgnoredException("Does this cause a leak?"); 107 | } catch (IgnoredException e) { 108 | // do nothing and keep leaking resources 109 | } 110 | } 111 | } 112 | 113 | /** 114 | * java.nio.file.FileSystemException: .: Too many open files 115 | * at sun.nio.fs.UnixException.translateToIOException (UnixException.java:100) 116 | * at sun.nio.fs.UnixException.rethrowAsIOException (UnixException.java:106) 117 | * at sun.nio.fs.UnixException.rethrowAsIOException (UnixException.java:111) 118 | * at sun.nio.fs.UnixFileSystemProvider.newDirectoryStream (UnixFileSystemProvider.java:419) 119 | * at java.nio.file.Files.newDirectoryStream (Files.java:476) 120 | * at java.nio.file.FileTreeWalker.visit (FileTreeWalker.java:300) 121 | * at java.nio.file.FileTreeWalker.walk (FileTreeWalker.java:322) 122 | * at java.nio.file.FileTreeIterator. (FileTreeIterator.java:71) 123 | * at java.nio.file.Files.walk (Files.java:3891) 124 | * at java.nio.file.Files.walk (Files.java:3945) 125 | * at Main.filesWalkLeak (Main.java:104) 126 | * at Main.run (Main.java:66) 127 | * at Main.main (Main.java:117) 128 | * at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:254) 129 | * at java.lang.Thread.run (Thread.java:831) 130 | * 131 | * @throws IOException 132 | */ 133 | private void filesWalkLeak() throws IOException { 134 | while (true) { 135 | try { 136 | Stream lines = Files.walk(directory.toPath()); 137 | throw new IgnoredException("Does this cause a leak?"); 138 | } catch (IgnoredException e) { 139 | // do nothing and keep leaking resources 140 | } 141 | } 142 | } 143 | 144 | /** 145 | * Nothing happened which I did not expect. 146 | * 147 | * @throws Exception 148 | */ 149 | private void javaSqlStatementLeak() throws Exception { 150 | final String url = "jdbc:" + dbType + "://" + hostname + ":" + port + "/"; 151 | try(Connection conn = DriverManager.getConnection(url, username, password)) { 152 | while (true) { 153 | try { 154 | Statement statement = conn.createStatement(); 155 | throw new IgnoredException("Does this cause a leak?"); 156 | } catch (IgnoredException e) { 157 | // do nothing and keep leaking resources 158 | } 159 | } 160 | } 161 | } 162 | 163 | /** 164 | * Nothing happened which I did not expect. 165 | * 166 | * @throws Exception 167 | */ 168 | private void javaSqlPreparedStatementLeak() throws Exception { 169 | final String url = "jdbc:" + dbType + "://" + hostname + ":" + port + "/"; 170 | try(Connection conn = DriverManager.getConnection(url, username, password)) { 171 | int i = 0; 172 | while (true) { 173 | try { 174 | PreparedStatement statement = conn.prepareStatement("select 1"); 175 | statement.execute(); 176 | throw new IgnoredException("Does this cause a leak?"); 177 | } catch (IgnoredException e) { 178 | // do nothing and keep leaking resources 179 | } 180 | } 181 | } 182 | } 183 | 184 | /** 185 | * java.sql.SQLNonTransientConnectionException: Could not connect to HostAddress{host='localhost', port=3306, type='master'}. Too many connections 186 | * at org.mariadb.jdbc.internal.util.exceptions.ExceptionFactory.createException (ExceptionFactory.java:73) 187 | * at org.mariadb.jdbc.internal.util.exceptions.ExceptionFactory.create (ExceptionFactory.java:188) 188 | * at org.mariadb.jdbc.internal.protocol.AbstractConnectProtocol.connectWithoutProxy (AbstractConnectProtocol.java:1402) 189 | * at org.mariadb.jdbc.internal.util.Utils.retrieveProxy (Utils.java:635) 190 | * at org.mariadb.jdbc.MariaDbConnection.newConnection (MariaDbConnection.java:150) 191 | * at org.mariadb.jdbc.Driver.connect (Driver.java:89) 192 | * at java.sql.DriverManager.getConnection (DriverManager.java:677) 193 | * at java.sql.DriverManager.getConnection (DriverManager.java:228) 194 | * at Main.javaSqlConnectionLeak (Main.java:181) 195 | * at Main.run (Main.java:74) 196 | * at Main.main (Main.java:194) 197 | * at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:254) 198 | * at java.lang.Thread.run (Thread.java:831) 199 | * Caused by: java.sql.SQLException: Too many connections 200 | * at org.mariadb.jdbc.internal.com.read.ReadInitialHandShakePacket. (ReadInitialHandShakePacket.java:92) 201 | * at org.mariadb.jdbc.internal.protocol.AbstractConnectProtocol.createConnection (AbstractConnectProtocol.java:527) 202 | * at org.mariadb.jdbc.internal.protocol.AbstractConnectProtocol.connectWithoutProxy (AbstractConnectProtocol.java:1389) 203 | * at org.mariadb.jdbc.internal.util.Utils.retrieveProxy (Utils.java:635) 204 | * at org.mariadb.jdbc.MariaDbConnection.newConnection (MariaDbConnection.java:150) 205 | * at org.mariadb.jdbc.Driver.connect (Driver.java:89) 206 | * at java.sql.DriverManager.getConnection (DriverManager.java:677) 207 | * at java.sql.DriverManager.getConnection (DriverManager.java:228) 208 | * at Main.javaSqlConnectionLeak (Main.java:181) 209 | * at Main.run (Main.java:74) 210 | * at Main.main (Main.java:194) 211 | * at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:254) 212 | * at java.lang.Thread.run (Thread.java:831) 213 | * 214 | * @throws Exception 215 | */ 216 | private void javaSqlConnectionLeak() throws Exception { 217 | final String url = "jdbc:" + dbType + "://" + hostname + ":" + port + "/"; 218 | 219 | while (true) { 220 | try { 221 | Connection conn = DriverManager.getConnection(url, username, password); 222 | throw new IgnoredException("Does this cause a leak?"); 223 | } catch (IgnoredException e) { 224 | // do nothing and keep leaking resources 225 | } 226 | } 227 | } 228 | 229 | /** 230 | * Nothing happened which I did not expect. 231 | * 232 | * @throws Exception 233 | */ 234 | private void javaSqlResultSetLeak() throws Exception { 235 | final String url = "jdbc:" + dbType + "://" + hostname + ":" + port + "/"; 236 | try(Connection conn = DriverManager.getConnection(url, username, password)) { 237 | while (true) { 238 | try { 239 | Statement statement = conn.createStatement(); 240 | statement.setFetchSize(1); 241 | ResultSet resultSet = statement.executeQuery("SELECT 1, 2" 242 | + " UNION SELECT 'a', 'b'" 243 | + " UNION SELECT '3', '4'"); 244 | throw new IgnoredException("Does this cause a leak?"); 245 | } catch (IgnoredException e) { 246 | // do nothing and keep leaking resources 247 | } 248 | } 249 | } 250 | } 251 | 252 | public static void main(String [] args) throws Exception { 253 | Main bean = new Main(); 254 | CmdLineParser parser = new CmdLineParser(bean); 255 | try { 256 | parser.parseArgument(args); 257 | bean.run(); 258 | } catch (CmdLineException e) { 259 | // handling of wrong arguments 260 | System.err.println(e.getMessage()); 261 | parser.printUsage(System.err); 262 | } 263 | 264 | } 265 | 266 | } 267 | 268 | enum Experiment { 269 | Files_lines, 270 | Files_walk, 271 | java_sql_Statement, 272 | java_sql_PreparedStatement, 273 | java_sql_Connection, 274 | java_sql_ResultSet 275 | } 276 | 277 | class IgnoredException extends Exception { 278 | 279 | IgnoredException(String s) { 280 | super(s); 281 | } 282 | } -------------------------------------------------------------------------------- /spark_protobuf/.gitignore: -------------------------------------------------------------------------------- 1 | spark-3.3.0-bin-hadoop3 2 | spark-3.3.0-bin-hadoop3.tgz 3 | target/ 4 | -------------------------------------------------------------------------------- /spark_protobuf/01_run_cluster.sh: -------------------------------------------------------------------------------- 1 | VERSION_SHORT=spark-3.3.0 2 | VERSION=spark-3.3.0-bin-hadoop3 3 | if [ ! -d "$VERSION" ] 4 | then 5 | curl -o $VERSION.tgz https://dlcdn.apache.org/spark/$VERSION_SHORT/$VERSION.tgz 6 | tar xf $VERSION.tgz 7 | rm $VERSION.tgz 8 | fi 9 | 10 | spark-3.3.0-bin-hadoop3/sbin/start-master.sh 11 | spark-3.3.0-bin-hadoop3/sbin/start-workers.sh 12 | 13 | -------------------------------------------------------------------------------- /spark_protobuf/02_run_word_count.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | < 2 | 4.0.0 3 | 4 | com.josephmate 5 | spark.protobuf.experiment 6 | 1.0.0 7 | 8 | 9 | 10 | 11 | org.apache.maven.plugins 12 | maven-compiler-plugin 13 | 3.8.0 14 | 15 | 17 16 | 17 | 18 | 19 | 20 | 21 | 22 | org.apache.spark 23 | spark-core_2.13 24 | 3.3.0 25 | 26 | 27 | 28 | org.apache.spark 29 | spark-sql_2.13 30 | 3.3.0 31 | provided 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /spark_protobuf/src/main/java/CharacterCount.java: -------------------------------------------------------------------------------- 1 | import org.apache.spark.sql.SparkSession; 2 | 3 | import java.util.Arrays; 4 | 5 | public class CharacterCount { 6 | public static void main(String[] args) { 7 | SparkSession sparkSession = SparkSession 8 | .builder() 9 | .appName("CharacterCount") 10 | .getOrCreate(); 11 | 12 | 13 | /* Copied from https://spark.apache.org/docs/latest/quick-start.html 14 | scala> val textFile = spark.read.textFile("README.md") 15 | scala> val wordCounts = textFile.flatMap(line => line.split(" ")).groupByKey(identity).count() 16 | scala> wordCounts.collect() 17 | */ 18 | System.out.println( 19 | sparkSession.read().text("README.md") 20 | .javaRDD() 21 | .flatMap(row -> Arrays.stream(row.toString().split("")).iterator() 22 | ) 23 | .groupBy(row -> row) 24 | .countByKey() 25 | ); 26 | /* 27 | {core,=1, =1, X=1, for=1, 2.=1, spark's=1, this=1, count=1, in=1, Intellij=1, ... 28 | */ 29 | 30 | } 31 | } -------------------------------------------------------------------------------- /spark_protobuf/src/main/java/WordCount.java: -------------------------------------------------------------------------------- 1 | import org.apache.spark.api.java.function.FlatMapFunction; 2 | import org.apache.spark.api.java.function.MapFunction; 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Encoders; 5 | import org.apache.spark.sql.Row; 6 | import org.apache.spark.sql.SparkSession; 7 | 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | public class WordCount { 12 | public static void main(String[] args) { 13 | SparkSession sparkSession = SparkSession 14 | .builder() 15 | .appName("WordCount") 16 | .getOrCreate(); 17 | 18 | Dataset dataSet = sparkSession.read().text("README.md"); 19 | dataSet.printSchema(); 20 | /* 21 | * root 22 | * |-- value: string (nullable = true) 23 | * 24 | * it's string, but how do i know what type that is in java? 25 | */ 26 | 27 | List types = sparkSession.read().text("README.md") 28 | .map((MapFunction) row -> row.getAs("value").getClass().getCanonicalName(), 29 | Encoders.STRING()) 30 | .collectAsList(); 31 | if (!types.isEmpty()) { 32 | System.out.println(types.get(0)); 33 | } 34 | /* 35 | * output: java.lang.String 36 | */ 37 | 38 | 39 | /* Copied from https://spark.apache.org/docs/latest/quick-start.html 40 | scala> val textFile = spark.read.textFile("README.md") 41 | scala> val wordCounts = textFile.flatMap(line => line.split(" ")).groupByKey(identity).count() 42 | scala> wordCounts.collect() 43 | */ 44 | System.out.println( 45 | sparkSession.read().text("README.md") 46 | .flatMap((FlatMapFunction) row -> Arrays.stream(row.getAs("value") 47 | .toString() 48 | .split(" ")) 49 | .iterator(), 50 | Encoders.STRING()) 51 | .groupByKey((MapFunction ) i -> i, Encoders.STRING() ) 52 | .count() 53 | .collectAsList() 54 | ); 55 | /* 56 | [(some,1), (column,1), (character,1), (sorts,1), (copy,1), ... 57 | */ 58 | 59 | } 60 | } -------------------------------------------------------------------------------- /spark_protobuf/src/main/java/WordCountJavaRdd.java: -------------------------------------------------------------------------------- 1 | import org.apache.spark.api.java.function.FlatMapFunction; 2 | import org.apache.spark.api.java.function.MapFunction; 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Encoders; 5 | import org.apache.spark.sql.Row; 6 | import org.apache.spark.sql.SparkSession; 7 | 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | public class WordCountJavaRdd { 12 | public static void main(String[] args) { 13 | SparkSession sparkSession = SparkSession 14 | .builder() 15 | .appName("WordCountJavaRdd") 16 | .getOrCreate(); 17 | 18 | 19 | /* Copied from https://spark.apache.org/docs/latest/quick-start.html 20 | scala> val textFile = spark.read.textFile("README.md") 21 | scala> val wordCounts = textFile.flatMap(line => line.split(" ")).groupByKey(identity).count() 22 | scala> wordCounts.collect() 23 | */ 24 | System.out.println( 25 | sparkSession.read().text("README.md") 26 | .javaRDD() 27 | .flatMap(row -> Arrays.stream(row.getAs("value") 28 | .toString() 29 | .split(" ")) 30 | .iterator() 31 | ) 32 | .groupBy(row -> row) 33 | .countByKey() 34 | ); 35 | /* 36 | {core,=1, =1, X=1, for=1, 2.=1, spark's=1, this=1, count=1, in=1, Intellij=1, ... 37 | */ 38 | 39 | } 40 | } -------------------------------------------------------------------------------- /stack_trace_super_sub_classes/Main.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | 3 | public final class Main { 4 | 5 | private static class Sub extends Super{ 6 | @Override 7 | public void callsSuper(boolean throwIt) { 8 | super.callsSuper(throwIt); 9 | throw new RuntimeException(); 10 | } 11 | } 12 | 13 | private static class Super { 14 | public void callsSuper(boolean throwIt) { 15 | if (throwIt) { 16 | throw new RuntimeException(); 17 | } 18 | } 19 | } 20 | 21 | public static void main(String [] args) throws Exception { 22 | try { 23 | new Sub().callsSuper(true); 24 | } catch (RuntimeException e) { 25 | e.printStackTrace(); 26 | } 27 | try { 28 | new Sub().callsSuper(false); 29 | } catch (RuntimeException e) { 30 | e.printStackTrace(); 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /stack_trace_super_sub_classes/README.md: -------------------------------------------------------------------------------- 1 | What does stack trace of the overrided method look like compared to super class's method? 2 | 3 | 4 | 1. Think about the answer. 5 | 2. Try to write up some expected stacktraces. 6 | 3. Try to write some experimental code to determine the answer and compare with 7 | your stack traces. 8 | 9 |
10 | Click to reveal my expected stack traces. 11 | 12 | If the subclass throws the exception, I would expect it to look something like: 13 | 14 | ``` 15 | RuntimeException 16 | Sub.method() 17 | Main.main() 18 | ``` 19 | 20 | If the super class throws the exception, I would expect it to look something 21 | like: 22 | ``` 23 | RuntimeException 24 | Super.method() 25 | Sub.method() 26 | Main.main() 27 | ``` 28 | since it needs to call the subclass first, so the subclass can call the super 29 | method. 30 | There's not way to directly call the super method. 31 |
32 | 33 |
34 | Click to reveal stack traces from my experiment. 35 | 36 | ``` 37 | java.lang.RuntimeException 38 | at Main$Super.callsSuper(Main.java:16) 39 | at Main$Sub.callsSuper(Main.java:8) 40 | at Main.main(Main.java:23) 41 | java.lang.RuntimeException 42 | at Main$Sub.callsSuper(Main.java:9) 43 | at Main.main(Main.java:28) 44 | ``` 45 | 46 | In the fist example, the sub class is called first, then the super method throws the exception. 47 | In the second example, the sub class throws the exception, so there's only two 48 | elements on the stack: the sub method and the main method. 49 |
50 | 51 | -------------------------------------------------------------------------------- /stack_trace_super_sub_classes/run.sh: -------------------------------------------------------------------------------- 1 | javac Main.java \ 2 | && java Main 3 | 4 | -------------------------------------------------------------------------------- /tree_map_corruption/cpp/.gitignore: -------------------------------------------------------------------------------- 1 | SimpleRepro 2 | -------------------------------------------------------------------------------- /tree_map_corruption/cpp/Makefile: -------------------------------------------------------------------------------- 1 | # Compiler 2 | CXX = g++ 3 | 4 | # Compiler flags 5 | CXXFLAGS = -std=c++11 -pthread 6 | 7 | # Target executable name 8 | TARGET = SimpleRepro 9 | 10 | # Source file 11 | SRC = main.cpp 12 | 13 | # Build target 14 | all: $(TARGET) 15 | 16 | $(TARGET): $(SRC) 17 | $(CXX) $(CXXFLAGS) -o $(TARGET) $(SRC) 18 | 19 | # Clean up build files 20 | clean: 21 | rm -f $(TARGET) 22 | 23 | .PHONY: all clean 24 | 25 | -------------------------------------------------------------------------------- /tree_map_corruption/cpp/README.md: -------------------------------------------------------------------------------- 1 | # Results 2 | 3 | Most of the time, the program succeds and runs like this with all threads 4 | completing. 5 | ``` 6 | ./SimpleRepro 7 | Thread Thread Thread Thread 02 started.3Thread started.Thread 1 started.4 8 | started.Thread 4: 10% complete (10/100 updates) 9 | Thread Thread Thread 4: 20% complete (20/100 updates) 10 | started.Thread 4: 30% complete (30/100 updates) 11 | 2: 10% complete (10/100 updates) 12 | Thread 4: 40% complete (40/100 updates) 13 | Thread 2: 20% complete (20/100 updates) 14 | Thread 4: 50% complete (50/100 updates) 15 | 0: 10% complete (10/100 updates) 16 | Thread 4: 60% complete (60/100 updates) 17 | Thread 0: 20% complete (20/100 updates) 18 | Thread 4: 70% complete (70/100 updates) 19 | Thread 0: 30% complete (30/100 updates) 20 | Thread 4: 80% complete (80/100 updates) 21 | Thread 0: 40% complete (40/100 updates) 22 | Thread 4: 90% complete (90/100 updates) 23 | Thread 0: 50% complete (50/100 updates) 24 | Thread 4: 100% complete (100/100 updates) 25 | Thread 4 completed.Thread 0: 60% complete (60/100 updates) 26 | Thread 0: 70% complete (70/100 updates) 27 | Thread 0: 80% complete (80/100 updates) 28 | Thread 2: 30% complete (Thread 30/100 updates) 29 | 3: 10% complete (10/100 updates) 30 | Thread 1: 10% complete (10/100 updates) 31 | Thread 3: 20Thread 2: 40% complete (40/100 updates) 32 | Thread 2: 50% complete (50/100 updates) 33 | Thread 2: 60% complete (60/100 updates) 34 | Thread 2: 70% complete (70/100 updates) 35 | Thread 2: 80% complete (80/100 updates) 36 | Thread 2: 90% complete (90/100 updates) 37 | Thread 1: 20% complete (Thread 2200: 90% complete (90/100 updates) 38 | /100 updates) 39 | Thread 0: 100% complete (100/100 updates) 40 | Thread 0 completed.Thread 1: 30% complete (30/100 updates) 41 | : % complete (20/100 updates) 42 | Thread 3: 30% complete (30/100 updates) 43 | 100% complete (100/100 updates) 44 | Thread 1: 40% complete (40/100 updates) 45 | Thread 1: 50% complete (50/100 updates) 46 | Thread 1: 60% complete (60/100 updates) 47 | Thread 1: 70% complete (70/100 updates) 48 | Thread 1: 80% complete (80/100 updates) 49 | Thread 1: 90% complete (90/100 updates) 50 | Thread 1: 100% complete (100/100 updates) 51 | Thread 1 completed.Thread 2 completed.Thread 3: 40% complete (40/100 updates) 52 | Thread 3: 50% complete (50/100 updates) 53 | Thread 3: 60% complete (60/100 updates) 54 | Thread 3: 70% complete (70/100 updates) 55 | Thread 3: 80% complete (80/100 updates) 56 | Thread 3: 90% complete (90/100 updates) 57 | Thread 3: 100% complete (100/100 updates) 58 | Thread 3 completed. 59 | ``` 60 | 61 | Occasionally, the program segfaults and crashes like before. 62 | This is what I was expecting the worst failure to look like. 63 | ``` 64 | ./SimpleRepro 65 | zsh: segmentation fault ./SimpleRepro 66 | ``` 67 | 68 | However, very rarely the output would look like indicating the threads got 69 | stuck. 70 | After waiting 10 minutes, the threads never completed. 71 | I did not expect this to happen because I thought an exception would be required 72 | for the problem to occur, that that is not happening here. 73 | ``` 74 | ./SimpleRepro 75 | Thread 0 started.Thread Thread 0: 10% complete (Thread 10/10012Thread started.3 76 | started.Thread 4 started.Thread 4: 10% complete (10/100 updates) 77 | updates) 78 | Thread 0: 20% complete (20/100 updates) 79 | Thread 0: 30% complete (30/100 updates) 80 | Thread 0: 40% complete (40/100 updates) 81 | Thread 1: 10% complete (10/100 updates) 82 | started.Thread 2: 10% complete (10/100 updates) 83 | Thread 1: 20% complete (20/100 updates) 84 | ``` 85 | 86 | From top we can see that the problem reproduced because of the high cpu 87 | utilization. 88 | ``` 89 | top 90 | PID COMMAND %CPU TIME #TH #WQ #PORT MEM PURG CMPRS PGRP 91 | PPID STATE BOOSTS %CPU_ME %CPU_OTHRS UID FAULTS COW MSGSENT 92 | MSGRECV 93 | 59815 SimpleRepro 170.8 08:49.61 6/5 0 15 372K 0B 308K 59815 94 | 44017 running *0[1] 0.00000 0.00000 501 376 85 69+ 95 | 13 96 | ``` 97 | 98 | # Running 99 | 100 | Make sure you have `g++` installed 101 | ``` 102 | make 103 | ./SimpleRepro 104 | ``` 105 | -------------------------------------------------------------------------------- /tree_map_corruption/cpp/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | int main(int argc, char* argv[]) { 8 | int numThreads = (argc >= 2) ? std::stoi(argv[1]) : 5; 9 | int numUpdates = (argc >= 3) ? std::stoi(argv[2]) : 100; 10 | int progressRatio = 10; 11 | 12 | // Equivalent of TreeMap in C++ is std::map 13 | // Uses same red black tree data structure 14 | std::map sortedMap; 15 | 16 | std::vector threads; 17 | std::atomic threadIdGenerator(0); 18 | 19 | for (int i = 0; i < numThreads; i++) { 20 | threads.emplace_back([&]() { 21 | // copy so that we don't use the latest value of i (5) 22 | int threadId = threadIdGenerator.fetch_add(1); 23 | std::random_device rd; 24 | std::mt19937 gen(rd()); 25 | std::uniform_int_distribution<> dis(0, 999); 26 | 27 | int progressThreshold = numUpdates / progressRatio; 28 | std::cout << "Thread " << threadId << " started."; 29 | for (int j = 0; j < numUpdates; j++) { 30 | try { 31 | int key = dis(gen); 32 | int value = dis(gen); 33 | 34 | sortedMap[key] = value; 35 | } catch (const std::exception& e) { 36 | std::cerr << "Caught exception on thread " << threadId << ":" << e.what() << std::endl; 37 | } 38 | 39 | if ((j + 1) % progressThreshold == 0) { 40 | std::cout << "Thread " << threadId << ": " 41 | << ((j + 1) * 100 / numUpdates) << "% complete (" 42 | << (j + 1) << "/" << numUpdates << " updates)\n"; 43 | } 44 | } 45 | std::cout << "Thread " << threadId << " completed."; 46 | }); 47 | } 48 | 49 | // Start all threads 50 | for (auto& thread : threads) { 51 | thread.join(); 52 | } 53 | 54 | return 0; 55 | } 56 | 57 | -------------------------------------------------------------------------------- /tree_map_corruption/csharp/.gitignore: -------------------------------------------------------------------------------- 1 | TreeMapCorruption/.vscode/ 2 | TreeMapCorruption/TreeMapCorruption/bin/ 3 | TreeMapCorruption/TreeMapCorruption/obj/ 4 | -------------------------------------------------------------------------------- /tree_map_corruption/csharp/README.md: -------------------------------------------------------------------------------- 1 | 2 | The problem reproduced in C#. 3 | Running the program sometimes results in it hanging and 300% CPU util. 4 | 5 | ``` 6 | dotnet run TreeMapCorruption.csproj 7 | TreeMapCorruption.csproj 8 | 0: Started 9 | 1: Started 10 | 2: Started 11 | 3: Started 12 | 4: Started 13 | Exception Message: Object reference not set to an instance of an object. 14 | Exception Message: Object reference not set to an instance of an object. 15 | Exception Message: Object reference not set to an instance of an object. 16 | 17 | ``` 18 | 19 | ``` 20 | ps aux | rg 'USER|[T]reeMap' 21 | USER PID %CPU %MEM VSZ RSS TT STAT STARTED TIME 22 | COMMAND 23 | joseph 83617 364.9 0.3 6608084 14388 s005 R+ 11:26pm 9:59.40 24 | /Users/joseph/projects/java-by-experiments-private/tree_map_corruption/csharp/TreeMapCorruption/TreeMapCorruption/bin/Debug/netcoreapp3.0/TreeMapCorruption 25 | TreeMapCorruption.csproj 26 | joseph 83666 0.0 0.0 4265240 12 s006 S+ 11:29pm 0:00.00 27 | rg USER|[T]reeMap 28 | joseph 83614 0.0 2.0 6933468 82860 s005 S+ 11:26pm 0:06.95 29 | dotnet run TreeMapCorruption.csproj 30 | ``` 31 | 32 | 33 | -------------------------------------------------------------------------------- /tree_map_corruption/csharp/TreeMapCorruption/TreeMapCorruption.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29326.143 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TreeMapCorruption", "TreeMapCorruption\TreeMapCorruption.csproj", "{B96A2CE6-8DC1-4A4E-9FBA-2ACF64AE1CBA}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {B96A2CE6-8DC1-4A4E-9FBA-2ACF64AE1CBA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {B96A2CE6-8DC1-4A4E-9FBA-2ACF64AE1CBA}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {B96A2CE6-8DC1-4A4E-9FBA-2ACF64AE1CBA}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {B96A2CE6-8DC1-4A4E-9FBA-2ACF64AE1CBA}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {6E06648F-043F-4340-948E-E5D9CBC0FDEE} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /tree_map_corruption/csharp/TreeMapCorruption/TreeMapCorruption/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading; 4 | 5 | namespace TreeMapCorruption 6 | { 7 | class Program 8 | { 9 | static void Main(string[] args) 10 | { 11 | for (int i = 0; i < args.Length; i++) 12 | { 13 | Console.WriteLine($"Argument {i}: {args[i]}"); 14 | } 15 | int numThreads = args.Length > 1 ? int.Parse(args[1]) : 5; 16 | int numUpdates = args.Length > 2 ? int.Parse(args[2]) : 10_000; 17 | 18 | // TreeMap equivalent in C# is SortedDictionary 19 | var sortedDictionary = new SortedDictionary(); 20 | 21 | var threads = new List(); 22 | for (int i = 0; i < numThreads; i++) 23 | { 24 | int myThreadId = i; 25 | var thread = new Thread(() => 26 | { 27 | Console.WriteLine($"{myThreadId}: Started"); 28 | var random = new Random(); 29 | for (int j = 0; j < numUpdates; j++) 30 | { 31 | try 32 | { 33 | sortedDictionary[random.Next(1000)] = random.Next(1000); 34 | } 35 | catch (Exception ex) 36 | { 37 | // Let it keep going to reproduce the issue 38 | Console.WriteLine("Exception Message: " + ex.Message); 39 | } 40 | } 41 | Console.WriteLine($"{myThreadId}: Finished"); 42 | }); 43 | threads.Add(thread); 44 | } 45 | 46 | foreach (var thread in threads) 47 | { 48 | thread.Start(); 49 | } 50 | 51 | foreach (var thread in threads) 52 | { 53 | thread.Join(); 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /tree_map_corruption/csharp/TreeMapCorruption/TreeMapCorruption/TreeMapCorruption.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.0 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /tree_map_corruption/golang/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ``` 4 | env GOTRACEBACK=all go run simple_repro.go 5 | 0: started 6 | 2: started 7 | 1: started 8 | 3: started 9 | 4: started 10 | ``` 11 | 12 | ``` 13 | ps aux | rg '[U]SER|[s]imple' 14 | USER PID %CPU %MEM VSZ RSS TT STAT STARTED TIME COMMAND 15 | joseph 84608 366.4 0.1 5495976 2408 s004 R+ 11:57pm 0:38.18 /var/folders/gd/y2bvmd0530568763mxnfnpdw0000gn/T/go-build2633847538/b001/exe/simple_repro 16 | joseph 84597 0.0 0.4 5540252 15992 s004 S+ 11:57pm 0:00.32 go run simple_repro.go 17 | ``` 18 | -------------------------------------------------------------------------------- /tree_map_corruption/golang/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/josephmate/java-by-experiments/tree_map_corruption/golang 2 | 3 | go 1.23.5 4 | 5 | require github.com/emirpasic/gods v1.18.1 // indirect 6 | -------------------------------------------------------------------------------- /tree_map_corruption/golang/go.sum: -------------------------------------------------------------------------------- 1 | github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= 2 | github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= 3 | -------------------------------------------------------------------------------- /tree_map_corruption/golang/simple_repro.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "strconv" 8 | "sync" 9 | "time" 10 | 11 | "github.com/emirpasic/gods/trees/redblacktree" 12 | ) 13 | 14 | func main() { 15 | // Parse command-line arguments 16 | numThreads := 5 17 | numUpdates := 1000 18 | 19 | if len(os.Args) >= 2 { 20 | numThreads, _ = strconv.Atoi(os.Args[1]) 21 | } 22 | if len(os.Args) >= 3 { 23 | numUpdates, _ = strconv.Atoi(os.Args[2]) 24 | } 25 | 26 | tree := redblacktree.NewWithIntComparator() 27 | 28 | var wg sync.WaitGroup 29 | 30 | for i := 0; i < numThreads; i++ { 31 | myThreadId := i 32 | wg.Add(1) 33 | go func() { 34 | defer wg.Done() 35 | fmt.Printf("%d: started\n", myThreadId) 36 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 37 | for j := 0; j < numUpdates; j++ { 38 | key := r.Intn(1000) 39 | value := r.Intn(1000) 40 | // not locking on purpose 41 | tree.Put(key, value) 42 | } 43 | fmt.Printf("%d: finished\n", myThreadId) 44 | }() 45 | } 46 | 47 | wg.Wait() 48 | } 49 | -------------------------------------------------------------------------------- /tree_map_corruption/java/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # SimpleRepro 4 | 5 | ``` 6 | # may need to run multiple times and adjust number of threads and number of updates 7 | # to get the problem to reproduce 8 | mvn install 9 | java \ 10 | -cp $(mvn dependency:build-classpath -Dmdep.outputFile=/dev/stdout -q):target/classes \ 11 | SimpleRepro 5 1000 12 | 13 | # in another terminal/process: 14 | kill -QUIT $(ps aux | rg '[S]impleRepro' | awk '{print $2}') 15 | 16 | # in original terminal/process: 17 | "Thread-0" #22 [14700] prio=5 os_prio=0 cpu=10359.38ms elapsed=11.49s tid=0x000001cdc35aaf60 nid=14700 runnable [0x00000047cfffe000] 18 | java.lang.Thread.State: RUNNABLE 19 | at java.util.TreeMap.put(java.base@19.0.1/TreeMap.java:826) 20 | at java.util.TreeMap.put(java.base@19.0.1/TreeMap.java:534) 21 | at SimpleRepro.lambda$main$0(SimpleRepro.java:29) 22 | at SimpleRepro$$Lambda$14/0x00000008010031f0.run(Unknown Source) 23 | at java.lang.Thread.run(java.base@19.0.1/Thread.java:1589) 24 | ``` 25 | 26 | 27 | # ExecutorUncaughtRepro 28 | 29 | Realistic looking reproduction of the problem using a ExecutorService. 30 | 31 | ``` 32 | # may need to run multiple times and adjust number of threads and number of updates 33 | # to get the problem to reproduce 34 | mvn install 35 | java \ 36 | -cp $(mvn dependency:build-classpath -Dmdep.outputFile=/dev/stdout -q):target/classes \ 37 | ExecutorUncaughtRepro 5 1000 38 | 39 | # in another terminal/process: 40 | kill -QUIT $(ps aux | rg '[E]xecutorUncaughtRepro' | awk '{print $2}') 41 | 42 | # in original terminal/process: 43 | "pool-1-thread-1" #22 [15356] prio=5 os_prio=0 cpu=17734.38ms elapsed=21.39s tid=0x0000023c45dd3e90 nid=15356 runnable [0x000000780b4fe000] 44 | java.lang.Thread.State: RUNNABLE 45 | at java.util.TreeMap.put(java.base@19.0.1/TreeMap.java:826) 46 | at java.util.TreeMap.put(java.base@19.0.1/TreeMap.java:534) 47 | at ExecutorUncaughtRepro.lambda$main$0(ExecutorUncaughtRepro.java:33) 48 | at ExecutorUncaughtRepro$$Lambda$14/0x00000008010031f0.run(Unknown Source) 49 | at java.util.concurrent.Executors$RunnableAdapter.call(java.base@19.0.1/Executors.java:577) 50 | at java.util.concurrent.FutureTask.run(java.base@19.0.1/FutureTask.java:317) 51 | at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@19.0.1/ThreadPoolExecutor.java:1144) 52 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@19.0.1/ThreadPoolExecutor.java:642) 53 | at java.lang.Thread.run(java.base@19.0.1/Thread.java:1589) 54 | 55 | ``` 56 | 57 | # GrpcRepo 58 | 59 | Realistic looking reproduction of the problem using a gRPC service. 60 | 61 | ``` 62 | # may need to run multiple times and adjust number of threads and number of updates, 63 | # hardcoded within the GrpcRepro code to get the problem to reproduce 64 | mvn install 65 | java \ 66 | -cp $(mvn dependency:build-classpath -Dmdep.outputFile=/dev/stdout -q):target/classes \ 67 | GrpcRepro 68 | 69 | # in another terminal/process: 70 | kill -QUIT $(ps aux | rg '[G]rpcRepro' | awk '{print $2}') 71 | 72 | # in original terminal/process: 73 | "grpc-default-executor-23" #54 [8796] daemon prio=5 os_prio=0 cpu=18671.88ms elapsed=175.50s tid=0x00000168b6c707c0 nid=8796 runnable [0x000000059fbfe000] 74 | java.lang.Thread.State: RUNNABLE 75 | at java.util.TreeMap.put(java.base@19.0.1/TreeMap.java:826) 76 | at java.util.TreeMap.put(java.base@19.0.1/TreeMap.java:534) 77 | at ReceiptProcessorServiceImpl.addReceipt(GrpcRepro.java:59) 78 | at ReceiptProcessorServiceGrpc$MethodHandlers.invoke(ReceiptProcessorServiceGrpc.java:185) 79 | at io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:182) 80 | at io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:346) 81 | at io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:860) 82 | at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) 83 | at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) 84 | at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@19.0.1/ThreadPoolExecutor.java:1144) 85 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@19.0.1/ThreadPoolExecutor.java:642) 86 | at java.lang.Thread.run(java.base@19.0.1/Thread.java:1589) 87 | ``` 88 | 89 | # ProtectedSimpleRepro 90 | 91 | 92 | ``` 93 | # Notice that no matter what parameters or use, or how many times you run it, 94 | # it never gets stuck in a loop. 95 | mvn install 96 | java \ 97 | -cp $(mvn dependency:build-classpath -Dmdep.outputFile=/dev/stdout -q):target/classes \ 98 | ProtectedSimpleRepro 5 1000 99 | Exception in thread "Thread-3" java.util.ConcurrentModificationException: TreeMap corrupted. Loop detected 100 | at ProtectedTreeMap.put(ProtectedTreeMap.java:840) 101 | at ProtectedTreeMap.put(ProtectedTreeMap.java:539) 102 | at ProtectedSimpleRepro.lambda$main$0(ProtectedSimpleRepro.java:29) 103 | at java.base/java.lang.Thread.run(Thread.java:833) 104 | 105 | # Much better than having an infinite loop! 106 | ``` 107 | -------------------------------------------------------------------------------- /tree_map_corruption/java/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.josephmate 5 | tree-map-corruption 6 | 1 7 | 8 | 9 | 17 10 | 17 11 | 17 12 | 17 13 | 3.24.1 14 | 1.57.2 15 | 16 | 17 | 18 | 19 | 20 | com.github.os72 21 | protoc-jar-maven-plugin 22 | 3.11.4 23 | 24 | 25 | generate-sources 26 | 27 | run 28 | 29 | 30 | com.google.protobuf:protoc:3.0.0 31 | 32 | src/main/protobuf 33 | 34 | 35 | 36 | java 37 | 38 | 39 | grpc-java 40 | io.grpc:protoc-gen-grpc-java:1.0.1 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | com.google.protobuf 53 | protobuf-java 54 | ${protobuf.version} 55 | 56 | 57 | 58 | 59 | io.grpc 60 | grpc-netty-shaded 61 | ${grpc.version} 62 | 63 | 64 | io.grpc 65 | grpc-protobuf 66 | ${grpc.version} 67 | 68 | 69 | io.grpc 70 | grpc-stub 71 | ${grpc.version} 72 | 73 | 74 | javax.annotation 75 | javax.annotation-api 76 | 1.3.2 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/java/ExecutorUncaughtRepro.java: -------------------------------------------------------------------------------- 1 | import java.util.Random; 2 | import java.util.TreeMap; 3 | import java.util.concurrent.ExecutorService; 4 | import java.util.concurrent.Executors; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | public class ExecutorUncaughtRepro { 8 | 9 | public static void main(String[] args) throws Exception { 10 | final int numThreads; 11 | if (args.length >= 1) { 12 | numThreads = Integer.parseInt(args[0]); 13 | } else { 14 | numThreads = 5; 15 | } 16 | 17 | final int numUpdatesPerThread; 18 | if (args.length >= 2) { 19 | numUpdatesPerThread = Integer.parseInt(args[1]); 20 | } else { 21 | numUpdatesPerThread = 100000; 22 | } 23 | 24 | final ExecutorService pool = Executors.newFixedThreadPool(numThreads); 25 | final TreeMap treeMap = new TreeMap<>(); 26 | 27 | Random random = new Random(); 28 | for (int i = 0; i < numThreads*numUpdatesPerThread; i++) { 29 | pool.submit( () -> { 30 | treeMap.put(random.nextInt(10000), random.nextInt(10000)); 31 | }); 32 | } 33 | 34 | pool.shutdown(); 35 | pool.awaitTermination(1, TimeUnit.DAYS); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/java/ExploringTreeMap.java: -------------------------------------------------------------------------------- 1 | import java.lang.reflect.Field; 2 | import java.util.Arrays; 3 | import java.util.TreeMap; 4 | 5 | public class ExploringTreeMap { 6 | 7 | /** 8 | * In order to run you need to add these JVM args: 9 | *
10 |      * --add-opens java.base/java.util=ALL-UNNAMED
11 |      * 
12 | * @param args 13 | */ 14 | public static void main(String[] args) throws Exception { 15 | 16 | TreeMap treeMap = new TreeMap<>(); 17 | for (int i = 0; i < 10; i++) { 18 | treeMap.put(i, i); 19 | } 20 | 21 | TreeMapExplorer treeMapExplorer = new TreeMapExplorer(treeMap); 22 | treeMapExplorer.print(); 23 | } 24 | 25 | 26 | } 27 | 28 | -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/java/GrpcRepro.java: -------------------------------------------------------------------------------- 1 | import java.util.ArrayList; 2 | import java.util.List; 3 | import java.util.Random; 4 | import java.util.TreeMap; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | import com.google.common.util.concurrent.ListenableFuture; 8 | import io.grpc.ManagedChannel; 9 | import io.grpc.ManagedChannelBuilder; 10 | import io.grpc.Server; 11 | import io.grpc.ServerBuilder; 12 | import io.grpc.stub.StreamObserver; 13 | 14 | /** 15 | * Reproduces the corrupted TreeMap issues through Grpc rather than directly through threads. 16 | * 17 | * Demonstrates that the simplified 2 thread reproduce code isn't that unrealistic. Exceptionsm 18 | * get accidentally swallowed all the time especially if there are RuntimeExceptions like NPEs. 19 | */ 20 | public class GrpcRepro { 21 | public static void main(String[] args) throws Exception { 22 | final Server server = ServerBuilder.forPort(0) 23 | .addService(new ReceiptProcessorServiceImpl()) 24 | .build() 25 | .start(); 26 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 27 | server.shutdownNow(); 28 | })); 29 | 30 | final int port = server.getPort(); 31 | System.out.println("Server started, listening on " + port); 32 | 33 | final ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", port) 34 | .usePlaintext() 35 | .build(); 36 | final ReceiptProcessorServiceGrpc.ReceiptProcessorServiceFutureStub futureStub = ReceiptProcessorServiceGrpc.newFutureStub(channel); 37 | 38 | List> futures = new ArrayList<>(); 39 | Random random = new Random(); 40 | for (int i = 0; i < 100_000; i++) { 41 | ReceiptProcessorServiceOuterClass.AddReceiptRequest request = ReceiptProcessorServiceOuterClass.AddReceiptRequest.newBuilder() 42 | .setTimestamp(random.nextInt(10_000)) 43 | .setTotalPrice(random.nextInt(10_000)) 44 | .build(); 45 | futures.add(futureStub.addReceipt(request)); 46 | } 47 | 48 | for (ListenableFuture future : futures) { 49 | future.get(); 50 | } 51 | 52 | server.shutdown(); 53 | server.awaitTermination(1, TimeUnit.DAYS); 54 | } 55 | } 56 | 57 | class ReceiptProcessorServiceImpl extends ReceiptProcessorServiceGrpc.ReceiptProcessorServiceImplBase { 58 | private final TreeMap receipts = new TreeMap<>(); 59 | 60 | @Override 61 | public void addReceipt( 62 | ReceiptProcessorServiceOuterClass.AddReceiptRequest req, 63 | StreamObserver responseObserver 64 | ) { 65 | int timestamp = req.getTimestamp(); 66 | int totalPrice = req.getTotalPrice(); 67 | receipts.put(timestamp, totalPrice); 68 | ReceiptProcessorServiceOuterClass.AddReceiptResponse response = ReceiptProcessorServiceOuterClass.AddReceiptResponse.newBuilder().build(); 69 | responseObserver.onNext(response); 70 | responseObserver.onCompleted(); 71 | } 72 | } -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/java/GrpcThrowNPE.java: -------------------------------------------------------------------------------- 1 | import java.util.ArrayList; 2 | import java.util.List; 3 | import java.util.Random; 4 | import java.util.TreeMap; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | import com.google.common.util.concurrent.ListenableFuture; 8 | import io.grpc.ManagedChannel; 9 | import io.grpc.ManagedChannelBuilder; 10 | import io.grpc.Server; 11 | import io.grpc.ServerBuilder; 12 | import io.grpc.stub.StreamObserver; 13 | 14 | public class GrpcThrowNPE { 15 | public static void main(String[] args) throws Exception { 16 | final Server server = ServerBuilder.forPort(0) 17 | .addService(new ReceiptProcessorThrowNPEImpl()) 18 | .build() 19 | .start(); 20 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 21 | server.shutdownNow(); 22 | })); 23 | 24 | final int port = server.getPort(); 25 | System.out.println("Server started, listening on " + port); 26 | 27 | final ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", port) 28 | .usePlaintext() 29 | .build(); 30 | final ReceiptProcessorServiceGrpc.ReceiptProcessorServiceFutureStub futureStub = ReceiptProcessorServiceGrpc.newFutureStub(channel); 31 | 32 | List> futures = new ArrayList<>(); 33 | Random random = new Random(); 34 | for (int i = 0; i < 100; i++) { 35 | ReceiptProcessorServiceOuterClass.AddReceiptRequest request = ReceiptProcessorServiceOuterClass.AddReceiptRequest.newBuilder() 36 | .setTimestamp(random.nextInt(10_000)) 37 | .setTotalPrice(random.nextInt(10_000)) 38 | .build(); 39 | futures.add(futureStub.addReceipt(request)); 40 | } 41 | 42 | for (ListenableFuture future : futures) { 43 | future.get(); 44 | } 45 | 46 | server.shutdown(); 47 | server.awaitTermination(1, TimeUnit.DAYS); 48 | } 49 | } 50 | 51 | class ReceiptProcessorThrowNPEImpl extends ReceiptProcessorServiceGrpc.ReceiptProcessorServiceImplBase { 52 | @Override 53 | public void addReceipt( 54 | ReceiptProcessorServiceOuterClass.AddReceiptRequest req, 55 | StreamObserver responseObserver 56 | ) { 57 | throw new NullPointerException(); 58 | } 59 | } -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/java/ProtectedSimpleRepro.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | 3 | public class ProtectedSimpleRepro { 4 | 5 | public static void main(String[] args) throws Exception { 6 | 7 | final int numThreads; 8 | if (args.length >= 1) { 9 | numThreads = Integer.parseInt(args[0]); 10 | } else { 11 | numThreads = 5; 12 | } 13 | 14 | final int numUpdates; 15 | if (args.length >= 2) { 16 | numUpdates = Integer.parseInt(args[1]); 17 | } else { 18 | numUpdates = 1000; 19 | } 20 | 21 | final ProtectedTreeMap treeMap = new ProtectedTreeMap<>(); 22 | 23 | List threads = new ArrayList<>(); 24 | for (int i = 0; i < numThreads; i++) { 25 | threads.add(new Thread(() -> { 26 | Random random = new Random(); 27 | for(int j = 0; j < numUpdates; j++) { 28 | try { 29 | treeMap.put(random.nextInt(1000), random.nextInt(1000)); 30 | } catch (NullPointerException e) { 31 | // let it keep going so we can reproduce the issue. 32 | } 33 | } 34 | })); 35 | } 36 | 37 | for (Thread thread : threads) { 38 | thread.start(); 39 | } 40 | 41 | for (Thread thread : threads) { 42 | thread.join(); 43 | } 44 | 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/java/ReproDetectDump.java: -------------------------------------------------------------------------------- 1 | import java.lang.reflect.Field; 2 | import java.util.ArrayList; 3 | import java.util.Arrays; 4 | import java.util.HashSet; 5 | import java.util.IdentityHashMap; 6 | import java.util.List; 7 | import java.util.Random; 8 | import java.util.Set; 9 | import java.util.TreeMap; 10 | 11 | /** 12 | * In order to run on jdk11+ you need to add these JVM args: 13 | *
 14 |  * --add-opens java.base/java.util=ALL-UNNAMED
 15 |  * 
16 | * 17 | * Need to run many experiments in order to reproduce the issue with so few entries. 18 | */ 19 | public class ReproDetectDump { 20 | 21 | 22 | public static void main(String[] args) throws Exception { 23 | 24 | final int numThreads; 25 | if (args.length >= 1) { 26 | numThreads = Integer.parseInt(args[0]); 27 | } else { 28 | numThreads = 3; 29 | } 30 | 31 | final int numUpdates; 32 | if (args.length >= 2) { 33 | numUpdates = Integer.parseInt(args[1]); 34 | } else { 35 | numUpdates = 4; 36 | } 37 | 38 | final int maxVal; 39 | if (args.length >= 3) { 40 | maxVal = Integer.parseInt(args[2]); 41 | } else { 42 | maxVal = 30; 43 | } 44 | 45 | final int numExperiments; 46 | if (args.length >= 4) { 47 | numExperiments = Integer.parseInt(args[3]); 48 | } else { 49 | numExperiments = 1000; 50 | } 51 | 52 | for (int experiment = 1; experiment <= numExperiments; experiment++) { 53 | final TreeMap treeMap = new TreeMap<>(); 54 | 55 | 56 | List threads = new ArrayList<>(); 57 | for (int i = 0; i < numThreads; i++) { 58 | threads.add(new Thread(() -> { 59 | Random random = new Random(); 60 | for (int j = 0; j < numUpdates; j++) { 61 | try { 62 | treeMap.put(random.nextInt(maxVal), random.nextInt(1000)); 63 | } catch (NullPointerException e) { 64 | // let it keep going so we can reproduce the issue. 65 | } 66 | } 67 | })); 68 | } 69 | 70 | DetectorThread detectorThread = new DetectorThread(treeMap, threads); 71 | 72 | for (Thread thread : threads) { 73 | thread.start(); 74 | } 75 | detectorThread.start(); 76 | 77 | for (Thread thread : threads) { 78 | thread.join(); 79 | } 80 | 81 | detectorThread.interrupt(); 82 | } 83 | } 84 | 85 | } 86 | 87 | class DetectorThread extends Thread { 88 | private final TreeMap treeMap; 89 | private final List threadsToStop; 90 | private static final Field treeMapRootField; 91 | private static final Field treeMapEntryLeft; 92 | private static final Field treeMapEntryRight; 93 | private static final Field treeMapEntryKey; 94 | 95 | static { 96 | try { 97 | treeMapRootField = TreeMap.class.getDeclaredField("root"); 98 | treeMapRootField.setAccessible(true); 99 | 100 | Class treeMapEntryClass = Arrays.stream(TreeMap.class.getDeclaredClasses()) 101 | .filter(clazz -> "java.util.TreeMap$Entry".equals(clazz.getName())) 102 | .findAny() 103 | .get(); 104 | 105 | treeMapEntryLeft = treeMapEntryClass.getDeclaredField("left"); 106 | treeMapEntryLeft.setAccessible(true); 107 | treeMapEntryRight = treeMapEntryClass.getDeclaredField("right"); 108 | treeMapEntryRight.setAccessible(true); 109 | treeMapEntryKey = treeMapEntryClass.getDeclaredField("key"); 110 | treeMapEntryKey.setAccessible(true); 111 | 112 | } catch (NoSuchFieldException e) { 113 | throw new RuntimeException(e); 114 | } 115 | } 116 | 117 | public DetectorThread( 118 | TreeMap treeMap, 119 | List threadsToStop 120 | ) { 121 | this.treeMap = treeMap; 122 | this.threadsToStop = threadsToStop; 123 | } 124 | 125 | private boolean isLoopDetected() throws Exception { 126 | return isLoopDetected(treeMapRootField.get(treeMap), new IdentityHashMap<>()); 127 | } 128 | 129 | /** 130 | * DFS to detect loop 131 | */ 132 | private boolean isLoopDetected(Object treeMapEntry, IdentityHashMap visited) throws Exception { 133 | if (treeMapEntry == null) { 134 | return false; 135 | } 136 | 137 | if (visited.containsKey(treeMapEntry)) { 138 | return true; 139 | } 140 | 141 | visited.put(treeMapEntry, treeMapEntry); 142 | 143 | 144 | Object left = treeMapEntryLeft.get(treeMapEntry); 145 | Object right = treeMapEntryRight.get(treeMapEntry); 146 | return isLoopDetected(left, visited) 147 | || isLoopDetected(right, visited); 148 | } 149 | 150 | @Override 151 | public void run() { 152 | try { 153 | while (!isLoopDetected()) { 154 | if (this.isInterrupted()) { 155 | return; 156 | } 157 | } 158 | System.out.println("Loop detected"); 159 | new TreeMapExplorer(treeMap).print(); 160 | for (Thread thread : threadsToStop) { 161 | thread.stop(); 162 | } 163 | } catch (Exception e) { 164 | e.printStackTrace(); 165 | } 166 | } 167 | } -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/java/SimpleRepro.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | 3 | public class SimpleRepro { 4 | 5 | public static void main(String[] args) throws Exception { 6 | 7 | final int numThreads; 8 | if (args.length >= 1) { 9 | numThreads = Integer.parseInt(args[0]); 10 | } else { 11 | numThreads = 5; 12 | } 13 | 14 | final int numUpdates; 15 | if (args.length >= 2) { 16 | numUpdates = Integer.parseInt(args[1]); 17 | } else { 18 | numUpdates = 1000; 19 | } 20 | 21 | final TreeMap treeMap = new TreeMap<>(); 22 | 23 | List threads = new ArrayList<>(); 24 | for (int i = 0; i < numThreads; i++) { 25 | threads.add(new Thread(() -> { 26 | Random random = new Random(); 27 | for(int j = 0; j < numUpdates; j++) { 28 | try { 29 | treeMap.put(random.nextInt(1000), random.nextInt(1000)); 30 | } catch (NullPointerException e) { 31 | // let it keep going so we can reproduce the issue. 32 | } 33 | } 34 | })); 35 | } 36 | 37 | for (Thread thread : threads) { 38 | thread.start(); 39 | } 40 | 41 | for (Thread thread : threads) { 42 | thread.join(); 43 | } 44 | 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/java/TreeMapExplorer.java: -------------------------------------------------------------------------------- 1 | import java.lang.reflect.Field; 2 | import java.util.Arrays; 3 | import java.util.IdentityHashMap; 4 | import java.util.TreeMap; 5 | 6 | class TreeMapExplorer { 7 | private final TreeMap treeMap; 8 | private static final Field treeMapRootField; 9 | private static final Field treeMapEntryLeft; 10 | private static final Field treeMapEntryRight; 11 | private static final Field treeMapEntryKey; 12 | private static final Field treeMapEntryColor; 13 | 14 | static { 15 | try { 16 | treeMapRootField = TreeMap.class.getDeclaredField("root"); 17 | treeMapRootField.setAccessible(true); 18 | 19 | Class treeMapEntryClass = Arrays.stream(TreeMap.class.getDeclaredClasses()) 20 | .filter(clazz -> "java.util.TreeMap$Entry".equals(clazz.getName())) 21 | .findAny() 22 | .get(); 23 | 24 | treeMapEntryLeft = treeMapEntryClass.getDeclaredField("left"); 25 | treeMapEntryLeft.setAccessible(true); 26 | treeMapEntryRight = treeMapEntryClass.getDeclaredField("right"); 27 | treeMapEntryRight.setAccessible(true); 28 | treeMapEntryKey = treeMapEntryClass.getDeclaredField("key"); 29 | treeMapEntryKey.setAccessible(true); 30 | treeMapEntryColor = treeMapEntryClass.getDeclaredField("color"); 31 | treeMapEntryColor.setAccessible(true); 32 | 33 | } catch (NoSuchFieldException e) { 34 | throw new RuntimeException(e); 35 | } 36 | } 37 | 38 | public TreeMapExplorer(TreeMap treeMap) { 39 | this.treeMap = treeMap; 40 | } 41 | 42 | public void print() throws Exception { 43 | print(treeMapRootField.get(treeMap), "", new IdentityHashMap<>()); 44 | } 45 | 46 | private void print( 47 | Object treeMapEntry, String tabs, IdentityHashMap visited 48 | ) throws Exception { 49 | if (treeMapEntry != null && !visited.containsKey(treeMapEntry)) { 50 | // in order traversal 51 | visited.put(treeMapEntry, treeMapEntry); 52 | print(treeMapEntryLeft.get(treeMapEntry), tabs + " ", visited); 53 | System.out.println(tabs + treeMapEntryKey.get(treeMapEntry) + ":" 54 | + (treeMapEntryColor.getBoolean(treeMapEntry) ? "BLACK" : "RED")); 55 | print(treeMapEntryRight.get(treeMapEntry), tabs + " ", visited); 56 | } else if (treeMapEntry != null && visited.containsKey(treeMapEntry)) { 57 | System.out.println(tabs + treeMapEntryKey.get(treeMapEntry) + ":" 58 | + (treeMapEntryColor.getBoolean(treeMapEntry) ? "BLACK" : "RED") 59 | + " CYCLE" 60 | ); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /tree_map_corruption/java/src/main/protobuf/ReceiptProcessorService.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message AddReceiptRequest { 4 | int32 timestamp = 1; 5 | int32 totalPrice = 2; 6 | } 7 | 8 | message AddReceiptResponse { 9 | } 10 | 11 | service ReceiptProcessorService { 12 | rpc AddReceipt(AddReceiptRequest) returns (AddReceiptResponse); 13 | } -------------------------------------------------------------------------------- /tree_map_corruption/ruby/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Results 3 | 4 | Even after trying all the following, I still wasn't able to reproduce the issue: 5 | 1. adding barrier 6 | 2. flushing output to stdout 7 | 3. increasing the amount of work per thread to 1,000,000 8 | 4. increasing the number of threads to 10 9 | 10 | I believe it might not be able to reproduce the problem due to the Global Interpreter Lock (GIL), 11 | and how it limits when threads can switch preventing such an interleaving of threads that cause an infinite loop. 12 | 13 | ``` 14 | ruby % ruby simple_repo.rb 15 | 1: waiting for other threads to start 16 | 2: waiting for other threads to start 17 | 3: waiting for other threads to start 18 | 4: waiting for other threads to start 19 | 5: waiting for other threads to start 20 | 6: waiting for other threads to start 21 | 7: waiting for other threads to start 22 | 8: waiting for other threads to start 23 | 9: waiting for other threads to start 24 | 10: waiting for other threads to start 25 | 10: started 26 | 1: started 27 | 2: started 28 | 3: started 29 | 4: started 30 | 5: started 31 | 6: started 32 | 7: started 33 | 8: started 34 | 9: started 35 | 1: finished 36 | 10: finished 37 | 2: finished 38 | 3: finished 39 | 4: finished 40 | 5: finished 41 | 7: finished 42 | 6: finished 43 | 8: finished 44 | 9: finished 45 | Done 46 | 47 | ``` 48 | 49 | # Running 50 | 51 | ## Using macports 52 | 53 | ``` 54 | sudo port install ruby 55 | sudo port install ruby34 56 | sudo port select --set ruby ruby34 57 | ruby --version 58 | ruby 2.6.10p210 (2022-04-12 revision 67958) [universal.x86_64-darwin20] 59 | # reopen terminal 60 | ruby --version 61 | ruby 3.4.1 (2024-12-25 revision 48d4efcb85) +PRISM [x86_64-darwin20] 62 | 63 | 64 | sudo gem install kanwei-algorithms 65 | sudo gem install concurrent-ruby 66 | ``` 67 | 68 | ## Using Mac's ruby doesn't work: 69 | ``` 70 | sudo gem install kanwei-algorithms 71 | 72 | 73 | Fetching kanwei-algorithms-0.2.0.gem 74 | Building native extensions. This could take a while... 75 | ERROR: Error installing kanwei-algorithms: 76 | ERROR: Failed to build gem native extension. 77 | 78 | current directory: 79 | /Library/Ruby/Gems/2.6.0/gems/kanwei-algorithms-0.2.0/ext/containers/deque 80 | /System/Library/Frameworks/Ruby.framework/Versions/2.6/usr/bin/ruby -I 81 | /System/Library/Frameworks/Ruby.framework/Versions/2.6/usr/lib/ruby/2.6.0 -r 82 | ./siteconf20250202-72756-eu9zss.rb extconf.rb 83 | creating Makefile 84 | 85 | current directory: 86 | /Library/Ruby/Gems/2.6.0/gems/kanwei-algorithms-0.2.0/ext/containers/deque 87 | make "DESTDIR=" clean 88 | 89 | current directory: 90 | /Library/Ruby/Gems/2.6.0/gems/kanwei-algorithms-0.2.0/ext/containers/deque 91 | make "DESTDIR=" 92 | make: *** No rule to make target 93 | `/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Frameworks/Ruby.framework/Versions/2.6/usr/include/ruby-2.6.0/universal-darwin20/ruby/config.h', 94 | needed by `deque.o'. Stop. 95 | 96 | make failed, exit code 2 97 | 98 | Gem files will remain installed in 99 | /Library/Ruby/Gems/2.6.0/gems/kanwei-algorithms-0.2.0 for inspection. 100 | Results logged to 101 | /Library/Ruby/Gems/2.6.0/extensions/universal-darwin-20/2.6.0/kanwei-algorithms-0.2.0/gem_make.out 102 | 103 | ``` 104 | 105 | > 106 | https://github.com/github-linguist/linguist/issues/5147 107 | 108 | -------------------------------------------------------------------------------- /tree_map_corruption/ruby/simple_repo.rb: -------------------------------------------------------------------------------- 1 | require 'algorithms' 2 | require 'concurrent' 3 | require 'thread' 4 | 5 | include Containers 6 | 7 | num_threads = (ARGV[0] || 10).to_i 8 | num_updates = (ARGV[1] || 10000000).to_i 9 | barrier = Concurrent::CyclicBarrier.new(num_threads) 10 | 11 | tree_map = RBTreeMap.new 12 | 13 | threads = [] 14 | 15 | for thread_id in 1..num_threads 16 | thread = Thread.new(thread_id) do |my_thread_id| 17 | puts "#{my_thread_id}: waiting for other threads to start" 18 | $stdout.flush 19 | barrier.wait 20 | puts "#{my_thread_id}: started" 21 | $stdout.flush 22 | random = Random.new 23 | for update_num in 1..num_updates do 24 | begin 25 | tree_map[random.rand(1000)] = random.rand(1000) 26 | rescue => e 27 | puts "Exception: #{e.message}" 28 | $stdout.flush 29 | end 30 | end 31 | puts "#{my_thread_id}: finished" 32 | $stdout.flush 33 | end 34 | threads.push(thread) 35 | end 36 | 37 | for thread in threads do 38 | thread.join 39 | end 40 | 41 | puts "Done" 42 | -------------------------------------------------------------------------------- /tree_map_corruption/rust/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /tree_map_corruption/rust/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "byteorder" 7 | version = "1.5.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 10 | 11 | [[package]] 12 | name = "cfg-if" 13 | version = "1.0.0" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 16 | 17 | [[package]] 18 | name = "getrandom" 19 | version = "0.2.15" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" 22 | dependencies = [ 23 | "cfg-if", 24 | "libc", 25 | "wasi", 26 | ] 27 | 28 | [[package]] 29 | name = "libc" 30 | version = "0.2.169" 31 | source = "registry+https://github.com/rust-lang/crates.io-index" 32 | checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" 33 | 34 | [[package]] 35 | name = "ppv-lite86" 36 | version = "0.2.20" 37 | source = "registry+https://github.com/rust-lang/crates.io-index" 38 | checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" 39 | dependencies = [ 40 | "zerocopy", 41 | ] 42 | 43 | [[package]] 44 | name = "proc-macro2" 45 | version = "1.0.93" 46 | source = "registry+https://github.com/rust-lang/crates.io-index" 47 | checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" 48 | dependencies = [ 49 | "unicode-ident", 50 | ] 51 | 52 | [[package]] 53 | name = "quote" 54 | version = "1.0.38" 55 | source = "registry+https://github.com/rust-lang/crates.io-index" 56 | checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" 57 | dependencies = [ 58 | "proc-macro2", 59 | ] 60 | 61 | [[package]] 62 | name = "rand" 63 | version = "0.8.5" 64 | source = "registry+https://github.com/rust-lang/crates.io-index" 65 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 66 | dependencies = [ 67 | "libc", 68 | "rand_chacha", 69 | "rand_core", 70 | ] 71 | 72 | [[package]] 73 | name = "rand_chacha" 74 | version = "0.3.1" 75 | source = "registry+https://github.com/rust-lang/crates.io-index" 76 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 77 | dependencies = [ 78 | "ppv-lite86", 79 | "rand_core", 80 | ] 81 | 82 | [[package]] 83 | name = "rand_core" 84 | version = "0.6.4" 85 | source = "registry+https://github.com/rust-lang/crates.io-index" 86 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 87 | dependencies = [ 88 | "getrandom", 89 | ] 90 | 91 | [[package]] 92 | name = "rbtree" 93 | version = "0.2.0" 94 | source = "registry+https://github.com/rust-lang/crates.io-index" 95 | checksum = "014d09f9f4dbb178240d78610a8ddbc18c57120b62f182fa859e0dd02b03708c" 96 | 97 | [[package]] 98 | name = "rust" 99 | version = "0.1.0" 100 | dependencies = [ 101 | "rand", 102 | "rbtree", 103 | ] 104 | 105 | [[package]] 106 | name = "syn" 107 | version = "2.0.98" 108 | source = "registry+https://github.com/rust-lang/crates.io-index" 109 | checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" 110 | dependencies = [ 111 | "proc-macro2", 112 | "quote", 113 | "unicode-ident", 114 | ] 115 | 116 | [[package]] 117 | name = "unicode-ident" 118 | version = "1.0.16" 119 | source = "registry+https://github.com/rust-lang/crates.io-index" 120 | checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" 121 | 122 | [[package]] 123 | name = "wasi" 124 | version = "0.11.0+wasi-snapshot-preview1" 125 | source = "registry+https://github.com/rust-lang/crates.io-index" 126 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 127 | 128 | [[package]] 129 | name = "zerocopy" 130 | version = "0.7.35" 131 | source = "registry+https://github.com/rust-lang/crates.io-index" 132 | checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" 133 | dependencies = [ 134 | "byteorder", 135 | "zerocopy-derive", 136 | ] 137 | 138 | [[package]] 139 | name = "zerocopy-derive" 140 | version = "0.7.35" 141 | source = "registry+https://github.com/rust-lang/crates.io-index" 142 | checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" 143 | dependencies = [ 144 | "proc-macro2", 145 | "quote", 146 | "syn", 147 | ] 148 | -------------------------------------------------------------------------------- /tree_map_corruption/rust/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rust" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | rbtree = "0.2.0" 8 | rand = "0.8" 9 | -------------------------------------------------------------------------------- /tree_map_corruption/rust/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ``` 6 | error[E0596]: cannot borrow data in an `Arc` as mutable 7 | --> src/main.rs:32:17 8 | | 9 | 32 | tree.insert(key, value); 10 | | ^^^^ cannot borrow as mutable 11 | 12 | ``` 13 | -------------------------------------------------------------------------------- /tree_map_corruption/rust/src/main.rs: -------------------------------------------------------------------------------- 1 | use rbtree::RBTree; 2 | use std::sync::Arc; 3 | use std::thread; 4 | use rand::Rng; 5 | 6 | fn main() { 7 | let args: Vec = std::env::args().collect(); 8 | 9 | let num_threads = if args.len() >= 2 { 10 | args[1].parse::().unwrap() 11 | } else { 12 | 5 13 | }; 14 | 15 | let num_updates = if args.len() >= 3 { 16 | args[2].parse::().unwrap() 17 | } else { 18 | 1000 19 | }; 20 | 21 | let tree = Arc::new(RBTree::new()); 22 | 23 | let mut handles = vec![]; 24 | 25 | for _ in 0..num_threads { 26 | let tree = Arc::clone(&tree); 27 | let handle = thread::spawn(move || { 28 | let mut rng = rand::thread_rng(); 29 | for _ in 0..num_updates { 30 | let key = rng.gen_range(0..1000); 31 | let value = rng.gen_range(0..1000); 32 | tree.insert(key, value); 33 | } 34 | }); 35 | handles.push(handle); 36 | } 37 | 38 | for handle in handles { 39 | handle.join().unwrap(); 40 | } 41 | } 42 | --------------------------------------------------------------------------------