├── release ├── .gitignore ├── publish-javadoc └── README.md ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── .gitignore ├── src ├── main │ └── java │ │ └── com │ │ └── swrve │ │ └── ratelimitedlogger │ │ ├── CounterMetric.java │ │ ├── package-info.java │ │ ├── Stopwatch.java │ │ ├── RateLimitedLogBuilder.java │ │ ├── Level.java │ │ ├── Registry.java │ │ ├── RateLimitedLogWithPattern.java │ │ ├── LogWithPatternAndLevel.java │ │ └── RateLimitedLog.java └── test │ └── java │ └── com │ └── swrve │ └── ratelimitedlogger │ ├── AtExitTest.java │ ├── ManyThreadsTest.java │ ├── MockLogger.java │ └── RateLimitedLogTest.java ├── CHANGELOG ├── jmh-tests ├── src │ └── main │ │ └── java │ │ └── com │ │ └── swrve │ │ └── ratelimitedlogger │ │ └── benchmarks │ │ ├── BenchWithStringKey.java │ │ ├── BenchLogWithPatternAndLevel.java │ │ └── BenchRateLimitedLogWithPattern.java ├── README.md └── pom.xml ├── .circleci └── config.yml ├── gradlew.bat ├── gradlew ├── README.md └── LICENSE /release/.gitignore: -------------------------------------------------------------------------------- 1 | signing 2 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | sonatypeUsername=override_for_packaging 2 | sonatypePassword=override_for_packaging 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Swrve/rate-limited-logger/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.swp 3 | *.swo 4 | .idea 5 | .gradle 6 | *.iml 7 | hs_err_pid* 8 | build 9 | jmh-tests/target 10 | release-signing 11 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Feb 10 22:27:54 IST 2015 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-bin.zip 7 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/CounterMetric.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | /** 4 | * An interface used to implement the target for RateLimitedLogWithPattern#withMetrics(), allowing 5 | * callers to provide their own metric-recording implementation. 6 | */ 7 | public interface CounterMetric { 8 | 9 | /** 10 | * Increment the value of the named metric @param metricName by 1. 11 | */ 12 | void increment(String metricName); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * an SLF4J-compatible, simple, fluent API for rate-limited logging in Java; start with {@link com.swrve.ratelimitedlogger.RateLimitedLog}. 3 | */ 4 | 5 | @ReturnValuesAreNonnullByDefault 6 | @DefaultAnnotation(NonNull.class) 7 | @DefaultAnnotationForParameters(NonNull.class) 8 | package com.swrve.ratelimitedlogger; 9 | 10 | import edu.umd.cs.findbugs.annotations.DefaultAnnotation; 11 | import edu.umd.cs.findbugs.annotations.DefaultAnnotationForParameters; 12 | import edu.umd.cs.findbugs.annotations.NonNull; 13 | import edu.umd.cs.findbugs.annotations.ReturnValuesAreNonnullByDefault; 14 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/Stopwatch.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | /** 6 | * Simple stopwatch implementation to avoid Guava dependency 7 | */ 8 | public class Stopwatch { 9 | 10 | private long startTime; 11 | 12 | public Stopwatch() { 13 | startTime = System.nanoTime(); 14 | } 15 | 16 | public Stopwatch(long startTime) { 17 | this.startTime = startTime; 18 | } 19 | 20 | public void start() { 21 | startTime = System.nanoTime(); 22 | } 23 | 24 | public long elapsedTime(TimeUnit timeUnit) { 25 | return timeUnit.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/com/swrve/ratelimitedlogger/AtExitTest.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.junit.Test; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.time.Duration; 8 | 9 | public class AtExitTest { 10 | private static final Logger logger = LoggerFactory.getLogger(AtExitTest.class); 11 | private static final RateLimitedLog rateLimitedLog = RateLimitedLog 12 | .withRateLimit(logger) 13 | .maxRate(20) 14 | .every(Duration.ofMinutes(1)) 15 | .build(); 16 | 17 | // test for https://github.com/Swrve/rate-limited-logger/issues/11 18 | @Test 19 | public void suppressionsOutputPriorToExit() throws InterruptedException { 20 | for (int i = 0; i < 90; i++) { 21 | rateLimitedLog.info("Testing: {}", i); 22 | } 23 | rateLimitedLog.info("End"); 24 | Thread.sleep(10); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 2 | == 2.0.2 == 3 | 4 | * Avert risk of GC pressure caused by long accidentally-interpolated log strings. 5 | 6 | * Switch from (obsolete) FindBugs to SpotBugs. 7 | 8 | * Switch from JSR-305 javax findbugs/spotbugs annotations back to the originals, 9 | due to licensing issues with the javax namespace. 10 | 11 | 12 | == 2.0.1 == 13 | 14 | * add a shutdownHook to ensure accumulated suppressions are flushed prior to process exit. 15 | 16 | * update Gradle version; thanks to Pine Muzine. 17 | 18 | 19 | == 2.0.0 == 20 | 21 | * Java 8 support, no Guava/Joda-Time dependencies; thanks to Fabien Comte . 22 | 23 | 24 | == 1.1.0 == 25 | 26 | * Support level-based suppression; thanks to Oleg Iakovlev. 27 | 28 | * Change source compatibility in build.gradle to Java 1.6; thanks to Oleg Iakovlev. 29 | 30 | * CircleCI configs to build pull requests and collect JUnit results; thanks to Marc O'Morain. 31 | 32 | * Implement SLF4J Logger interface; thanks to Robert Clancy. 33 | 34 | 35 | == 1.0 == 36 | 37 | * Initial public release. 38 | -------------------------------------------------------------------------------- /release/publish-javadoc: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # release/publish-javadoc -- builds and pushes the generated javadoc to the Github Pages site 4 | 5 | set -exu 6 | 7 | ./gradlew clean 8 | ./gradlew javadoc jar 9 | 10 | version=$(ls build/libs/rate-limited-logger-*.jar \ 11 | | sed -e 's/^.*logger-//' -e 's/.jar//') 12 | 13 | # Annoying javadoc bug: it'll include these annotations in the output page! remove them 14 | sed -i '' \ 15 | -e 's/@ParametersAreNonnullByDefault//' \ 16 | -e 's/@DefaultAnnotation.value=javax.annotation.Nonnull.class.//' \ 17 | build/docs/javadoc/com/swrve/ratelimitedlogger/package-summary.html 18 | 19 | rm -rf build/gh-pages 20 | git clone --quiet --branch=gh-pages git@github.com:Swrve/rate-limited-logger.git build/gh-pages 21 | cd build/gh-pages 22 | 23 | # ensure we don't keep any cruft around from previous builds of this version's javadoc 24 | [ -d javadoc/$version ] && git rm -rf javadoc/$version 25 | 26 | mkdir -p javadoc 27 | cp -R ../docs/javadoc javadoc/$version 28 | 29 | git add -f javadoc 30 | git commit -m "Rebuild javadoc" 31 | git push -fq origin gh-pages 32 | 33 | -------------------------------------------------------------------------------- /jmh-tests/src/main/java/com/swrve/ratelimitedlogger/benchmarks/BenchWithStringKey.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger.benchmarks; 2 | 3 | import com.swrve.ratelimitedlogger.RateLimitedLog; 4 | import org.openjdk.jmh.annotations.*; 5 | 6 | import java.time.Duration; 7 | import java.util.concurrent.TimeUnit; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | @Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS ) 12 | @Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS ) 13 | @State(Scope.Benchmark) 14 | public class BenchWithStringKey { 15 | private static final Logger logger = LoggerFactory.getLogger(BenchWithStringKey.class); 16 | private static final RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 17 | .maxRate(1).every(Duration.ofSeconds(1000)) 18 | .build(); 19 | 20 | @Setup 21 | public void prepare() { 22 | // simulate a bunch of unimportant log lines (to fill out the registry) 23 | for (int i = 0; i < 100; i++) { 24 | rateLimitedLog.info("unused_" + i); 25 | } 26 | } 27 | 28 | @Benchmark 29 | @BenchmarkMode(Mode.SampleTime) 30 | @OutputTimeUnit(TimeUnit.NANOSECONDS) 31 | public void testMethod() { 32 | rateLimitedLog.info("test"); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # https://circleci.com/docs/2.0/configuration-reference/ 2 | version: 2.1 3 | 4 | jobs: 5 | build: 6 | 7 | environment: 8 | # Configure the JVM and Gradle to avoid OOM errors 9 | _JAVA_OPTIONS: "-Xmx2g" 10 | GRADLE_OPTS: "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2" 11 | 12 | docker: 13 | - image: cimg/openjdk:11.0.9 14 | 15 | steps: 16 | - checkout 17 | - restore_cache: 18 | key: v1-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} 19 | - restore_cache: 20 | key: v1-gradle-cache-{{ checksum "build.gradle" }} 21 | - run: 22 | name: Run tests 23 | command: ./gradlew test 24 | - save_cache: 25 | paths: 26 | - ~/.gradle/wrapper 27 | key: v1-gradle-wrapper-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }} 28 | - save_cache: 29 | paths: 30 | - ~/.gradle/caches 31 | key: v1-gradle-cache-{{ checksum "build.gradle" }} 32 | - store_test_results: 33 | path: build/test-results/test 34 | - store_artifacts: 35 | path: build/test-results/test 36 | when: always 37 | - run: 38 | name: Assemble JAR 39 | command: ./gradlew assemble 40 | - store_artifacts: 41 | path: build/libs 42 | -------------------------------------------------------------------------------- /jmh-tests/src/main/java/com/swrve/ratelimitedlogger/benchmarks/BenchLogWithPatternAndLevel.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger.benchmarks; 2 | 3 | import com.swrve.ratelimitedlogger.*; 4 | import com.swrve.ratelimitedlogger.Level; 5 | import org.openjdk.jmh.annotations.*; 6 | 7 | import java.time.Duration; 8 | import java.util.concurrent.TimeUnit; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | @Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS ) 13 | @Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS ) 14 | @State(Scope.Benchmark) 15 | public class BenchLogWithPatternAndLevel { 16 | private static final Logger logger = LoggerFactory.getLogger(BenchLogWithPatternAndLevel.class); 17 | private static final RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 18 | .maxRate(1).every(Duration.ofSeconds(1000)) 19 | .build(); 20 | 21 | private LogWithPatternAndLevel testMessage; 22 | 23 | @Setup 24 | public void prepare() { 25 | // simulate a bunch of unimportant log lines (to fill out the registry) 26 | for (int i = 0; i < 100; i++) { 27 | rateLimitedLog.info("unused_" + i); 28 | } 29 | testMessage = rateLimitedLog.get("test", Level.INFO); 30 | } 31 | 32 | @Benchmark 33 | @BenchmarkMode(Mode.SampleTime) 34 | @OutputTimeUnit(TimeUnit.NANOSECONDS) 35 | public void testMethod() { 36 | testMessage.log(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /jmh-tests/src/main/java/com/swrve/ratelimitedlogger/benchmarks/BenchRateLimitedLogWithPattern.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger.benchmarks; 2 | 3 | import com.swrve.ratelimitedlogger.RateLimitedLog; 4 | import com.swrve.ratelimitedlogger.RateLimitedLogWithPattern; 5 | import org.openjdk.jmh.annotations.*; 6 | 7 | import java.time.Duration; 8 | import java.util.concurrent.TimeUnit; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | @Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS ) 13 | @Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS ) 14 | @State(Scope.Benchmark) 15 | public class BenchRateLimitedLogWithPattern { 16 | private static final Logger logger = LoggerFactory.getLogger(BenchRateLimitedLogWithPattern.class); 17 | private static final RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 18 | .maxRate(1).every(Duration.ofSeconds(1000)) 19 | .build(); 20 | 21 | private RateLimitedLogWithPattern testMessage; 22 | 23 | @Setup 24 | public void prepare() { 25 | // simulate a bunch of unimportant log lines (to fill out the registry) 26 | for (int i = 0; i < 100; i++) { 27 | rateLimitedLog.info("unused_" + i); 28 | } 29 | testMessage = rateLimitedLog.get("test"); 30 | } 31 | 32 | @Benchmark 33 | @BenchmarkMode(Mode.SampleTime) 34 | @OutputTimeUnit(TimeUnit.NANOSECONDS) 35 | public void testMethod() { 36 | testMessage.info(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /release/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## How to push a new release 3 | 4 | ## Step 0: check the version numbers! 5 | 6 | Check README.md and build.gradle; the correct version number should 7 | crop up in several locations. 8 | 9 | 10 | ## Step 1: regenerate javadocs 11 | 12 | Run "release/publish-javadoc" to regenerate the javadoc website. 13 | Note: this is ok to run at any time before release, assuming the 14 | release number in build.gradle has already been updated to point 15 | at the next release number. 16 | 17 | 18 | ## Step 2: Sign the JARs 19 | 20 | Get hold of the release-signing secrets. You need the gradle.properties 21 | and the gnupg directory which appear in the "release/signing" subdirectory. 22 | 23 | Run: 24 | 25 | cp release/signing/gradle.properties ~/.gradle/gradle.properties 26 | ./gradlew signArchives 27 | 28 | Verify that this step was not "SKIPPED" -- this will happen if gradle 29 | thinks it is a snapshot release, but for a genuine release it should 30 | always be signed. 31 | 32 | Warning: do NOT check in the "signing" dir! It contains secret key data. 33 | 34 | 35 | ## Step 3: Uploading new signed JARs 36 | 37 | Run 38 | 39 | ./gradlew uploadArchives 40 | 41 | Go to https://oss.sonatype.org/#stagingRepositories and log in using 42 | the credentials in release/signing/gradle.properties . 43 | 44 | Scroll to the bottom of the list looking for a task starting with 45 | "comswrve". Tick it then hit the "Close" button and enter a brief 46 | message indicating that it's a release. 47 | 48 | After a minute or so (keep hitting Refresh), you should be able to then hit the 49 | "Release" button. 50 | 51 | Warning: again, do NOT check in the "signing" dir! It contains secret key data. 52 | 53 | 54 | ## Step 3: Tag 55 | 56 | git tag -a release-N.N.N -m 'new release' 57 | git push origin release-N.N.N 58 | 59 | 60 | ## See Also / Links 61 | 62 | http://zserge.com/blog/gradle-maven-publish.html 63 | -------------------------------------------------------------------------------- /src/test/java/com/swrve/ratelimitedlogger/ManyThreadsTest.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.junit.Test; 4 | 5 | import java.time.Duration; 6 | import java.util.concurrent.ExecutorService; 7 | import java.util.concurrent.Executors; 8 | import java.util.concurrent.TimeUnit; 9 | import java.util.concurrent.atomic.AtomicBoolean; 10 | 11 | import static org.hamcrest.CoreMatchers.*; 12 | import static org.junit.Assert.assertThat; 13 | 14 | public class ManyThreadsTest { 15 | 16 | @Test 17 | public void manyThreads() throws InterruptedException { 18 | MockLogger logger = new MockLogger(); 19 | 20 | final RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 21 | .maxRate(1).every(Duration.ofMillis(100)) 22 | .build(); 23 | 24 | final AtomicBoolean done = new AtomicBoolean(false); 25 | 26 | assertThat(logger.infoMessageCount, equalTo(0)); 27 | 28 | ExecutorService exec = Executors.newFixedThreadPool(10); 29 | for (int thread = 0; thread < 10; thread++) { 30 | exec.submit(() -> { 31 | while (!done.get()) { 32 | for (int i = 0; i < 1000; i++) { 33 | rateLimitedLog.info("manyThreads {}", Thread.currentThread().getId()); 34 | } 35 | } 36 | }); 37 | } 38 | 39 | for (int sec = 0; sec < 10; sec++) { 40 | Thread.sleep(100L); 41 | logger.info("slept for one sec"); 42 | } 43 | 44 | done.set(true); 45 | exec.shutdown(); 46 | exec.awaitTermination(60, TimeUnit.SECONDS); 47 | 48 | // now, test that the background suppression-logging thread is still working; the bug previously 49 | // was that a precondition failure had crashed it 50 | int c = logger.infoMessageCount; 51 | Thread.sleep(200L); 52 | for (int i = 0; i < 10; i++) { 53 | rateLimitedLog.info("manyThreads 2"); 54 | } 55 | Thread.sleep(200L); 56 | 57 | // ensure that "similar messages suppressed" took place 58 | assertThat(logger.infoMessageCount, not(equalTo(c + 10))); 59 | assertThat(logger.getInfoLastMessage().get(), startsWith("(suppressed ")); 60 | 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /jmh-tests/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This is a set of JMH microbenchmarks for RateLimitedLogger. 4 | 5 | 6 | ## Building and Running 7 | 8 | run: 9 | 10 | ``` 11 | ( cd .. ; ./gradlew clean jar ) 12 | mvn install:install-file -DgroupId=com.swrve -Dpackaging=jar \ 13 | -DartifactId=rate-limited-logger -DgeneratePom=true \ 14 | -Dversion=99.9 -Dfile=../build/libs/rate-limited-logger-1.1.jar \ 15 | && mvn clean install && java -jar target/benchmarks.jar 16 | ``` 17 | 18 | 19 | ## Last Results 20 | 21 | ``` 22 | Result: 56.023 ?(99.9%) 0.751 ns/op [Average] 23 | Statistics: (min, avg, max) = (0.000, 56.023, 10992.000), stdev = 235.553 24 | Confidence interval (99.9%): [55.272, 56.775] 25 | Samples, N = 1063912 26 | mean = 56.023 ?(99.9%) 0.751 ns/op 27 | min = 0.000 ns/op 28 | p( 0.0000) = 0.000 ns/op 29 | p(50.0000) = 0.000 ns/op 30 | p(90.0000) = 0.000 ns/op 31 | p(95.0000) = 1000.000 ns/op 32 | p(99.0000) = 1000.000 ns/op 33 | p(99.9000) = 1000.000 ns/op 34 | p(99.9900) = 1000.000 ns/op 35 | p(99.9990) = 8992.000 ns/op 36 | p(99.9999) = 10928.598 ns/op 37 | max = 10992.000 ns/op 38 | 39 | Result: 58.433 ?(99.9%) 1.788 ns/op [Average] 40 | Statistics: (min, avg, max) = (0.000, 58.433, 802816.000), stdev = 684.113 41 | Confidence interval (99.9%): [56.644, 60.221] 42 | Samples, N = 1584857 43 | mean = 58.433 ?(99.9%) 1.788 ns/op 44 | min = 0.000 ns/op 45 | p( 0.0000) = 0.000 ns/op 46 | p(50.0000) = 0.000 ns/op 47 | p(90.0000) = 0.000 ns/op 48 | p(95.0000) = 1000.000 ns/op 49 | p(99.0000) = 1000.000 ns/op 50 | p(99.9000) = 1000.000 ns/op 51 | p(99.9900) = 1000.000 ns/op 52 | p(99.9990) = 8992.000 ns/op 53 | p(99.9999) = 374157.196 ns/op 54 | max = 802816.000 ns/op 55 | 56 | Result: 73.518 ?(99.9%) 0.728 ns/op [Average] 57 | Statistics: (min, avg, max) = (0.000, 73.518, 32960.000), stdev = 277.994 58 | Confidence interval (99.9%): [72.789, 74.246] 59 | Samples, N = 1577061 60 | mean = 73.518 ?(99.9%) 0.728 ns/op 61 | min = 0.000 ns/op 62 | p( 0.0000) = 0.000 ns/op 63 | p(50.0000) = 0.000 ns/op 64 | p(90.0000) = 0.000 ns/op 65 | p(95.0000) = 1000.000 ns/op 66 | p(99.0000) = 1000.000 ns/op 67 | p(99.9000) = 1000.000 ns/op 68 | p(99.9900) = 2000.000 ns/op 69 | p(99.9990) = 12000.000 ns/op 70 | p(99.9999) = 31242.663 ns/op 71 | max = 32960.000 ns/op 72 | 73 | 74 | # Run complete. Total time: 00:07:47 75 | 76 | Benchmark Mode Cnt Score Error Units 77 | BenchLogWithPatternAndLevel.testMethod sample 1063912 56.023 ? 0.751 ns/op 78 | BenchRateLimitedLogWithPattern.testMethod sample 1584857 58.433 ? 1.788 ns/op 79 | BenchWithStringKey.testMethod sample 1577061 73.518 ? 0.728 ns/op 80 | ``` 81 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/RateLimitedLogBuilder.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.slf4j.Logger; 4 | 5 | import edu.umd.cs.findbugs.annotations.Nullable; 6 | import net.jcip.annotations.NotThreadSafe; 7 | import java.time.Duration; 8 | import java.util.Objects; 9 | 10 | /** 11 | * Factory to create new RateLimitedLog instances in a fluent Builder style. Start with 12 | * RateLimitedLog.withRateLimit(logger). 13 | */ 14 | @NotThreadSafe 15 | public class RateLimitedLogBuilder { 16 | private final Logger logger; 17 | private final int maxRate; 18 | private final Duration periodLength; 19 | private Stopwatch stopwatch = new Stopwatch(); 20 | private @Nullable CounterMetric stats = null; 21 | 22 | public static class MissingRateAndPeriod { 23 | private final Logger logger; 24 | 25 | MissingRateAndPeriod(Logger logger) { 26 | this.logger = logger; 27 | } 28 | 29 | /** 30 | * Specify the maximum count of logs in every time period. Required. 31 | */ 32 | public MissingPeriod maxRate(int rate) { 33 | return new MissingPeriod(logger, rate); 34 | } 35 | } 36 | 37 | public static class MissingPeriod { 38 | private final Logger logger; 39 | private final int maxRate; 40 | 41 | private MissingPeriod(Logger logger, int rate) { 42 | Objects.requireNonNull(logger); 43 | this.logger = logger; 44 | this.maxRate = rate; 45 | } 46 | 47 | /** 48 | * Specify the time period. Required. 49 | */ 50 | public RateLimitedLogBuilder every(Duration duration) { 51 | Objects.requireNonNull(duration); 52 | return new RateLimitedLogBuilder(logger, maxRate, duration); 53 | } 54 | } 55 | 56 | private RateLimitedLogBuilder(Logger logger, int maxRate, Duration periodLength) { 57 | this.logger = logger; 58 | this.maxRate = maxRate; 59 | this.periodLength = periodLength; 60 | } 61 | 62 | /** 63 | * Specify that the rate-limited logger should compute time using @param stopwatch. 64 | */ 65 | public RateLimitedLogBuilder withStopwatch(Stopwatch stopwatch) { 66 | this.stopwatch = Objects.requireNonNull(stopwatch); 67 | return this; 68 | } 69 | 70 | /** 71 | * Optional: should we record metrics about the call rate using @param stats. Default is not to record metrics 72 | */ 73 | public RateLimitedLogBuilder recordMetrics(CounterMetric stats) { 74 | this.stats = Objects.requireNonNull(stats); 75 | return this; 76 | } 77 | 78 | /** 79 | * @return a fully-built RateLimitedLog matching the requested configuration. 80 | */ 81 | public RateLimitedLog build() { 82 | if (maxRate <= 0) { 83 | throw new IllegalArgumentException("maxRate must be > 0"); 84 | } 85 | if (periodLength.toMillis() <= 0) { 86 | throw new IllegalArgumentException("period must be non-zero"); 87 | } 88 | stopwatch.start(); 89 | return new RateLimitedLog(logger, 90 | new RateLimitedLogWithPattern.RateAndPeriod(maxRate, periodLength), stopwatch, 91 | stats, RateLimitedLog.REGISTRY); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/Level.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.Marker; 5 | 6 | /** 7 | * Our supported logging levels. These match SLF4J. 8 | */ 9 | public enum Level { 10 | TRACE("trace") { 11 | @Override 12 | void log(Logger logger, String msg, Object... arguments) { 13 | logger.trace(msg, arguments); 14 | } 15 | @Override 16 | void log(Logger logger, String msg, Throwable t) { 17 | logger.trace(msg, t); 18 | } 19 | @Override 20 | void log(Logger logger, String msg, Marker marker, Object... arguments) { 21 | logger.trace(marker, msg, arguments); 22 | } 23 | @Override 24 | void log(Logger logger, String msg, Marker marker, Throwable t) { 25 | logger.trace(marker, msg, t); 26 | } 27 | }, 28 | DEBUG("debug") { 29 | @Override 30 | void log(Logger logger, String msg, Object... arguments) { 31 | logger.debug(msg, arguments); 32 | } 33 | @Override 34 | void log(Logger logger, String msg, Throwable t) { 35 | logger.debug(msg, t); 36 | } 37 | @Override 38 | void log(Logger logger, String msg, Marker marker, Object... arguments) { 39 | logger.debug(marker, msg, arguments); 40 | } 41 | @Override 42 | void log(Logger logger, String msg, Marker marker, Throwable t) { 43 | logger.debug(marker, msg, t); 44 | } 45 | }, 46 | INFO("info") { 47 | @Override 48 | void log(Logger logger, String msg, Object... arguments) { 49 | logger.info(msg, arguments); 50 | } 51 | @Override 52 | void log(Logger logger, String msg, Throwable t) { 53 | logger.info(msg, t); 54 | } 55 | @Override 56 | void log(Logger logger, String msg, Marker marker, Object... arguments) { 57 | logger.info(marker, msg, arguments); 58 | } 59 | @Override 60 | void log(Logger logger, String msg, Marker marker, Throwable t) { 61 | logger.info(marker, msg, t); 62 | } 63 | }, 64 | WARN("warn") { 65 | @Override 66 | void log(Logger logger, String msg, Object... arguments) { 67 | logger.warn(msg, arguments); 68 | } 69 | @Override 70 | void log(Logger logger, String msg, Throwable t) { 71 | logger.warn(msg, t); 72 | } 73 | @Override 74 | void log(Logger logger, String msg, Marker marker, Object... arguments) { 75 | logger.warn(marker, msg, arguments); 76 | } 77 | @Override 78 | void log(Logger logger, String msg, Marker marker, Throwable t) { 79 | logger.warn(marker, msg, t); 80 | } 81 | }, 82 | ERROR("error") { 83 | @Override 84 | void log(Logger logger, String msg, Object... arguments) { 85 | logger.error(msg, arguments); 86 | } 87 | @Override 88 | void log(Logger logger, String msg, Throwable t) { 89 | logger.error(msg, t); 90 | } 91 | @Override 92 | void log(Logger logger, String msg, Marker marker, Object... arguments) { 93 | logger.error(marker, msg, arguments); 94 | } 95 | @Override 96 | void log(Logger logger, String msg, Marker marker, Throwable t) { 97 | logger.error(marker, msg, t); 98 | } 99 | }; 100 | 101 | private final String levelName; 102 | 103 | Level(String levelName) { 104 | this.levelName = levelName; 105 | } 106 | 107 | public String getLevelName() { 108 | return levelName; 109 | } 110 | 111 | abstract void log(Logger logger, String msg, Object... arguments); 112 | abstract void log(Logger logger, String msg, Throwable t); 113 | abstract void log(Logger logger, String msg, Marker marker, Object... arguments); 114 | abstract void log(Logger logger, String msg, Marker marker, Throwable t); 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/Registry.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import net.jcip.annotations.ThreadSafe; 7 | import java.time.Duration; 8 | import java.util.Locale; 9 | import java.util.Map; 10 | import java.util.concurrent.*; 11 | import java.util.concurrent.atomic.AtomicLong; 12 | 13 | /** 14 | * Internal registry of LogWithPatternAndLevel objects, allowing periodic resets of their counters. 15 | */ 16 | @ThreadSafe 17 | class Registry { 18 | private static final Logger logger = LoggerFactory.getLogger(Registry.class); 19 | 20 | private final ConcurrentHashMap> registry 21 | = new ConcurrentHashMap<>(); 22 | 23 | private final ThreadFactory threadFactory = new ThreadFactory() { 24 | final AtomicLong count = new AtomicLong(0); 25 | 26 | @Override 27 | public Thread newThread(Runnable r) { 28 | Thread thread = new Thread(r); 29 | thread.setName(String.format(Locale.ROOT, "RateLimitedLogRegistry-%d", count.getAndIncrement())); 30 | thread.setDaemon(true); 31 | return thread; 32 | } 33 | }; 34 | 35 | private final ScheduledExecutorService resetScheduler = Executors.newScheduledThreadPool(1, threadFactory); 36 | 37 | Registry() { 38 | // this will ensure that we will always flush any suppressed logs prior to exiting a process 39 | Runtime.getRuntime().addShutdownHook(new Thread(this::flush)); 40 | } 41 | 42 | /** 43 | * Register a new @param log, with a reset periodicity of @param period. This happens relatively infrequently, 44 | * so synchronization is ok (and safer) 45 | * 46 | * @throws IllegalStateException if we run out of space in the registry for that period. 47 | */ 48 | synchronized void register(LogWithPatternAndLevel log, Duration period) { 49 | 50 | // if we haven't seen this period before, we'll need to add a schedule to the ScheduledExecutorService 51 | // to perform a counter reset with that periodicity, otherwise we can count on the existing schedule 52 | // taking care of it. 53 | boolean needToScheduleReset = false; 54 | 55 | ConcurrentHashMap logLinesForPeriod = registry.get(period); 56 | if (logLinesForPeriod == null) { 57 | needToScheduleReset = true; 58 | logLinesForPeriod = new ConcurrentHashMap<>(); 59 | registry.put(period, logLinesForPeriod); 60 | 61 | } else { 62 | if (logLinesForPeriod.get(log) != null) { 63 | return; // this has already been registered 64 | } 65 | } 66 | logLinesForPeriod.put(log, Boolean.TRUE); 67 | 68 | if (needToScheduleReset) { 69 | final ConcurrentHashMap finalLogLinesForPeriod = logLinesForPeriod; 70 | resetScheduler.scheduleWithFixedDelay(() -> { 71 | try { 72 | resetAllCounters(finalLogLinesForPeriod); 73 | } catch (Exception e) { 74 | //noinspection AccessToStaticFieldLockedOnInstance 75 | logger.warn("failed to reset counters: " + e, e); 76 | // but carry on in the next iteration 77 | } 78 | }, period.toMillis(), period.toMillis(), TimeUnit.MILLISECONDS); 79 | } 80 | } 81 | 82 | private void resetAllCounters(ConcurrentHashMap logLinesForPeriod) { 83 | for (LogWithPatternAndLevel log : logLinesForPeriod.keySet()) { 84 | log.periodicReset(); 85 | } 86 | } 87 | 88 | synchronized void flush() { 89 | for (Map.Entry> 90 | entry : registry.entrySet()) { 91 | 92 | ConcurrentHashMap logLinesForPeriod = entry.getValue(); 93 | resetAllCounters(logLinesForPeriod); 94 | logLinesForPeriod.clear(); 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # For Cygwin, ensure paths are in UNIX format before anything is touched. 46 | if $cygwin ; then 47 | [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 48 | fi 49 | 50 | # Attempt to set APP_HOME 51 | # Resolve links: $0 may be a link 52 | PRG="$0" 53 | # Need this for relative symlinks. 54 | while [ -h "$PRG" ] ; do 55 | ls=`ls -ld "$PRG"` 56 | link=`expr "$ls" : '.*-> \(.*\)$'` 57 | if expr "$link" : '/.*' > /dev/null; then 58 | PRG="$link" 59 | else 60 | PRG=`dirname "$PRG"`"/$link" 61 | fi 62 | done 63 | SAVED="`pwd`" 64 | cd "`dirname \"$PRG\"`/" >&- 65 | APP_HOME="`pwd -P`" 66 | cd "$SAVED" >&- 67 | 68 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 69 | 70 | # Determine the Java command to use to start the JVM. 71 | if [ -n "$JAVA_HOME" ] ; then 72 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 73 | # IBM's JDK on AIX uses strange locations for the executables 74 | JAVACMD="$JAVA_HOME/jre/sh/java" 75 | else 76 | JAVACMD="$JAVA_HOME/bin/java" 77 | fi 78 | if [ ! -x "$JAVACMD" ] ; then 79 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 80 | 81 | Please set the JAVA_HOME variable in your environment to match the 82 | location of your Java installation." 83 | fi 84 | else 85 | JAVACMD="java" 86 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 87 | 88 | Please set the JAVA_HOME variable in your environment to match the 89 | location of your Java installation." 90 | fi 91 | 92 | # Increase the maximum file descriptors if we can. 93 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 94 | MAX_FD_LIMIT=`ulimit -H -n` 95 | if [ $? -eq 0 ] ; then 96 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 97 | MAX_FD="$MAX_FD_LIMIT" 98 | fi 99 | ulimit -n $MAX_FD 100 | if [ $? -ne 0 ] ; then 101 | warn "Could not set maximum file descriptor limit: $MAX_FD" 102 | fi 103 | else 104 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 105 | fi 106 | fi 107 | 108 | # For Darwin, add options to specify how the application appears in the dock 109 | if $darwin; then 110 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 111 | fi 112 | 113 | # For Cygwin, switch paths to Windows format before running java 114 | if $cygwin ; then 115 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 116 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 158 | function splitJvmOpts() { 159 | JVM_OPTS=("$@") 160 | } 161 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 162 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 163 | 164 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 165 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | RateLimitedLogger [![CircleCI badge](https://circleci.com/gh/Swrve/rate-limited-logger.svg?style=svg&circle-token=a2d7a24d30021fc04658b58c24c1758e891e66fc)](https://circleci.com/gh/Swrve/rate-limited-logger) 2 | ======== 3 | 4 | An SLF4J-compatible, simple, fluent API for rate-limited logging in Java. 5 | 6 | Logging is vital for production-ready, operable code; however, in certain 7 | situations, it can be dangerous. It is easy to wipe out throughput of a 8 | performance-critical backend component by several orders of magnitude with disk 9 | I/O in a performance hotspot, caused by a misplaced log call, or input data 10 | changing to something slightly unexpected. 11 | 12 | With RateLimitedLogger, however, this risk is avoided. A RateLimitedLog object 13 | tracks the rate of log message emission, imposes an internal rate limit, and 14 | will efficiently suppress logging if this is exceeded. When a log is 15 | suppressed, at the end of the limit period, another log message is output 16 | indicating how many log lines were suppressed. 17 | 18 | This style of rate limiting is the same as the one used by UNIX syslog; this 19 | means it should be comprehensible, easy to predict, and familiar to many users, 20 | unlike more complex adaptive rate limits. 21 | 22 | RateLimitedLogger wraps your existing SLF4J loggers, so should be easy to plug 23 | into existing Java code. 24 | 25 | 26 | ## Binaries: 27 | 28 | This module is available in the Maven Central repository at 29 | http://search.maven.org/#search%7Cga%7C1%7Ca%3A%22rate-limited-logger%22 30 | 31 | WARNING: version 2.0.0 drops support for Java 6 and 7, and is Java 8-only. It 32 | also requires (minor) code changes to use the java.time classes instead of 33 | Joda-Time. If you still need support for Java 6 or 7, use version 1.1.0 or 34 | earlier. 35 | 36 | Maven: 37 | 38 | ``` 39 | 40 | com.swrve 41 | rate-limited-logger 42 | 2.0.2 43 | 44 | ``` 45 | 46 | Gradle: 47 | 48 | ``` 49 | compile group: 'com.swrve', name: 'rate-limited-logger', version: '2.0.2' 50 | ``` 51 | 52 | Sample code: 53 | 54 | ``` 55 | private static final Logger logger = LoggerFactory.getLogger(getClass()); 56 | 57 | private static final RateLimitedLog rateLimitedLog = RateLimitedLog 58 | .withRateLimit(logger) 59 | .maxRate(5).every(Duration.ofSeconds(10)) 60 | .build(); 61 | ``` 62 | 63 | This will wrap an existing SLF4J Logger object, allowing a max of 5 messages 64 | to be output every 10 seconds, suppressing any more than that. 65 | 66 | 67 | ## More documentation 68 | 69 | Javadoc can be found at http://swrve.github.io/rate-limited-logger/javadoc/2.0.2/ 70 | 71 | 72 | ## Sample output 73 | 74 | ``` 75 | 22:16:03.584 [main] INFO Demo - message 1 76 | 22:16:03.604 [main] INFO Demo - message 2 77 | 22:16:03.634 [main] INFO Demo - message 3 78 | 22:16:04.986 [RateLimitedLogRegistry-0] INFO Demo - (suppressed 39 logs similar to 'message {}' in PT1.0S) 79 | ``` 80 | 81 | ## Interpolation 82 | 83 | Each log message has its own internal rate-limiting AtomicLong counter. In 84 | other words, if you have 2 log messages, you can safely reuse the same 85 | RateLimitedLog object to log both, and a high rate of one will not cause the 86 | other to be suppressed as a side effect. 87 | 88 | However, this means that if you wish to include dynamic, variable data in the 89 | log output, you will need to use SLF4J-style templates, instead of ("foo " + 90 | bar + " baz") string interpolation. For example: 91 | 92 | ``` 93 | rateLimitedLog.info("Just saw an event of type {}: {}", event.getType(), event); 94 | ``` 95 | 96 | "{}" will implicitly invoke an object's toString() method, so toString() does 97 | not need to be called explicitly when logging. (This has obvious performance 98 | benefits, in that those toString() methods will not be called at all once the 99 | rate limits have been exceeded or if the log-level threshold isn't reached.) 100 | 101 | A RateLimitedLog object has a limited capacity for the number of log messages 102 | it'll hold; if over 1000 different strings are used as the message template for 103 | a single RateLimitedLog object, it is assumed that the caller is accidentally 104 | using an already-interpolated string containing variable data in place of the 105 | template, and the current set of logs will be flushed in order to avoid an 106 | OutOfMemory condition. This has a performance impact, but at least it won't 107 | lose data! 108 | 109 | 110 | ## Performance 111 | 112 | In tests using JMH with Java 7 on a 2012 Macbook Pro, using RateLimitedLog with 113 | logger.info("string") ran in, on average, 73 nanoseconds per op, with a P99.99 114 | of 2000 ns/op and a P99.999 of 12000 ns/op, once the rate limit was exceeded. 115 | 116 | Where performance is critical, note that you can obtain a reference to the 117 | LogWithPatternAndLevel object for an individual log template and level, which will 118 | then avoid a ConcurrentHashMap and AtomicReferenceArray lookup: 119 | 120 | ``` 121 | ref = logger.get("string", Level.INFO) 122 | ref.log() 123 | ``` 124 | 125 | Using this approach, the average post-ratelimit time dropped to 56 nanoseconds 126 | per op, with a P99.99 of 1000 ns/op and a P99.999 of 8992 ns/op. 127 | 128 | More details: https://github.com/Swrve/rate-limited-logger/tree/master/jmh-tests 129 | 130 | 131 | ## Thread-Safety 132 | 133 | The RateLimitedLogger library is thread-safe. Under heavy load, though, it is 134 | possible for more log messages to be exceeded than the limit specifies, for a 135 | short period after the limit is exceeded (typically on the order of a duration 136 | of a few milliseconds). 137 | 138 | 139 | ## Dependencies 140 | 141 | All versions are minimum versions -- later versions should also work fine. 142 | 143 | - Java 8 144 | - SLF4J API 1.7.7 145 | - Findbugs Annotations 1.0.0 146 | - Findbugs JSR-305 Annotations 2.0.2 147 | 148 | version 1.1.0 of RateLimitedLogger supports Java 7, but requires Guava 15.0 and 149 | Joda-Time 2.3 in addition to the above. 150 | 151 | 152 | ## License 153 | 154 | (c) Copyright 2014-2018 Swrve Mobile Inc or its licensors. 155 | Distributed under version 2.0 of the Apache License, see "LICENSE". 156 | 157 | 158 | ## Building 159 | 160 | Build all JARs, test, measure coverage: 161 | 162 | ``` 163 | ./gradlew all 164 | ``` 165 | 166 | 167 | ## Credits 168 | 169 | - The Swrve team: http://www.swrve.com/ 170 | - Our dev blog: http://swrveengineering.wordpress.com/ 171 | 172 | - Fabien Comte , for Java 8 support 173 | 174 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/RateLimitedLogWithPattern.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.Marker; 5 | 6 | import edu.umd.cs.findbugs.annotations.Nullable; 7 | import net.jcip.annotations.ThreadSafe; 8 | import java.time.Duration; 9 | import java.util.Objects; 10 | import java.util.concurrent.atomic.AtomicReferenceArray; 11 | 12 | /** 13 | * An individual log pattern. Each object is rate-limited individually but with separation on the log level. 14 | * 15 | * These objects are thread-safe. 16 | */ 17 | @ThreadSafe 18 | public class RateLimitedLogWithPattern { 19 | 20 | private final String message; 21 | private final RateAndPeriod rateAndPeriod; 22 | private final Logger logger; 23 | private final Registry registry; 24 | private final @Nullable CounterMetric stats; 25 | private final Stopwatch stopwatch; 26 | private final AtomicReferenceArray levels; 27 | 28 | RateLimitedLogWithPattern(String message, RateAndPeriod rateAndPeriod, Registry registry, @Nullable CounterMetric stats, Stopwatch stopwatch, Logger logger) { 29 | this.message = message; 30 | this.rateAndPeriod = rateAndPeriod; 31 | this.registry = registry; 32 | this.logger = logger; 33 | this.stats = stats; 34 | this.stopwatch = stopwatch; 35 | this.levels = new AtomicReferenceArray<>(Level.values().length); 36 | } 37 | 38 | /** 39 | * logging APIs. 40 | * 41 | * These can use the SLF4J style of templating to parameterize the Logs. 42 | * See http://www.slf4j.org/api/org/slf4j/helpers/MessageFormatter.html . 43 | * 44 | * 
 45 |      *    rateLimitedLog.info("Just saw an event of type {}: {}", event.getType(), event);
 46 |      *
47 | * 48 | * @param args the varargs list of arguments matching the message template 49 | */ 50 | public void trace(Object... args) { 51 | get(Level.TRACE).log(args); 52 | } 53 | 54 | public void trace(Throwable t) { 55 | get(Level.TRACE).log(t); 56 | } 57 | 58 | public void trace(Marker marker, Object... args) { 59 | get(Level.TRACE).log(marker, args); 60 | } 61 | 62 | public void trace(Marker marker, Throwable t) { 63 | get(Level.TRACE).log(marker, t); 64 | } 65 | 66 | public void debug(Object... args) { 67 | get(Level.DEBUG).log(args); 68 | } 69 | 70 | public void debug(Throwable t) { 71 | get(Level.DEBUG).log(t); 72 | } 73 | 74 | public void debug(Marker marker, Object... args) { 75 | get(Level.DEBUG).log(marker, args); 76 | } 77 | 78 | public void debug(Marker marker, Throwable t) { 79 | get(Level.DEBUG).log(marker, t); 80 | } 81 | 82 | public void info(Object... args) { 83 | get(Level.INFO).log(args); 84 | } 85 | 86 | public void info(Throwable t) { 87 | get(Level.INFO).log(t); 88 | } 89 | 90 | public void info(Marker marker, Object... args) { 91 | get(Level.INFO).log(marker, args); 92 | } 93 | 94 | public void info(Marker marker, Throwable t) { 95 | get(Level.INFO).log(marker, t); 96 | } 97 | 98 | public void warn(Object... args) { 99 | get(Level.WARN).log(args); 100 | } 101 | 102 | public void warn(Throwable t) { 103 | get(Level.WARN).log(t); 104 | } 105 | 106 | public void warn(Marker marker, Object... args) { 107 | get(Level.WARN).log(marker, args); 108 | } 109 | 110 | public void warn(Marker marker, Throwable t) { 111 | get(Level.WARN).log(marker, t); 112 | } 113 | 114 | public void error(Object... args) { 115 | get(Level.ERROR).log(args); 116 | } 117 | 118 | public void error(Throwable t) { 119 | get(Level.ERROR).log(t); 120 | } 121 | 122 | public void error(Marker marker, Object... args) { 123 | get(Level.ERROR).log(marker, args); 124 | } 125 | 126 | public void error(Marker marker, Throwable t) { 127 | get(Level.ERROR).log(marker, t); 128 | } 129 | 130 | /** 131 | * Two RateLimitedLogWithPattern objects are considered equal if their messages match; the 132 | * RateAndPeriods are not significant. 133 | */ 134 | @Override 135 | public boolean equals(@Nullable Object o) { 136 | if (this == o) { 137 | return true; 138 | } 139 | if (o == null || getClass() != o.getClass()) { 140 | return false; 141 | } 142 | return (message.equals(((RateLimitedLogWithPattern) o).message)); 143 | } 144 | 145 | @Override 146 | public int hashCode() { 147 | return message.hashCode(); 148 | } 149 | 150 | /** 151 | * @return a LogWithPatternAndLevel object for the supplied @param level . 152 | * This can be cached and reused by callers in performance-sensitive 153 | * cases to avoid performing two ConcurrentHashMap lookups. 154 | * 155 | * Note that the string is the sole key used, so the same string cannot be reused with differing period 156 | * settings; any periods which differ from the first one used are ignored. 157 | * 158 | * @throws IllegalStateException if we exceed the limit on number of RateLimitedLogWithPattern objects 159 | * in any one period; if this happens, it's probable that an already-interpolated string is 160 | * accidentally being used as a log pattern. 161 | */ 162 | public LogWithPatternAndLevel get(Level level) { 163 | int l = level.ordinal(); 164 | 165 | LogWithPatternAndLevel got = levels.get(l); 166 | if (got != null) { 167 | return got; 168 | } 169 | 170 | // slow path: create a new LogWithPatternAndLevel 171 | LogWithPatternAndLevel newValue = new LogWithPatternAndLevel(message, 172 | level, rateAndPeriod, stats, stopwatch, logger); 173 | 174 | boolean wasSet = levels.compareAndSet(l, null, newValue); 175 | if (!wasSet) { 176 | return Objects.requireNonNull(levels.get(l)); 177 | } else { 178 | // ensure we'll reset the counter once every period 179 | registry.register(newValue, rateAndPeriod.periodLength); 180 | return newValue; 181 | } 182 | } 183 | 184 | public static final class RateAndPeriod { 185 | final int maxRate; 186 | final Duration periodLength; 187 | 188 | public RateAndPeriod(int maxRate, Duration periodLength) { 189 | this.maxRate = maxRate; 190 | this.periodLength = periodLength; 191 | } 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /jmh-tests/pom.xml: -------------------------------------------------------------------------------- 1 | 31 | 32 | 34 | 4.0.0 35 | 36 | org.sample 37 | test 38 | 1.0 39 | jar 40 | 41 | JMH benchmark sample: Java 42 | 43 | 47 | 48 | 49 | 3.0 50 | 51 | 52 | 53 | 54 | org.openjdk.jmh 55 | jmh-core 56 | ${jmh.version} 57 | 58 | 59 | org.openjdk.jmh 60 | jmh-generator-annprocess 61 | ${jmh.version} 62 | provided 63 | 64 | 65 | com.swrve 66 | rate-limited-logger 67 | 99.9 68 | 69 | 70 | ch.qos.logback 71 | logback-classic 72 | 1.1.2 73 | 74 | 75 | 76 | 77 | UTF-8 78 | 1.5.1 79 | 1.8 80 | benchmarks 81 | 82 | 83 | 84 | 85 | 86 | org.apache.maven.plugins 87 | maven-compiler-plugin 88 | 3.1 89 | 90 | ${javac.target} 91 | ${javac.target} 92 | ${javac.target} 93 | 94 | 95 | 96 | org.apache.maven.plugins 97 | maven-shade-plugin 98 | 2.2 99 | 100 | 101 | package 102 | 103 | shade 104 | 105 | 106 | ${uberjar.name} 107 | 108 | 109 | org.openjdk.jmh.Main 110 | 111 | 112 | 113 | 114 | 118 | *:* 119 | 120 | META-INF/*.SF 121 | META-INF/*.DSA 122 | META-INF/*.RSA 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | maven-clean-plugin 135 | 2.5 136 | 137 | 138 | maven-deploy-plugin 139 | 2.8.1 140 | 141 | 142 | maven-install-plugin 143 | 2.5.1 144 | 145 | 146 | maven-jar-plugin 147 | 2.4 148 | 149 | 150 | maven-javadoc-plugin 151 | 2.9.1 152 | 153 | 154 | maven-resources-plugin 155 | 2.6 156 | 157 | 158 | maven-site-plugin 159 | 3.3 160 | 161 | 162 | maven-source-plugin 163 | 2.2.1 164 | 165 | 166 | maven-surefire-plugin 167 | 2.17 168 | 169 | 170 | 171 | 172 | 173 | 174 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/LogWithPatternAndLevel.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.Marker; 5 | 6 | import edu.umd.cs.findbugs.annotations.Nullable; 7 | import net.jcip.annotations.GuardedBy; 8 | import net.jcip.annotations.ThreadSafe; 9 | import java.time.Duration; 10 | import java.util.concurrent.TimeUnit; 11 | import java.util.concurrent.atomic.AtomicLong; 12 | 13 | /** 14 | * An individual log pattern and level - the unit of rate limiting. Each object is rate-limited 15 | * individually. 16 | * 17 | * Thread-safe. 18 | */ 19 | @ThreadSafe 20 | public class LogWithPatternAndLevel { 21 | 22 | private static final long NOT_RATE_LIMITED_YET = 0L; 23 | private static final String RATE_LIMITED_COUNT_SUFFIX = "_rate_limited_log_count"; 24 | 25 | private final String message; 26 | private final Level level; 27 | private final RateLimitedLogWithPattern.RateAndPeriod rateAndPeriod; 28 | private final Logger logger; 29 | private final @Nullable CounterMetric stats; 30 | private final Stopwatch stopwatch; 31 | 32 | /** 33 | * Number of observed logs in the current time period based on the log level. 34 | */ 35 | private final AtomicLong counter = new AtomicLong(0L); // mutable 36 | 37 | /** 38 | * When we exceed the rate limit during a period, we record when. If the rate limit has not been exceeded, the 39 | * magic value of NOT_RATE_LIMITED_YET will be recorded. 40 | */ 41 | private final AtomicLong rateLimitedAt = new AtomicLong(NOT_RATE_LIMITED_YET); // mutable 42 | 43 | LogWithPatternAndLevel(String message, Level level, 44 | RateLimitedLogWithPattern.RateAndPeriod rateAndPeriod, 45 | @Nullable CounterMetric stats, 46 | Stopwatch stopwatch, Logger logger) { 47 | this.message = message; 48 | this.level = level; 49 | this.rateAndPeriod = rateAndPeriod; 50 | this.logger = logger; 51 | this.stats = stats; 52 | this.stopwatch = stopwatch; 53 | } 54 | 55 | /** 56 | * logging APIs. 57 | * 58 | * These can use the SLF4J style of templating to parameterize the Logs. 59 | * See http://www.slf4j.org/api/org/slf4j/helpers/MessageFormatter.html . 60 | * 61 | * 
 62 |      *    rateLimitedLog.info("Just saw an event of type {}: {}", event.getType(), event);
 63 |      *
64 | * 65 | * @param args the varargs list of arguments matching the message template 66 | */ 67 | public void log(Object... args) { 68 | if (!isRateLimited()) { 69 | level.log(logger, message, args); 70 | } 71 | incrementStats(); 72 | } 73 | 74 | public void log(Throwable t) { 75 | if (!isRateLimited()) { 76 | level.log(logger, message, t); 77 | } 78 | incrementStats(); 79 | } 80 | 81 | public void log(Marker marker, Object... args) { 82 | if (!isRateLimited()) { 83 | level.log(logger, message, marker, args); 84 | } 85 | incrementStats(); 86 | } 87 | 88 | public void log(Marker marker, Throwable t) { 89 | if (!isRateLimited()) { 90 | level.log(logger, message, marker, t); 91 | } 92 | incrementStats(); 93 | } 94 | 95 | @SuppressWarnings("BooleanMethodIsAlwaysInverted") 96 | private boolean isRateLimited() { 97 | 98 | // note: this method is not synchronized, for performance. If we exceed the maxRate, we will start checking 99 | // haveExceededLimit, and if that's still false, we enter the synchronized haveJustExceededRateLimit() method. 100 | // 101 | // There is still potential for a race -- the rate of incrementing could be so high that we are already 102 | // over the maxRate by the time the reset thread runs, but the haveJustExceededRateLimit() hasn't yet been 103 | // run in this thread. In this scenario, we will fail to notice that we are over the limit, but when 104 | // the next iteration runs, we will correctly report the correct number of suppressions and the time 105 | // when haveJustExceededRateLimit() eventually got to execute. We will also potentially log a small 106 | // number more lines to the logger than the rate limit allows. 107 | // 108 | long count = counter.incrementAndGet(); 109 | if (count < rateAndPeriod.maxRate) { 110 | return false; 111 | } else if (count >= rateAndPeriod.maxRate && rateLimitedAt.get() == NOT_RATE_LIMITED_YET) { 112 | haveJustExceededRateLimit(); 113 | return false; // we still issue this final log, though 114 | } else { 115 | return true; 116 | } 117 | } 118 | 119 | /** 120 | * Reset the counter and suppression details, if necessary. This is called once every period, by the Registry. 121 | */ 122 | synchronized void periodicReset() { 123 | long whenLimited = rateLimitedAt.getAndSet(NOT_RATE_LIMITED_YET); 124 | if (whenLimited != NOT_RATE_LIMITED_YET) { 125 | reportSuppression(whenLimited); 126 | } 127 | } 128 | 129 | @GuardedBy("this") 130 | private void reportSuppression(long whenLimited) { 131 | long count = counter.get(); 132 | counter.addAndGet(-count); 133 | long numSuppressed = count - rateAndPeriod.maxRate; 134 | if (numSuppressed == 0) { 135 | return; // special case: we hit the rate limit, but did not actually exceed it -- nothing got suppressed, so there's no need to log 136 | } 137 | Duration howLong = Duration.ofMillis(elapsedMsecs()).minusMillis(whenLimited); 138 | level.log(logger, "(suppressed {} logs similar to '{}' in {})", numSuppressed, message, howLong); 139 | } 140 | 141 | private synchronized void haveJustExceededRateLimit() { 142 | rateLimitedAt.set(elapsedMsecs()); 143 | } 144 | 145 | private long elapsedMsecs() { 146 | long elapsed = stopwatch.elapsedTime(TimeUnit.MILLISECONDS); 147 | if (elapsed == NOT_RATE_LIMITED_YET) { 148 | elapsed++; // avoid using the magic value by "rounding up" 149 | } 150 | return elapsed; 151 | } 152 | 153 | /** 154 | * Increment a counter metric called "{level}_rate_limited_log_count", where "{level}" is the log 155 | * level in question. This is still performed even when a log is rate limited, since incrementing 156 | * a counter metric is cheap! 157 | * 158 | * This deliberately doesn't attempt to use counter metrics named after the log message, since 159 | * extracting that without making a mess is complex, and if that's desired, it's easy enough 160 | * for calling code to do it instead. As an "early warning" indicator that lots of logging 161 | * activity took place, this is useful enough. 162 | */ 163 | private void incrementStats() { 164 | if(stats != null) { 165 | stats.increment(level.getLevelName() + RATE_LIMITED_COUNT_SUFFIX); 166 | } 167 | } 168 | 169 | /** 170 | * Two RateLimitedLogWithPattern objects are considered equal if their messages match; the 171 | * RateAndPeriods are not significant. 172 | */ 173 | @Override 174 | public boolean equals(@Nullable Object o) { 175 | if (this == o) { 176 | return true; 177 | } 178 | if (o == null || getClass() != o.getClass()) { 179 | return false; 180 | } 181 | LogWithPatternAndLevel other = (LogWithPatternAndLevel) o; 182 | return other.level == level && (message.equals(other.message)); 183 | } 184 | 185 | @Override 186 | public int hashCode() { 187 | int result = message.hashCode(); 188 | result = 31 * result + level.hashCode(); 189 | return result; 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /src/test/java/com/swrve/ratelimitedlogger/MockLogger.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.slf4j.Marker; 6 | import org.slf4j.helpers.MessageFormatter; 7 | 8 | import java.util.Optional; 9 | 10 | /** 11 | * An implementation of Logger suitable for the rate-limited log unit tests 12 | */ 13 | class MockLogger implements Logger { 14 | private static final Logger logger = LoggerFactory.getLogger(MockLogger.class); 15 | public int infoMessageCount; 16 | public int debugMessageCount; 17 | public int warnMessageCount; 18 | public int errorMessageCount; 19 | int traceMessageCount; 20 | String infoLastMessage; 21 | 22 | @Override 23 | public boolean isDebugEnabled() { 24 | return true; 25 | } 26 | 27 | @Override 28 | public void debug(String msg) { 29 | logger.info("[debug] "+msg); 30 | debugMessageCount++; 31 | } 32 | 33 | @Override 34 | public void debug(String format, Object arg) { 35 | throw new IllegalStateException("not supported"); 36 | } 37 | 38 | @Override 39 | public void debug(String format, Object arg1, Object arg2) { 40 | throw new IllegalStateException("not supported"); 41 | } 42 | 43 | @Override 44 | public void debug(String msg, Object... arguments) { 45 | debug(MessageFormatter.arrayFormat(msg, arguments).getMessage()); 46 | } 47 | 48 | @Override 49 | public void debug(String msg, Throwable t) { 50 | throw new IllegalStateException("not supported"); 51 | } 52 | 53 | @Override 54 | public boolean isDebugEnabled(Marker marker) { 55 | return false; 56 | } 57 | 58 | @Override 59 | public void debug(Marker marker, String msg) { 60 | throw new IllegalStateException("not supported"); 61 | } 62 | 63 | @Override 64 | public void debug(Marker marker, String format, Object arg) { 65 | throw new IllegalStateException("not supported"); 66 | } 67 | 68 | @Override 69 | public void debug(Marker marker, String format, Object arg1, Object arg2) { 70 | throw new IllegalStateException("not supported"); 71 | } 72 | 73 | @Override 74 | public void debug(Marker marker, String format, Object... arguments) { 75 | throw new IllegalStateException("not supported"); 76 | } 77 | 78 | @Override 79 | public void debug(Marker marker, String msg, Throwable t) { 80 | throw new IllegalStateException("not supported"); 81 | } 82 | 83 | @Override 84 | public boolean isErrorEnabled() { 85 | return true; 86 | } 87 | 88 | @Override 89 | public void error(String msg) { 90 | logger.info("[error] "+msg); 91 | errorMessageCount++; 92 | } 93 | 94 | @Override 95 | public void error(String format, Object arg) { 96 | throw new IllegalStateException("not supported"); 97 | } 98 | 99 | @Override 100 | public void error(String format, Object arg1, Object arg2) { 101 | throw new IllegalStateException("not supported"); 102 | } 103 | 104 | @Override 105 | public void error(String msg, Object... arguments) { 106 | error(MessageFormatter.arrayFormat(msg, arguments).getMessage()); 107 | } 108 | 109 | @Override 110 | public void error(String msg, Throwable t) { 111 | throw new IllegalStateException("not supported"); 112 | } 113 | 114 | @Override 115 | public boolean isErrorEnabled(Marker marker) { 116 | return false; 117 | } 118 | 119 | @Override 120 | public void error(Marker marker, String msg) { 121 | throw new IllegalStateException("not supported"); 122 | } 123 | 124 | @Override 125 | public void error(Marker marker, String format, Object arg) { 126 | throw new IllegalStateException("not supported"); 127 | } 128 | 129 | @Override 130 | public void error(Marker marker, String format, Object arg1, Object arg2) { 131 | throw new IllegalStateException("not supported"); 132 | } 133 | 134 | @Override 135 | public void error(Marker marker, String format, Object... arguments) { 136 | throw new IllegalStateException("not supported"); 137 | } 138 | 139 | @Override 140 | public void error(Marker marker, String msg, Throwable t) { 141 | throw new IllegalStateException("not supported"); 142 | } 143 | 144 | @Override 145 | public boolean isInfoEnabled() { 146 | return true; 147 | } 148 | 149 | @Override 150 | public void info(String msg) { 151 | logger.info("[info] "+msg); 152 | infoLastMessage = msg; 153 | infoMessageCount++; 154 | } 155 | 156 | @Override 157 | public void info(String format, Object arg) { 158 | throw new IllegalStateException("not supported"); 159 | } 160 | 161 | @Override 162 | public void info(String format, Object arg1, Object arg2) { 163 | throw new IllegalStateException("not supported"); 164 | } 165 | 166 | @Override 167 | public void info(String msg, Object... arguments) { 168 | info(MessageFormatter.arrayFormat(msg, arguments).getMessage()); 169 | } 170 | 171 | @Override 172 | public void info(String msg, Throwable t) { 173 | throw new IllegalStateException("not supported"); 174 | } 175 | 176 | @Override 177 | public boolean isInfoEnabled(Marker marker) { 178 | return false; 179 | } 180 | 181 | @Override 182 | public void info(Marker marker, String msg) { 183 | throw new IllegalStateException("not supported"); 184 | } 185 | 186 | @Override 187 | public void info(Marker marker, String format, Object arg) { 188 | throw new IllegalStateException("not supported"); 189 | } 190 | 191 | @Override 192 | public void info(Marker marker, String format, Object arg1, Object arg2) { 193 | throw new IllegalStateException("not supported"); 194 | } 195 | 196 | @Override 197 | public void info(Marker marker, String format, Object... arguments) { 198 | throw new IllegalStateException("not supported"); 199 | } 200 | 201 | @Override 202 | public void info(Marker marker, String msg, Throwable t) { 203 | throw new IllegalStateException("not supported"); 204 | } 205 | 206 | @Override 207 | public String getName() { 208 | return null; 209 | } 210 | 211 | @Override 212 | public boolean isTraceEnabled() { 213 | return true; 214 | } 215 | 216 | @Override 217 | public void trace(String msg) { 218 | logger.info("[trace] "+msg); 219 | traceMessageCount++; 220 | } 221 | 222 | @Override 223 | public void trace(String format, Object arg) { 224 | throw new IllegalStateException("not supported"); 225 | } 226 | 227 | @Override 228 | public void trace(String format, Object arg1, Object arg2) { 229 | throw new IllegalStateException("not supported"); 230 | } 231 | 232 | @Override 233 | public void trace(String msg, Object... arguments) { 234 | trace(MessageFormatter.arrayFormat(msg, arguments).getMessage()); 235 | } 236 | 237 | @Override 238 | public void trace(String msg, Throwable t) { 239 | throw new IllegalStateException("not supported"); 240 | } 241 | 242 | @Override 243 | public boolean isTraceEnabled(Marker marker) { 244 | return false; 245 | } 246 | 247 | @Override 248 | public void trace(Marker marker, String msg) { 249 | throw new IllegalStateException("not supported"); 250 | } 251 | 252 | @Override 253 | public void trace(Marker marker, String format, Object arg) { 254 | throw new IllegalStateException("not supported"); 255 | } 256 | 257 | @Override 258 | public void trace(Marker marker, String format, Object arg1, Object arg2) { 259 | throw new IllegalStateException("not supported"); 260 | } 261 | 262 | @Override 263 | public void trace(Marker marker, String format, Object... argArray) { 264 | throw new IllegalStateException("not supported"); 265 | } 266 | 267 | @Override 268 | public void trace(Marker marker, String msg, Throwable t) { 269 | throw new IllegalStateException("not supported"); 270 | } 271 | 272 | @Override 273 | public boolean isWarnEnabled() { 274 | return true; 275 | } 276 | 277 | @Override 278 | public void warn(String msg) { 279 | logger.info("[warn] "+msg); 280 | warnMessageCount++; 281 | } 282 | 283 | @Override 284 | public void warn(String format, Object arg) { 285 | throw new IllegalStateException("not supported"); 286 | } 287 | 288 | @Override 289 | public void warn(String msg, Object... arguments) { 290 | warn(MessageFormatter.arrayFormat(msg, arguments).getMessage()); 291 | } 292 | 293 | @Override 294 | public void warn(String format, Object arg1, Object arg2) { 295 | throw new IllegalStateException("not supported"); 296 | } 297 | 298 | @Override 299 | public void warn(String msg, Throwable t) { 300 | throw new IllegalStateException("not supported"); 301 | } 302 | 303 | @Override 304 | public boolean isWarnEnabled(Marker marker) { 305 | return false; 306 | } 307 | 308 | @Override 309 | public void warn(Marker marker, String msg) { 310 | throw new IllegalStateException("not supported"); 311 | } 312 | 313 | @Override 314 | public void warn(Marker marker, String format, Object arg) { 315 | throw new IllegalStateException("not supported"); 316 | } 317 | 318 | @Override 319 | public void warn(Marker marker, String format, Object arg1, Object arg2) { 320 | throw new IllegalStateException("not supported"); 321 | } 322 | 323 | @Override 324 | public void warn(Marker marker, String format, Object... arguments) { 325 | throw new IllegalStateException("not supported"); 326 | } 327 | 328 | @Override 329 | public void warn(Marker marker, String msg, Throwable t) { 330 | throw new IllegalStateException("not supported"); 331 | } 332 | 333 | public Optional getInfoLastMessage() { 334 | return Optional.of(infoLastMessage); 335 | } 336 | } 337 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /src/test/java/com/swrve/ratelimitedlogger/RateLimitedLogTest.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.junit.Test; 4 | 5 | import java.time.Duration; 6 | import java.util.concurrent.atomic.AtomicBoolean; 7 | import java.util.concurrent.atomic.AtomicLong; 8 | 9 | import static org.hamcrest.CoreMatchers.equalTo; 10 | import static org.hamcrest.CoreMatchers.not; 11 | import static org.junit.Assert.assertThat; 12 | 13 | public class RateLimitedLogTest { 14 | 15 | @Test 16 | public void basic() { 17 | MockLogger logger = new MockLogger(); 18 | final AtomicLong mockTime = new AtomicLong(0L); 19 | 20 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 21 | .maxRate(1).every(Duration.ofMillis(10)) 22 | .withStopwatch(createStopwatch(mockTime)) 23 | .build(); 24 | 25 | assertThat(logger.infoMessageCount, equalTo(0)); 26 | 27 | mockTime.set(1L); 28 | rateLimitedLog.info("basic"); 29 | mockTime.set(2L); 30 | 31 | assertThat(logger.infoMessageCount, equalTo(1)); 32 | } 33 | 34 | @Test 35 | public void trace() { 36 | MockLogger logger = new MockLogger(); 37 | final AtomicLong mockTime = new AtomicLong(0L); 38 | 39 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 40 | .maxRate(1).every(Duration.ofMillis(10)) 41 | .withStopwatch(createStopwatch(mockTime)) 42 | .build(); 43 | 44 | rateLimitedLog.trace("trace"); 45 | assertThat(logger.traceMessageCount, equalTo(1)); 46 | } 47 | 48 | @Test 49 | public void rateLimitingWorks() throws InterruptedException { 50 | MockLogger logger = new MockLogger(); 51 | final AtomicLong mockTime = new AtomicLong(0L); 52 | 53 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 54 | .maxRate(1).every(Duration.ofMillis(500)) 55 | .withStopwatch(createStopwatch(mockTime)) 56 | .build(); 57 | 58 | assertThat(logger.infoMessageCount, equalTo(0)); 59 | 60 | mockTime.set(1L); 61 | rateLimitedLog.info("rateLimited {}", 1); 62 | mockTime.set(2L); 63 | rateLimitedLog.info("rateLimited {}", 2); 64 | 65 | // the second message should have been suppressed 66 | assertThat(logger.infoMessageCount, equalTo(1)); 67 | 68 | Thread.sleep(600L); 69 | mockTime.set(601L); 70 | 71 | // zeroing the counter should produce a "similar messages suppressed" message 72 | assertThat(logger.infoMessageCount, equalTo(2)); 73 | 74 | Thread.sleep(500L); 75 | mockTime.set(1101L); 76 | 77 | // no logs in the meantime, so no new message 78 | assertThat(logger.infoMessageCount, equalTo(2)); 79 | } 80 | 81 | @Test 82 | public void rateLimitingWorksTwice() throws InterruptedException { 83 | MockLogger logger = new MockLogger(); 84 | final AtomicLong mockTime = new AtomicLong(0L); 85 | 86 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 87 | .maxRate(1).every(Duration.ofMillis(500)) 88 | .withStopwatch(createStopwatch(mockTime)) 89 | .build(); 90 | 91 | assertThat(logger.infoMessageCount, equalTo(0)); 92 | 93 | mockTime.set(1L); 94 | rateLimitedLog.info("rateLimitingWorks2 {}", 1); 95 | mockTime.set(2L); 96 | rateLimitedLog.info("rateLimitingWorks2 {}", 2); 97 | 98 | // the second message should have been suppressed 99 | assertThat(logger.infoMessageCount, equalTo(1)); 100 | 101 | Thread.sleep(1000L); 102 | mockTime.set(601L); 103 | 104 | // by now, we should have seen the "similar messages suppressed" message 105 | assertThat(logger.infoMessageCount, equalTo(2)); 106 | 107 | rateLimitedLog.info("rateLimitingWorks2 {}", 3); 108 | rateLimitedLog.info("rateLimitingWorks2 {}", 4); 109 | rateLimitedLog.info("rateLimitingWorks2 {}", 5); 110 | 111 | // should have suppressed 4 and 5 112 | assertThat(logger.infoMessageCount, equalTo(3)); 113 | 114 | Thread.sleep(500L); 115 | mockTime.set(1101L); 116 | 117 | // should have seen a second "suppressed" message after 500ms 118 | assertThat(logger.infoMessageCount, equalTo(4)); 119 | } 120 | 121 | @Test 122 | public void rateLimitingNonZeroRateAndAllThresholds() throws InterruptedException { 123 | MockLogger logger = new MockLogger(); 124 | final AtomicLong mockTime = new AtomicLong(0L); 125 | 126 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 127 | .maxRate(3).every(Duration.ofMillis(500)) 128 | .withStopwatch(createStopwatch(mockTime)) 129 | .build(); 130 | 131 | assertThat(logger.infoMessageCount, equalTo(0)); 132 | 133 | mockTime.set(1L); 134 | rateLimitedLog.info("rateLimitingNonZeroRateAndAllThresholds {}", 1); 135 | rateLimitedLog.info("rateLimitingNonZeroRateAndAllThresholds {}", 1); 136 | rateLimitedLog.info("rateLimitingNonZeroRateAndAllThresholds {}", 1); 137 | rateLimitedLog.info("rateLimitingNonZeroRateAndAllThresholds {}", 1); 138 | mockTime.set(2L); 139 | rateLimitedLog.debug("rateLimitingNonZeroRateAndAllThresholds {}", 2); 140 | mockTime.set(498L); 141 | rateLimitedLog.warn("rateLimitingNonZeroRateAndAllThresholds {}", 3); 142 | mockTime.set(499L); 143 | rateLimitedLog.trace("rateLimitingNonZeroRateAndAllThresholds {}", 5); 144 | 145 | assertThat(logger.infoMessageCount, equalTo(3)); 146 | assertThat(logger.debugMessageCount, equalTo(1)); 147 | assertThat(logger.warnMessageCount, equalTo(1)); 148 | assertThat(logger.errorMessageCount, equalTo(0)); 149 | assertThat(logger.traceMessageCount, equalTo(1)); 150 | 151 | Thread.sleep(600L); 152 | mockTime.set(600L); 153 | 154 | // zeroing the counter should produce a "similar messages suppressed" message 155 | assertThat(logger.infoMessageCount, equalTo(4)); 156 | 157 | // and now we should be able to log a message 158 | rateLimitedLog.error("rateLimitingNonZeroRateAndAllThresholds {0}", 1); 159 | assertThat(logger.errorMessageCount, equalTo(1)); 160 | } 161 | 162 | @Test 163 | public void multiplePeriods() throws InterruptedException { 164 | MockLogger logger = new MockLogger(); 165 | final AtomicLong mockTime = new AtomicLong(0L); 166 | 167 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 168 | .maxRate(1).every(Duration.ofMillis(200)) 169 | .withStopwatch(createStopwatch(mockTime)) 170 | .build(); 171 | 172 | RateLimitedLog rateLimitedLog2 = RateLimitedLog.withRateLimit(logger) 173 | .maxRate(1).every(Duration.ofMillis(300)) 174 | .withStopwatch(createStopwatch(mockTime)) 175 | .build(); 176 | 177 | assertThat(logger.infoMessageCount, equalTo(0)); 178 | 179 | mockTime.set(1L); 180 | rateLimitedLog.info("multiplePeriods {}", 1); 181 | mockTime.set(2L); 182 | rateLimitedLog.info("multiplePeriods {}", 2); 183 | mockTime.set(3L); 184 | rateLimitedLog2.info("multiplePeriods2 {}", 1); 185 | mockTime.set(4L); 186 | rateLimitedLog2.info("multiplePeriods2 {}", 2); 187 | 188 | // the second message should have been suppressed 189 | assertThat(logger.infoMessageCount, equalTo(2)); 190 | 191 | Thread.sleep(250L); 192 | mockTime.set(251L); 193 | assertThat(logger.infoMessageCount, equalTo(3)); 194 | 195 | Thread.sleep(100L); 196 | mockTime.set(351L); 197 | assertThat(logger.infoMessageCount, equalTo(4)); 198 | } 199 | 200 | @Test 201 | public void withMetrics() { 202 | MockLogger logger = new MockLogger(); 203 | final AtomicLong mockTime = new AtomicLong(0L); 204 | final AtomicBoolean statsCalled = new AtomicBoolean(false); 205 | 206 | CounterMetric mockStats = name -> statsCalled.set(true); 207 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 208 | .maxRate(1).every(Duration.ofMillis(10)) 209 | .withStopwatch(createStopwatch(mockTime)) 210 | .recordMetrics(mockStats) 211 | .build(); 212 | 213 | assertThat(logger.infoMessageCount, equalTo(0)); 214 | 215 | mockTime.set(1L); 216 | rateLimitedLog.info("withMetrics"); 217 | mockTime.set(2L); 218 | 219 | assertThat(logger.infoMessageCount, equalTo(1)); 220 | assertThat(statsCalled.get(), equalTo(true)); 221 | } 222 | 223 | @Test 224 | public void testGet() { 225 | MockLogger logger = new MockLogger(); 226 | final AtomicLong mockTime = new AtomicLong(0L); 227 | 228 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 229 | .maxRate(1).every(Duration.ofMillis(10)) 230 | .withStopwatch(createStopwatch(mockTime)) 231 | .build(); 232 | 233 | RateLimitedLogWithPattern line = rateLimitedLog.get("testGet"); 234 | 235 | assertThat(logger.infoMessageCount, equalTo(0)); 236 | 237 | mockTime.set(1L); 238 | line.info(); 239 | 240 | assertThat(logger.infoMessageCount, equalTo(1)); 241 | 242 | RateLimitedLogWithPattern line2 = rateLimitedLog.get("testGet"); 243 | assertThat(line2, equalTo(line)); 244 | assertThat(line2, not(equalTo(null))); 245 | 246 | RateLimitedLogWithPattern line3 = rateLimitedLog.get("testGet2"); 247 | assertThat(line3, not(equalTo(line))); 248 | assertThat(line3, not(equalTo(null))); 249 | 250 | assertThat(line2.hashCode(), equalTo(line.hashCode())); 251 | assertThat(line2.hashCode(), not(equalTo(line3.hashCode()))); 252 | } 253 | 254 | @Test 255 | public void testGetWithLevel() { 256 | MockLogger logger = new MockLogger(); 257 | final AtomicLong mockTime = new AtomicLong(0L); 258 | 259 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 260 | .maxRate(1).every(Duration.ofMillis(10)) 261 | .withStopwatch(createStopwatch(mockTime)) 262 | .build(); 263 | 264 | LogWithPatternAndLevel line = rateLimitedLog.get("testGet", Level.INFO); 265 | 266 | assertThat(logger.infoMessageCount, equalTo(0)); 267 | 268 | mockTime.set(1L); 269 | line.log(); 270 | 271 | assertThat(logger.infoMessageCount, equalTo(1)); 272 | 273 | LogWithPatternAndLevel line2 = rateLimitedLog.get("testGet", Level.INFO); 274 | assertThat(line2, equalTo(line)); 275 | assertThat(line2, not(equalTo(null))); 276 | 277 | LogWithPatternAndLevel line3 = rateLimitedLog.get("testGet", Level.TRACE); 278 | assertThat(line3, not(equalTo(line))); 279 | assertThat(line3, not(equalTo(null))); 280 | 281 | assertThat(line2.hashCode(), equalTo(line.hashCode())); 282 | assertThat(line2.hashCode(), not(equalTo(line3.hashCode()))); 283 | } 284 | 285 | // Ensure we won't see silly localised formats like "10,000" instead of "10000". 286 | // 287 | @Test 288 | public void testLocaleIgnored() { 289 | MockLogger logger = new MockLogger(); 290 | final AtomicLong mockTime = new AtomicLong(0L); 291 | 292 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 293 | .maxRate(1).every(Duration.ofMillis(10)) 294 | .withStopwatch(createStopwatch(mockTime)) 295 | .build(); 296 | 297 | mockTime.set(1L); 298 | rateLimitedLog.info("locale {}", 10000); 299 | assertThat(logger.getInfoLastMessage().get(), equalTo("locale 10000")); 300 | } 301 | 302 | // Ensure that the out-of-cache-capacity logic doesn't lose data. 303 | @Test 304 | public void outOfCacheCapacity() { 305 | MockLogger logger = new MockLogger(); 306 | final AtomicLong mockTime = new AtomicLong(0L); 307 | 308 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 309 | .maxRate(1).every(Duration.ofMillis(10)) 310 | .withStopwatch(createStopwatch(mockTime)) 311 | .build(); 312 | 313 | for (int i = 0; i < RateLimitedLog.MAX_PATTERNS_PER_LOG + 2; i++) { 314 | rateLimitedLog.info("cache " + i); 315 | assertThat(logger.getInfoLastMessage().get(), equalTo("cache " + i)); // no loss 316 | } 317 | 318 | // check that the cache was wiped once it filled up 319 | assertThat(rateLimitedLog.knownPatterns.size(), equalTo(1)); 320 | } 321 | 322 | // Ensure that the out-of-cache-capacity logic doesn't lose data. 323 | @Test 324 | public void overlongKeyStrings() { 325 | MockLogger logger = new MockLogger(); 326 | final AtomicLong mockTime = new AtomicLong(0L); 327 | 328 | RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger) 329 | .maxRate(1).every(Duration.ofMillis(10)) 330 | .withStopwatch(createStopwatch(mockTime)) 331 | .build(); 332 | 333 | StringBuilder s = new StringBuilder(); 334 | for (int i = 0; i < 400; i++) { 335 | s.append("this string is too long for the cache "); // 38 chars 336 | } 337 | String overLongString = s.toString(); 338 | 339 | rateLimitedLog.info(overLongString + " 1"); 340 | rateLimitedLog.info(overLongString + " 2"); 341 | 342 | // check they shared the same rate limit 343 | assertThat(logger.getInfoLastMessage().get(), equalTo(overLongString + " 1")); 344 | assertThat(rateLimitedLog.knownPatterns.size(), equalTo(1)); 345 | } 346 | 347 | private Stopwatch createStopwatch(final AtomicLong mockTime) { 348 | return new Stopwatch(mockTime.get()); 349 | } 350 | 351 | } 352 | -------------------------------------------------------------------------------- /src/main/java/com/swrve/ratelimitedlogger/RateLimitedLog.java: -------------------------------------------------------------------------------- 1 | package com.swrve.ratelimitedlogger; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.Marker; 5 | 6 | import net.jcip.annotations.ThreadSafe; 7 | import java.util.Objects; 8 | import java.util.concurrent.ConcurrentHashMap; 9 | 10 | /** 11 | * An SLF4J-compatible API for rate-limited logging. Example usage: 12 | * 13 | * 
 14 |  *    private static final Logger logger = LoggerFactory.getLogger(getClass());
 15 |  *    private static final RateLimitedLog rateLimitedLog = RateLimitedLog.withRateLimit(logger)
 16 |  *             .maxRate(5).every(Duration.ofSeconds(10))
 17 |  *             .build();
 18 |  *
19 | * 20 | * This will wrap an existing SLF4J Logger object, allowing a max of 5 messages to be output every 10 seconds, 21 | * suppressing any more than that. When a log is suppressed, at the end of the 10-second period, another 22 | * log message is output indicating how many logs were hidden. This style of rate limiting is the same as the 23 | * one used by UNIX syslog, so should be comprehensible, easy to predict, and familiar to many users, unlike 24 | * more complex adaptive rate limits. 25 | * 26 | * Each log message has its own internal rate limiting counter. In other words, if you have 2 log messages, you can 27 | * safely reuse the same RateLimitedLog object to log both, and the rate of one will not caused the other to 28 | * be suppressed as a side effect. However, this means that if you wish to include dynamic, variable data in the log 29 | * output, you will need to use SLF4J-style templates, instead of ("foo " + bar + " baz") string interpolation. 30 | * For example: 31 | * 32 | * 
 33 |  *    rateLimitedLog.info("Just saw an event of type {}: {}", event.getType(), event);
 34 |  *
35 | * 36 | * "{}" will implicitly invoke an object's toString() method, so toString() does not need 37 | * to be called explicitly when logging. (This has obvious performance benefits, in that 38 | * those toString() methods will not be called at all once the rate limits have been exceeded.) 39 | * 40 | * Where performance is critical, note that you can obtain a reference to the RateLimitedLogWithPattern object 41 | * for an individual log template, which will avoid a ConcurrentHashMap lookup. 42 | * 43 | * The RateLimitedLog objects are thread-safe. 44 | */ 45 | @ThreadSafe 46 | public class RateLimitedLog implements Logger { 47 | /** 48 | * A singleton registry of rate-limited logs, so we can reset them periodically 49 | */ 50 | static final Registry REGISTRY = new Registry(); 51 | 52 | /** 53 | * We have a limit of 1000 knownPattern objects per RateLimitedLog; exceed this, and it's 54 | * probable that an already-interpolated string is accidentally being used as a 55 | * pattern. 56 | */ 57 | static final int MAX_PATTERNS_PER_LOG = 1000; 58 | 59 | /** 60 | * Impose a limit of this many characters in the knownPattern hash; this helps avoid 61 | * a situation where an already-interpolated string is accidentally being used as a 62 | * pattern, and some very large strings have been interpolated into it, resulting in 63 | * high memory consumption and GC pressure. 64 | */ 65 | private static final int MAX_PATTERN_LENGTH = 8192; 66 | 67 | final ConcurrentHashMap knownPatterns 68 | = new ConcurrentHashMap<>(); 69 | 70 | private final Logger logger; 71 | private final RateLimitedLogWithPattern.RateAndPeriod rateAndPeriod; 72 | private final Registry registry; 73 | private final Stopwatch stopwatch; 74 | private final CounterMetric stats; 75 | 76 | /** 77 | * Start building a new RateLimitedLog, wrapping the SLF4J logger @param logger. 78 | */ 79 | public static RateLimitedLogBuilder.MissingRateAndPeriod withRateLimit(Logger logger) { 80 | return new RateLimitedLogBuilder.MissingRateAndPeriod(Objects.requireNonNull(logger)); 81 | } 82 | 83 | // package-local ctor called by the Builder 84 | @SuppressWarnings("SameParameterValue") 85 | RateLimitedLog(Logger logger, RateLimitedLogWithPattern.RateAndPeriod rateAndPeriod, Stopwatch stopwatch, CounterMetric stats, Registry registry) { 86 | this.logger = logger; 87 | this.rateAndPeriod = rateAndPeriod; 88 | this.registry = registry; 89 | this.stats = stats; 90 | this.stopwatch = stopwatch; 91 | } 92 | 93 | @Override 94 | public String getName() { 95 | return logger.getName(); 96 | } 97 | 98 | @Override 99 | public boolean isTraceEnabled() { 100 | return logger.isTraceEnabled(); 101 | } 102 | 103 | @Override 104 | public void trace(String msg) { 105 | get(msg).trace(msg); 106 | } 107 | 108 | @Override 109 | public void trace(String format, Object arg) { 110 | get(format).trace(arg); 111 | } 112 | 113 | @Override 114 | public void trace(String format, Object arg1, Object arg2) { 115 | get(format).trace(arg1, arg2); 116 | } 117 | 118 | @Override 119 | public void trace(String format, Object... arguments) { 120 | get(format).trace(arguments); 121 | } 122 | 123 | @Override 124 | public void trace(String msg, Throwable t) { 125 | get(msg).trace(t); 126 | } 127 | 128 | @Override 129 | public boolean isTraceEnabled(Marker marker) { 130 | return logger.isTraceEnabled(marker); 131 | } 132 | 133 | @Override 134 | public void trace(Marker marker, String msg) { 135 | get(msg).trace(marker, msg); 136 | } 137 | 138 | @Override 139 | public void trace(Marker marker, String format, Object arg) { 140 | get(format).trace(marker, arg); 141 | } 142 | 143 | @Override 144 | public void trace(Marker marker, String format, Object arg1, Object arg2) { 145 | get(format).trace(marker, arg1, arg2); 146 | } 147 | 148 | @Override 149 | public void trace(Marker marker, String format, Object... argArray) { 150 | get(format).trace(marker, argArray); 151 | } 152 | 153 | @Override 154 | public void trace(Marker marker, String msg, Throwable t) { 155 | get(msg).trace(marker, t); 156 | } 157 | 158 | @Override 159 | public boolean isDebugEnabled() { 160 | return logger.isDebugEnabled(); 161 | } 162 | 163 | @Override 164 | public void debug(String msg) { 165 | get(msg).debug(msg); 166 | } 167 | 168 | @Override 169 | public void debug(String format, Object arg) { 170 | get(format).debug(arg); 171 | } 172 | 173 | @Override 174 | public void debug(String format, Object arg1, Object arg2) { 175 | get(format).debug(arg1, arg2); 176 | } 177 | 178 | @Override 179 | public void debug(String format, Object... arguments) { 180 | get(format).debug(arguments); 181 | } 182 | 183 | @Override 184 | public void debug(String msg, Throwable t) { 185 | get(msg).debug(t); 186 | } 187 | 188 | @Override 189 | public boolean isDebugEnabled(Marker marker) { 190 | return logger.isDebugEnabled(marker); 191 | } 192 | 193 | @Override 194 | public void debug(Marker marker, String msg) { 195 | get(msg).debug(marker, msg); 196 | } 197 | 198 | @Override 199 | public void debug(Marker marker, String format, Object arg) { 200 | get(format).debug(marker, arg); 201 | } 202 | 203 | @Override 204 | public void debug(Marker marker, String format, Object arg1, Object arg2) { 205 | get(format).debug(marker, arg1, arg2); 206 | } 207 | 208 | @Override 209 | public void debug(Marker marker, String format, Object... argArray) { 210 | get(format).debug(marker, argArray); 211 | } 212 | 213 | @Override 214 | public void debug(Marker marker, String msg, Throwable t) { 215 | get(msg).debug(marker, t); 216 | } 217 | 218 | @Override 219 | public boolean isInfoEnabled() { 220 | return logger.isInfoEnabled(); 221 | } 222 | 223 | @Override 224 | public void info(String msg) { 225 | get(msg).info(msg); 226 | } 227 | 228 | @Override 229 | public void info(String format, Object arg) { 230 | get(format).info(arg); 231 | } 232 | 233 | @Override 234 | public void info(String format, Object arg1, Object arg2) { 235 | get(format).info(arg1, arg2); 236 | } 237 | 238 | @Override 239 | public void info(String format, Object... arguments) { 240 | get(format).info(arguments); 241 | } 242 | 243 | @Override 244 | public void info(String msg, Throwable t) { 245 | get(msg).info(t); 246 | } 247 | 248 | @Override 249 | public boolean isInfoEnabled(Marker marker) { 250 | return logger.isInfoEnabled(marker); 251 | } 252 | 253 | @Override 254 | public void info(Marker marker, String msg) { 255 | get(msg).info(marker, msg); 256 | } 257 | 258 | @Override 259 | public void info(Marker marker, String format, Object arg) { 260 | get(format).info(marker, arg); 261 | } 262 | 263 | @Override 264 | public void info(Marker marker, String format, Object arg1, Object arg2) { 265 | get(format).info(marker, arg1, arg2); 266 | } 267 | 268 | @Override 269 | public void info(Marker marker, String format, Object... argArray) { 270 | get(format).info(marker, argArray); 271 | } 272 | 273 | @Override 274 | public void info(Marker marker, String msg, Throwable t) { 275 | get(msg).info(marker, t); 276 | } 277 | @Override 278 | public boolean isWarnEnabled() { 279 | return logger.isWarnEnabled(); 280 | } 281 | 282 | @Override 283 | public void warn(String msg) { 284 | get(msg).warn(msg); 285 | } 286 | 287 | @Override 288 | public void warn(String format, Object arg) { 289 | get(format).warn(arg); 290 | } 291 | 292 | @Override 293 | public void warn(String format, Object arg1, Object arg2) { 294 | get(format).warn(arg1, arg2); 295 | } 296 | 297 | @Override 298 | public void warn(String format, Object... arguments) { 299 | get(format).warn(arguments); 300 | } 301 | 302 | @Override 303 | public void warn(String msg, Throwable t) { 304 | get(msg).warn(t); 305 | } 306 | 307 | @Override 308 | public boolean isWarnEnabled(Marker marker) { 309 | return logger.isWarnEnabled(marker); 310 | } 311 | 312 | @Override 313 | public void warn(Marker marker, String msg) { 314 | get(msg).warn(marker, msg); 315 | } 316 | 317 | @Override 318 | public void warn(Marker marker, String format, Object arg) { 319 | get(format).warn(marker, arg); 320 | } 321 | 322 | @Override 323 | public void warn(Marker marker, String format, Object arg1, Object arg2) { 324 | get(format).warn(marker, arg1, arg2); 325 | } 326 | 327 | @Override 328 | public void warn(Marker marker, String format, Object... argArray) { 329 | get(format).warn(marker, argArray); 330 | } 331 | 332 | @Override 333 | public void warn(Marker marker, String msg, Throwable t) { 334 | get(msg).warn(marker, t); 335 | } 336 | 337 | @Override 338 | public boolean isErrorEnabled() { 339 | return logger.isErrorEnabled(); 340 | } 341 | 342 | @Override 343 | public void error(String msg) { 344 | get(msg).error(msg); 345 | } 346 | 347 | @Override 348 | public void error(String format, Object arg) { 349 | get(format).error(arg); 350 | } 351 | 352 | @Override 353 | public void error(String format, Object arg1, Object arg2) { 354 | get(format).error(arg1, arg2); 355 | } 356 | 357 | @Override 358 | public void error(String format, Object... arguments) { 359 | get(format).error(arguments); 360 | } 361 | 362 | @Override 363 | public void error(String msg, Throwable t) { 364 | get(msg).error(t); 365 | } 366 | 367 | @Override 368 | public boolean isErrorEnabled(Marker marker) { 369 | return logger.isErrorEnabled(marker); 370 | } 371 | 372 | @Override 373 | public void error(Marker marker, String msg) { 374 | get(msg).error(marker, msg); 375 | } 376 | 377 | @Override 378 | public void error(Marker marker, String format, Object arg) { 379 | get(format).error(marker, arg); 380 | } 381 | 382 | @Override 383 | public void error(Marker marker, String format, Object arg1, Object arg2) { 384 | get(format).error(marker, arg1, arg2); 385 | } 386 | 387 | @Override 388 | public void error(Marker marker, String format, Object... argArray) { 389 | get(format).error(marker, argArray); 390 | } 391 | 392 | @Override 393 | public void error(Marker marker, String msg, Throwable t) { 394 | get(msg).error(marker, t); 395 | } 396 | 397 | /** 398 | * @return a RateLimitedLogWithPattern object for the supplied @param message. This can be cached and 399 | * reused by callers in performance-sensitive cases to avoid performing a ConcurrentHashMap lookup. 400 | * 401 | * Note that the string is the sole key used, so the same string cannot be reused with differing period 402 | * settings; any periods which differ from the first one used are ignored. 403 | * 404 | * The first 8192 characters of the message are used as the key, so if an extremely long log pattern 405 | * is used, with differences only after that threshold, they will share the same rate limiter. 406 | * 407 | * @throws IllegalStateException if we exceed the limit on number of RateLimitedLogWithPattern objects 408 | * in any one period; if this happens, it's probable that an already-interpolated string is 409 | * accidentally being used as a log pattern. 410 | */ 411 | public RateLimitedLogWithPattern get(final String message) { 412 | String key = (message.length() > MAX_PATTERN_LENGTH) ? message.substring(0, MAX_PATTERN_LENGTH) : message; 413 | 414 | // fast path: hopefully we can do this without creating a Supplier object 415 | RateLimitedLogWithPattern got = knownPatterns.get(key); 416 | if (got != null) { 417 | return got; 418 | } 419 | 420 | // before we create another one, check cache capacity first 421 | if (knownPatterns.size() > MAX_PATTERNS_PER_LOG) { 422 | outOfCacheCapacity(); 423 | } 424 | 425 | // slow path: create a RateLimitedLogWithPattern 426 | RateLimitedLogWithPattern newValue = new RateLimitedLogWithPattern(message, rateAndPeriod, registry, stats, stopwatch, logger); 427 | RateLimitedLogWithPattern oldValue = knownPatterns.putIfAbsent(key, newValue); 428 | if (oldValue != null) { 429 | return oldValue; 430 | } else { 431 | return newValue; 432 | } 433 | } 434 | 435 | /** 436 | * @return a LogWithPatternAndLevel object for the supplied @param message and 437 | * @param level . This can be cached and reused by callers in performance-sensitive 438 | * cases to avoid performing two ConcurrentHashMap lookups. 439 | * 440 | * Note that the string is the sole key used, so the same string cannot be reused with differing period 441 | * settings; any periods which differ from the first one used are ignored. 442 | * 443 | * @throws IllegalStateException if we exceed the limit on number of RateLimitedLogWithPattern objects 444 | * in any one period; if this happens, it's probable that an already-interpolated string is 445 | * accidentally being used as a log pattern. 446 | */ 447 | public LogWithPatternAndLevel get(String pattern, Level level) { 448 | return get(pattern).get(level); 449 | } 450 | 451 | /** 452 | * We've run out of capacity in our cache of RateLimitedLogWithPattern objects. This probably 453 | * means that the caller is accidentally calling us with an already-interpolated string, instead 454 | * of using the pattern as the key and letting us do the interpolation. Don't lose data; 455 | * instead, fall back to flushing the entire cache but carrying on. The worst-case scenario 456 | * here is that we flush the logs far more frequently than their requested durations, potentially 457 | * allowing the logging to impact throughput, but we don't lose any log data. 458 | */ 459 | private void outOfCacheCapacity() { 460 | synchronized (knownPatterns) { 461 | if (knownPatterns.size() > MAX_PATTERNS_PER_LOG) { 462 | logger.warn("out of capacity in RateLimitedLog registry; accidentally " + 463 | "using interpolated strings as patterns?"); 464 | registry.flush(); 465 | knownPatterns.clear(); 466 | } 467 | } 468 | } 469 | } 470 | --------------------------------------------------------------------------------