├── .gitattributes ├── .gitignore ├── .ruby-version ├── .tool-versions ├── Dockerfile ├── Gemfile ├── Gemfile.lock ├── README.md ├── Rakefile ├── app ├── controllers │ ├── application_controller.rb │ ├── concerns │ │ └── .keep │ └── pessoas_controller.rb ├── jobs │ ├── pessoa_flush_job.rb │ └── pessoa_job.rb └── models │ ├── application_record.rb │ ├── concerns │ └── .keep │ └── pessoa.rb ├── bin ├── bundle ├── rails ├── rake └── setup ├── config.ru ├── config ├── application.rb ├── boot.rb ├── credentials.yml.enc ├── database.yml ├── environment.rb ├── environments │ ├── development.rb │ ├── production.rb │ └── test.rb ├── initializers │ ├── cors.rb │ ├── filter_parameter_logging.rb │ ├── inflections.rb │ └── sidekiq.rb ├── locales │ └── en.yml ├── puma.rb └── routes.rb ├── db ├── migrate │ ├── 20230825134432_create_pessoas.rb │ ├── 20230825151541_add_pg_trgm_extension.rb │ └── 20230827015102_add_index_pessoas_on_search.rb ├── schema.rb ├── seeds.rb └── structure.sql ├── docker-compose.yml ├── entrypoint.sh ├── imgs ├── graphs1.png └── graphs2.png ├── lib ├── redis_queue.rb ├── tag_coder.rb └── tasks │ └── .keep ├── log └── .keep ├── nginx.conf ├── postgresql.conf ├── public └── robots.txt ├── storage └── .keep ├── test ├── controllers │ ├── .keep │ └── pessoas_controller_test.rb ├── fixtures │ ├── files │ │ └── .keep │ └── pessoas.yml ├── integration │ └── .keep ├── jobs │ ├── pessoa_flush_job_test.rb │ └── pessoa_job_test.rb ├── lib │ └── redis_queue_test.rb ├── mailers │ └── .keep ├── models │ ├── .keep │ └── pessoa_test.rb └── test_helper.rb ├── tmp ├── .keep ├── pids │ └── .keep └── storage │ └── .keep ├── tmuxp-monitor.yaml └── vendor └── .keep /.gitattributes: -------------------------------------------------------------------------------- 1 | # See https://git-scm.com/docs/gitattributes for more about git attribute files. 2 | 3 | # Mark the database schema as having been generated. 4 | db/schema.rb linguist-generated 5 | 6 | # Mark any vendored files as having been vendored. 7 | vendor/* linguist-vendored 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files for more about ignoring files. 2 | # 3 | # If you find yourself ignoring temporary files generated by your text editor 4 | # or operating system, you probably want to add a global ignore instead: 5 | # git config --global core.excludesfile '~/.gitignore_global' 6 | 7 | # Ignore bundler config. 8 | /.bundle 9 | 10 | # Ignore the default SQLite database. 11 | /db/*.sqlite3 12 | /db/*.sqlite3-* 13 | 14 | # Ignore all logfiles and tempfiles. 15 | /log/* 16 | /tmp/* 17 | !/log/.keep 18 | !/tmp/.keep 19 | 20 | # Ignore pidfiles, but keep the directory. 21 | /tmp/pids/* 22 | !/tmp/pids/ 23 | !/tmp/pids/.keep 24 | 25 | # Ignore uploaded files in development. 26 | /storage/* 27 | !/storage/.keep 28 | /tmp/storage/* 29 | !/tmp/storage/ 30 | !/tmp/storage/.keep 31 | 32 | # Ignore master key for decrypting credentials and more. 33 | /config/master.key 34 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | ruby-3.2.2 2 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | ruby 3.3.0-preview2 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1 2 | 3 | # Make sure RUBY_VERSION matches the Ruby version in .ruby-version and Gemfile 4 | ARG RUBY_VERSION=3.3.0-preview2 5 | FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim as base 6 | 7 | # Rails app lives here 8 | WORKDIR /rails 9 | 10 | # Set production environment 11 | ENV RAILS_ENV="production" \ 12 | BUNDLE_DEPLOYMENT="1" \ 13 | BUNDLE_PATH="/usr/local/bundle" \ 14 | BUNDLE_WITHOUT="development" 15 | 16 | 17 | # Throw-away build stage to reduce size of final image 18 | FROM base as build 19 | 20 | # Install packages needed to build gems 21 | RUN apt-get update -qq && \ 22 | apt-get install --no-install-recommends -y build-essential git libpq-dev libvips pkg-config 23 | 24 | # Install application gems 25 | COPY Gemfile Gemfile.lock ./ 26 | RUN bundle install && \ 27 | rm -rf ~/.bundle/ "${BUNDLE_PATH}"/ruby/*/cache "${BUNDLE_PATH}"/ruby/*/bundler/gems/*/.git && \ 28 | bundle exec bootsnap precompile --gemfile 29 | 30 | 31 | # Copy application code 32 | COPY . . 33 | 34 | # Precompile bootsnap code for faster boot times 35 | RUN bundle exec bootsnap precompile app/ lib/ 36 | 37 | 38 | # Final stage for app image 39 | FROM base 40 | 41 | # Install packages needed for deployment 42 | RUN apt-get update -qq && \ 43 | apt-get install --no-install-recommends -y curl libvips postgresql-client netcat-traditional && \ 44 | rm -rf /var/lib/apt/lists /var/cache/apt/archives 45 | 46 | # Copy built artifacts: gems, application 47 | COPY --from=build /usr/local/bundle /usr/local/bundle 48 | COPY --from=build /rails /rails 49 | 50 | # Run and own only the runtime files as a non-root user for security 51 | # RUN useradd rails --create-home --shell /bin/bash && \ 52 | # chown -R rails:rails db log storage tmp && \ 53 | # USER rails:rails 54 | 55 | # Entrypoint prepares the database. 56 | # ENTRYPOINT ["/rails/bin/docker-entrypoint"] 57 | 58 | # Start the server by default, this can be overwritten at runtime 59 | EXPOSE 3000 60 | CMD ["sh", "/rails/entrypoint.sh"] 61 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | git_source(:github) { |repo| "https://github.com/#{repo}.git" } 3 | 4 | ruby "3.3.0.preview2" 5 | 6 | # Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main" 7 | # gem "rails", "~> 7.0.7", ">= 7.0.7.2" 8 | gem "rails", github: "rails/rails", ref: "5cf742ef514b2184eb3d617e558c77f1bd6ceb11" 9 | 10 | # Use sqlite3 as the database for Active Record 11 | # gem "sqlite3", "~> 1.4" 12 | gem "pg" 13 | 14 | # Use the Puma web server [https://github.com/puma/puma] 15 | gem "puma", "~> 6.0" 16 | 17 | # Build JSON APIs with ease [https://github.com/rails/jbuilder] 18 | # gem "jbuilder" 19 | 20 | # Use Redis adapter to run Action Cable in production 21 | # gem "redis", "~> 4.0" 22 | 23 | # Use Kredis to get higher-level data types in Redis [https://github.com/rails/kredis] 24 | # gem "kredis" 25 | 26 | # Use Active Model has_secure_password [https://guides.rubyonrails.org/active_model_basics.html#securepassword] 27 | # gem "bcrypt", "~> 3.1.7" 28 | 29 | # Windows does not include zoneinfo files, so bundle the tzinfo-data gem 30 | gem "tzinfo-data", platforms: %i[ mingw mswin x64_mingw jruby ] 31 | 32 | # Reduces boot times through caching; required in config/boot.rb 33 | gem "bootsnap", require: false 34 | 35 | # Use Active Storage variants [https://guides.rubyonrails.org/active_storage_overview.html#transforming-images] 36 | # gem "image_processing", "~> 1.2" 37 | 38 | # Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible 39 | # gem "rack-cors" 40 | gem 'redis' 41 | gem "sidekiq", "~> 7.1" 42 | 43 | group :development, :test do 44 | # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem 45 | gem "debug", platforms: %i[ mri mingw x64_mingw ] 46 | gem "timecop", "~> 0.9.8" 47 | gem "mock_redis" 48 | end 49 | 50 | group :development do 51 | # Speed up commands on slow machines / big apps [https://github.com/rails/spring] 52 | # gem "spring" 53 | end 54 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GIT 2 | remote: https://github.com/rails/rails.git 3 | revision: 5cf742ef514b2184eb3d617e558c77f1bd6ceb11 4 | ref: 5cf742ef514b2184eb3d617e558c77f1bd6ceb11 5 | specs: 6 | actioncable (7.1.0.alpha) 7 | actionpack (= 7.1.0.alpha) 8 | activesupport (= 7.1.0.alpha) 9 | nio4r (~> 2.0) 10 | websocket-driver (>= 0.6.1) 11 | zeitwerk (~> 2.6) 12 | actionmailbox (7.1.0.alpha) 13 | actionpack (= 7.1.0.alpha) 14 | activejob (= 7.1.0.alpha) 15 | activerecord (= 7.1.0.alpha) 16 | activestorage (= 7.1.0.alpha) 17 | activesupport (= 7.1.0.alpha) 18 | mail (>= 2.7.1) 19 | net-imap 20 | net-pop 21 | net-smtp 22 | actionmailer (7.1.0.alpha) 23 | actionpack (= 7.1.0.alpha) 24 | actionview (= 7.1.0.alpha) 25 | activejob (= 7.1.0.alpha) 26 | activesupport (= 7.1.0.alpha) 27 | mail (~> 2.5, >= 2.5.4) 28 | net-imap 29 | net-pop 30 | net-smtp 31 | rails-dom-testing (~> 2.2) 32 | actionpack (7.1.0.alpha) 33 | actionview (= 7.1.0.alpha) 34 | activesupport (= 7.1.0.alpha) 35 | nokogiri (>= 1.8.5) 36 | rack (>= 2.2.4) 37 | rack-session (>= 1.0.1) 38 | rack-test (>= 0.6.3) 39 | rails-dom-testing (~> 2.2) 40 | rails-html-sanitizer (~> 1.6) 41 | actiontext (7.1.0.alpha) 42 | actionpack (= 7.1.0.alpha) 43 | activerecord (= 7.1.0.alpha) 44 | activestorage (= 7.1.0.alpha) 45 | activesupport (= 7.1.0.alpha) 46 | globalid (>= 0.6.0) 47 | nokogiri (>= 1.8.5) 48 | actionview (7.1.0.alpha) 49 | activesupport (= 7.1.0.alpha) 50 | builder (~> 3.1) 51 | erubi (~> 1.11) 52 | rails-dom-testing (~> 2.2) 53 | rails-html-sanitizer (~> 1.6) 54 | activejob (7.1.0.alpha) 55 | activesupport (= 7.1.0.alpha) 56 | globalid (>= 0.3.6) 57 | activemodel (7.1.0.alpha) 58 | activesupport (= 7.1.0.alpha) 59 | activerecord (7.1.0.alpha) 60 | activemodel (= 7.1.0.alpha) 61 | activesupport (= 7.1.0.alpha) 62 | timeout (>= 0.4.0) 63 | activestorage (7.1.0.alpha) 64 | actionpack (= 7.1.0.alpha) 65 | activejob (= 7.1.0.alpha) 66 | activerecord (= 7.1.0.alpha) 67 | activesupport (= 7.1.0.alpha) 68 | marcel (~> 1.0) 69 | activesupport (7.1.0.alpha) 70 | base64 71 | concurrent-ruby (~> 1.0, >= 1.0.2) 72 | connection_pool (>= 2.2.5) 73 | drb 74 | i18n (>= 1.6, < 2) 75 | minitest (>= 5.1) 76 | mutex_m 77 | tzinfo (~> 2.0) 78 | rails (7.1.0.alpha) 79 | actioncable (= 7.1.0.alpha) 80 | actionmailbox (= 7.1.0.alpha) 81 | actionmailer (= 7.1.0.alpha) 82 | actionpack (= 7.1.0.alpha) 83 | actiontext (= 7.1.0.alpha) 84 | actionview (= 7.1.0.alpha) 85 | activejob (= 7.1.0.alpha) 86 | activemodel (= 7.1.0.alpha) 87 | activerecord (= 7.1.0.alpha) 88 | activestorage (= 7.1.0.alpha) 89 | activesupport (= 7.1.0.alpha) 90 | bundler (>= 1.15.0) 91 | railties (= 7.1.0.alpha) 92 | railties (7.1.0.alpha) 93 | actionpack (= 7.1.0.alpha) 94 | activesupport (= 7.1.0.alpha) 95 | irb 96 | rackup (>= 1.0.0) 97 | rake (>= 12.2) 98 | thor (~> 1.0, >= 1.2.2) 99 | zeitwerk (~> 2.6) 100 | 101 | GEM 102 | remote: https://rubygems.org/ 103 | specs: 104 | base64 (0.1.1) 105 | bootsnap (1.16.0) 106 | msgpack (~> 1.2) 107 | builder (3.2.4) 108 | concurrent-ruby (1.2.2) 109 | connection_pool (2.4.1) 110 | crass (1.0.6) 111 | date (3.3.3) 112 | debug (1.8.0) 113 | irb (>= 1.5.0) 114 | reline (>= 0.3.1) 115 | drb (2.1.1) 116 | ruby2_keywords 117 | erubi (1.12.0) 118 | globalid (1.2.0) 119 | activesupport (>= 6.1) 120 | i18n (1.14.1) 121 | concurrent-ruby (~> 1.0) 122 | io-console (0.6.0) 123 | irb (1.7.4) 124 | reline (>= 0.3.6) 125 | loofah (2.21.3) 126 | crass (~> 1.0.2) 127 | nokogiri (>= 1.12.0) 128 | mail (2.8.1) 129 | mini_mime (>= 0.1.1) 130 | net-imap 131 | net-pop 132 | net-smtp 133 | marcel (1.0.2) 134 | mini_mime (1.1.5) 135 | mini_portile2 (2.8.4) 136 | minitest (5.19.0) 137 | mock_redis (0.37.0) 138 | msgpack (1.7.2) 139 | mutex_m (0.1.2) 140 | net-imap (0.3.7) 141 | date 142 | net-protocol 143 | net-pop (0.1.2) 144 | net-protocol 145 | net-protocol (0.2.1) 146 | timeout 147 | net-smtp (0.3.3) 148 | net-protocol 149 | nio4r (2.5.9) 150 | nokogiri (1.15.4) 151 | mini_portile2 (~> 2.8.2) 152 | racc (~> 1.4) 153 | pg (1.5.3) 154 | puma (6.3.1) 155 | nio4r (~> 2.0) 156 | racc (1.7.1) 157 | rack (3.0.8) 158 | rack-session (2.0.0) 159 | rack (>= 3.0.0) 160 | rack-test (2.1.0) 161 | rack (>= 1.3) 162 | rackup (2.1.0) 163 | rack (>= 3) 164 | webrick (~> 1.8) 165 | rails-dom-testing (2.2.0) 166 | activesupport (>= 5.0.0) 167 | minitest 168 | nokogiri (>= 1.6) 169 | rails-html-sanitizer (1.6.0) 170 | loofah (~> 2.21) 171 | nokogiri (~> 1.14) 172 | rake (13.0.6) 173 | redis (4.8.1) 174 | redis-client (0.16.0) 175 | connection_pool 176 | reline (0.3.8) 177 | io-console (~> 0.5) 178 | ruby2_keywords (0.0.5) 179 | sidekiq (7.1.2) 180 | concurrent-ruby (< 2) 181 | connection_pool (>= 2.3.0) 182 | rack (>= 2.2.4) 183 | redis-client (>= 0.14.0) 184 | thor (1.2.2) 185 | timecop (0.9.8) 186 | timeout (0.4.0) 187 | tzinfo (2.0.6) 188 | concurrent-ruby (~> 1.0) 189 | webrick (1.8.1) 190 | websocket-driver (0.7.6) 191 | websocket-extensions (>= 0.1.0) 192 | websocket-extensions (0.1.5) 193 | zeitwerk (2.6.11) 194 | 195 | PLATFORMS 196 | x86_64-linux 197 | 198 | DEPENDENCIES 199 | bootsnap 200 | debug 201 | mock_redis 202 | pg 203 | puma (~> 6.0) 204 | rails! 205 | redis 206 | sidekiq (~> 7.1) 207 | timecop (~> 0.9.8) 208 | tzinfo-data 209 | 210 | RUBY VERSION 211 | ruby 3.3.0.preview2 212 | 213 | BUNDLED WITH 214 | 2.4.19 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rinha Backend - API 2 | 3 | This project was inspired by a community driven challenge that ran on August 2023: 4 | 5 | Challenge [INSTRUCTIONS](https://github.com/zanfranceschi/rinha-de-backend-2023-q3/blob/main/INSTRUCOES.md): 6 | 7 | Challenge [STRESS TEST](https://github.com/zanfranceschi/rinha-de-backend-2023-q3/blob/main/stress-test/run-test.sh) 8 | 9 | Unfortunatelly I only heard about it a few days after it was closed, but I decided to try it out here. 10 | 11 | The API itself is not the challenge, but to make it fit within the confines of a measle 1.5 vCPU and 3GB of RAM, and endure a very heavy, almost DDoS-like Gatling stress test. It's brutal test. 12 | 13 | The goal of this version was to make a fully Rails API based app, with enough speed optimizations without completely breaking the framework. Some tricks are not recommended for real production usage, but it leverages the fact that this is a performance oriented challenge. 14 | 15 | This version is OVERENGINEERED. I already missed the deadline anyway, so I went all in in experimentation. I implemented Rails.cache with Redis and async jobs with Sidekiq to queue up the inserts and another job to flush bulk inserts. You don't need it to peak in this stress test. 16 | 17 | [@lazaronixon](https://github.com/lazaronixon/rinha_de_backend/) did a much simpler and straight forward version that already maxes out the stress test insertion criteria. But this is for fun. 18 | 19 | I did had a discovery: I spent hours testing this thinking that my version had some bug that was keeping it down, below 20k even. But turns out that this line in docker-compose.yml is not working properly and I have no idea why: 20 | 21 | volumes: 22 | - ./postgresql.conf:/docker-entrypoint-initdb.d/postgresql.conf 23 | 24 | In this config I had set a high max_connections, but the container was actually loading with just the default 100. That was the problem. You can check the database by connecting to it directly after loading docker compose up: 25 | 26 | ❯ docker-compose exec postgres psql -U postgres 27 | psql (15.4 (Debian 15.4-1.pgdg120+1)) 28 | Type "help" for help. 29 | 30 | postgres=# SHOW max_connections; 31 | max_connections 32 | ----------------- 33 | 100 34 | (1 row) 35 | 36 | To make sure it's actually increasing the max connections, I had to do: 37 | 38 | command: postgres -c 'max_connections=450' 39 | 40 | ### RESULTS 41 | 42 | ``` 43 | ================================================================================ 44 | ---- Global Information -------------------------------------------------------- 45 | > request count 108213 (OK=97044 KO=11169 ) 46 | > min response time 0 (OK=0 KO=3 ) 47 | > max response time 60001 (OK=60000 KO=60001 ) 48 | > mean response time 1855 (OK=1661 KO=3536 ) 49 | > std deviation 8622 (OK=7720 KO=14116 ) 50 | > response time 50th percentile 25 (OK=90 KO=7 ) 51 | > response time 75th percentile 602 (OK=684 KO=9 ) 52 | > response time 95th percentile 1999 (OK=1980 KO=60000 ) 53 | > response time 99th percentile 54713 (OK=52025 KO=60001 ) 54 | > mean requests/sec 468.455 (OK=420.104 KO=48.351) 55 | ---- Response Time Distribution ------------------------------------------------ 56 | > t < 800 ms 76144 ( 70%) 57 | > 800 ms <= t < 1200 ms 9516 ( 9%) 58 | > t >= 1200 ms 11384 ( 11%) 59 | > failed 11169 ( 10%) 60 | ---- Errors -------------------------------------------------------------------- 61 | > j.i.IOException: Premature close 10512 (94.12%) 62 | > Request timeout to localhost/127.0.0.1:9999 after 60000 ms 581 ( 5.20%) 63 | > status.find.in(201,422,400), but actually found 502 51 ( 0.46%) 64 | > status.find.in([200, 209], 304), found 502 16 ( 0.14%) 65 | > status.find.is(400), but actually found 502 6 ( 0.05%) 66 | > status.find.in(201,422,400), but actually found 504 3 ( 0.03%) 67 | ================================================================================ 68 | ``` 69 | 70 | It finished with almost 40K!! So this puts it near the top!! 71 | 72 | ![graph 1](imgs/graphs1.png) 73 | 74 | ![graph 2](imgs/graphs2.png) 75 | 76 | 77 | ### HOW TO RUN 78 | 79 | To run this application: 80 | 81 | docker-compose up 82 | 83 | Just for the first time run: 84 | 85 | docker-compose exec api1 rails db:create 86 | docker-compose exec api1 rails db:migrate 87 | 88 | Application should respond at: 89 | 90 | http://0.0.0.0:9999 91 | 92 | 93 | Run stress test: 94 | 95 | wget https://repo1.maven.org/maven2/io/gatling/highcharts/gatling-charts-highcharts-bundle/3.9.5/gatling-charts-highcharts-bundle-3.9.5-bundle.zip 96 | unzip gatling-charts-highcharts-bundle-3.9.5-bundle.zip 97 | sudo mv gatling-charts-highcharts-bundle-3.9.5-bundle /opt 98 | sudo ln -s /opt/gatling-charts-highcharts-bundle-3.9.5-bundle /opt/gatling 99 | 100 | cd .. 101 | git clone https://github.com/zanfranceschi/rinha-de-backend-2023-q3.git 102 | cd rinha-de-backend-2023-q3/stress-test 103 | 104 | Edit the stress-test run-test.sh variables accordingly: 105 | 106 | GATLING_BIN_DIR=/opt/gatling/bin 107 | 108 | WORKSPACE=$HOME/Projects/rinha-de-backend-2023-q3/stress-test 109 | 110 | Run the stress test: 111 | 112 | ./run-test.sh # after docker-compose up 113 | 114 | On AWS EC2, create a t2.medium instance (2vCPUs 4GB RAM), download the .pem key: 115 | 116 | chmod 400 yourkey.pem 117 | ssh -i yourkey.pem -o IdentitiesOnly=yes ec2-user@yourinstanceaddress.com 118 | 119 | SSH in: 120 | 121 | # Install Docker 122 | sudo amazon-linux-extras install docker 123 | sudo service docker start 124 | sudo usermod -a -G docker ec2-user 125 | 126 | # Install Docker Compose 127 | sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose 128 | sudo chmod +x /usr/local/bin/docker-compose 129 | 130 | # install Git 131 | sudo yum update -y 132 | sudo yum install git -y 133 | git clone https://github.com/akitaonrails/rinhabackend-rails-api.git 134 | 135 | # run docker compose 136 | cd rinhabackend-rails-api 137 | docker-compose up --force-recreate -d 138 | 139 | Don't forget to change run-test.sh and RinhaBackendSimulation.scala to add your own PATH and HOST name (in case you're running on AWS EC2). Also don't forget to edit docker-compose.yml to not build locally, but fetch the image from docker.io. 140 | 141 | And in case you're wondering how a more "realistic" setup would look like, I created a "docker-compose-ideal.yml" that breaks the 1.5 vCPU and 3GB of RAM to a whopping 30 vCPUs and 22GB of RAM. If you have the horsepower for that, run: 142 | 143 | docker-compose -f docker-compose-ideal.yml up --force-recreate --build 144 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # Add your own tasks in files placed in lib/tasks ending in .rake, 2 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 3 | 4 | require_relative "config/application" 5 | 6 | Rails.application.load_tasks 7 | -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | class ApplicationController < ActionController::API 2 | end 3 | -------------------------------------------------------------------------------- /app/controllers/concerns/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/app/controllers/concerns/.keep -------------------------------------------------------------------------------- /app/controllers/pessoas_controller.rb: -------------------------------------------------------------------------------- 1 | class PessoasController < ApplicationController 2 | JSON_FIELDS = %i[id apelido nome nascimento stack].freeze 3 | CACHE_EXPIRES = ENV.fetch('CACHE_EXPIRES_SECONDS', 2).to_i.seconds 4 | 5 | before_action :set_pessoa, only: %i[show update destroy] 6 | before_action :validate_params, only: %i[create update] 7 | 8 | # GET /pessoas?t=query 9 | def index 10 | if params[:t] 11 | pessoas = Pessoa.search(params[:t]).limit(50) 12 | 13 | # # HTTP caching 14 | # - no need here because the stress test don't call the same URL twice 15 | # fresh_when etag: Digest::MD5.hexdigest(pessoas.to_json) 16 | 17 | render json: pessoas 18 | else 19 | head :bad_request 20 | end 21 | end 22 | 23 | # GET /pessoas/1 24 | def show 25 | if @pessoa 26 | # HTTP caching 27 | # - no need here because the stress test don't call the same URL twice 28 | # fresh_when @pessoa 29 | render json: @pessoa, only: JSON_FIELDS 30 | else 31 | head :not_found 32 | end 33 | end 34 | 35 | def contagem 36 | # hack: wait for Sidekiq to empty its queue before providing the final count 37 | # the stress test don't count this request for performance 38 | PessoaJob.perform_async(:flush, nil) 39 | sleep 3 40 | 41 | render plain: "batch queue: #{Rails.cache.read('insert_buffer-counter')} total: #{Pessoa.count}" 42 | end 43 | 44 | # POST /pessoas 45 | def create 46 | @pessoa = Pessoa.new(pessoa_params) 47 | 48 | if @pessoa.valid? 49 | # hack to find duplicate without hitting the db 50 | if Rails.cache.fetch("a/#{@pessoa.apelido}") 51 | head :unprocessable_entity 52 | return 53 | end 54 | 55 | PessoaJob.perform_async(:create, pessoa_params.merge(id: @pessoa.id).to_h) 56 | 57 | # hack so SHOW works before Sidekiq hits the db eventually 58 | Rails.cache.write("p/#{@pessoa.id}", @pessoa, expires_in: CACHE_EXPIRES) 59 | Rails.cache.write("a/#{@pessoa.apelido}", '', expires_in: CACHE_EXPIRES) 60 | 61 | # head :created, location: pessoa_url(@pessoa) 62 | head :created, location: "http://localhost:9999/pessoas/#{@pessoa.id}" 63 | else 64 | head :unprocessable_entity 65 | end 66 | end 67 | 68 | # PATCH/PUT /pessoas/1 69 | def update 70 | if @pessoa.valid? 71 | PessoaJob.perform_async(:update, pessoa_params.merge(id: params[:id]).to_h) 72 | head :ok 73 | else 74 | head :unprocessable_entity 75 | end 76 | end 77 | 78 | # DELETE /pessoas/1 79 | def destroy 80 | @pessoa.destroy 81 | end 82 | 83 | private 84 | 85 | # Use callbacks to share common setup or constraints between actions. 86 | def set_pessoa 87 | @pessoa = Rails.cache.fetch("p/#{params[:id]}", expires_in: CACHE_EXPIRES) do 88 | Pessoa.find_by(id: params[:id]) 89 | end 90 | end 91 | 92 | # Only allow a list of trusted parameters through. 93 | def pessoa_params 94 | params.require(:pessoa).permit(:apelido, :nome, :nascimento, stack: []) 95 | end 96 | 97 | def validate_params 98 | p = pessoa_params 99 | 100 | unless p[:apelido].is_a?(String) && p[:nome].is_a?(String) 101 | head :bad_request 102 | return 103 | end 104 | 105 | begin 106 | pessoa_params[:nascimento] = Time.parse(pessoa_params[:nascimento]) 107 | rescue 108 | head :bad_request 109 | return 110 | end 111 | 112 | if p[:stack] && !p[:stack].all? { |elem| elem.is_a?(String) } 113 | head :bad_request 114 | return 115 | end 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /app/jobs/pessoa_flush_job.rb: -------------------------------------------------------------------------------- 1 | class PessoaFlushJob 2 | include Sidekiq::Job 3 | 4 | sidekiq_options queue: 'flush' 5 | 6 | def buffer 7 | @@buffer ||= RedisQueue.new(PessoaJob::BUFFER_KEY) 8 | end 9 | 10 | def perform 11 | buffer_snapshot = buffer.fetch_batch(PessoaJob::BUFFER_SIZE) 12 | buffer_snapshot.each do |b| 13 | b['stack'] = nil unless b.key?('stack') 14 | b['nascimento'] = nil unless b.key?('nascimento') 15 | end 16 | 17 | begin 18 | Pessoa.insert_all(buffer_snapshot, returning: false) 19 | rescue => e 20 | Sidekiq.logger.error "ERROR: #{e} #{buffer_snapshot}" 21 | ensure 22 | buffer.counter 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /app/jobs/pessoa_job.rb: -------------------------------------------------------------------------------- 1 | class PessoaJob 2 | include Sidekiq::Job 3 | 4 | BUFFER_SIZE = ENV.fetch('JOB_BATCH_SIZE', 10).to_i 5 | BUFFER_KEY = 'insert_buffer'.freeze 6 | 7 | sidekiq_options queue: BUFFER_KEY 8 | 9 | def buffer 10 | @@buffer ||= RedisQueue.new(BUFFER_KEY) 11 | end 12 | 13 | def perform(action, params) 14 | case action 15 | when 'create' 16 | buffer.push(params) unless params.blank? 17 | PessoaFlushJob.perform_async if buffer.size >= BUFFER_SIZE 18 | when 'flush' 19 | PessoaFlushJob.perform_async 20 | when 'update' 21 | Pessoa.upsert(params, id: params[:id]) 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /app/models/application_record.rb: -------------------------------------------------------------------------------- 1 | class ApplicationRecord < ActiveRecord::Base 2 | primary_abstract_class 3 | end 4 | -------------------------------------------------------------------------------- /app/models/concerns/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/app/models/concerns/.keep -------------------------------------------------------------------------------- /app/models/pessoa.rb: -------------------------------------------------------------------------------- 1 | # Credit: https://github.com/lazaronixon/rinha_de_backend/blob/main/app/models/pessoa.rb 2 | class Pessoa < ApplicationRecord 3 | self.ignored_columns = %w[searchable] 4 | 5 | before_validation :set_id, on: :create 6 | 7 | serialize :stack, type: Array, coder: TagCoder 8 | scope :search, ->(value) { where('pessoas.searchable ILIKE ?', "%#{value}%") } 9 | 10 | validates :apelido, presence: true, length: { maximum: 32 } 11 | validates :nome, presence: true, length: { maximum: 100 } 12 | validates :nascimento, presence: true 13 | 14 | private 15 | 16 | def set_id 17 | self.id ||= SecureRandom.uuid 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /bin/bundle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'bundle' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "rubygems" 12 | 13 | m = Module.new do 14 | module_function 15 | 16 | def invoked_as_script? 17 | File.expand_path($0) == File.expand_path(__FILE__) 18 | end 19 | 20 | def env_var_version 21 | ENV["BUNDLER_VERSION"] 22 | end 23 | 24 | def cli_arg_version 25 | return unless invoked_as_script? # don't want to hijack other binstubs 26 | return unless "update".start_with?(ARGV.first || " ") # must be running `bundle update` 27 | bundler_version = nil 28 | update_index = nil 29 | ARGV.each_with_index do |a, i| 30 | if update_index && update_index.succ == i && a =~ Gem::Version::ANCHORED_VERSION_PATTERN 31 | bundler_version = a 32 | end 33 | next unless a =~ /\A--bundler(?:[= ](#{Gem::Version::VERSION_PATTERN}))?\z/ 34 | bundler_version = $1 35 | update_index = i 36 | end 37 | bundler_version 38 | end 39 | 40 | def gemfile 41 | gemfile = ENV["BUNDLE_GEMFILE"] 42 | return gemfile if gemfile && !gemfile.empty? 43 | 44 | File.expand_path("../Gemfile", __dir__) 45 | end 46 | 47 | def lockfile 48 | lockfile = 49 | case File.basename(gemfile) 50 | when "gems.rb" then gemfile.sub(/\.rb$/, ".locked") 51 | else "#{gemfile}.lock" 52 | end 53 | File.expand_path(lockfile) 54 | end 55 | 56 | def lockfile_version 57 | return unless File.file?(lockfile) 58 | lockfile_contents = File.read(lockfile) 59 | return unless lockfile_contents =~ /\n\nBUNDLED WITH\n\s{2,}(#{Gem::Version::VERSION_PATTERN})\n/ 60 | Regexp.last_match(1) 61 | end 62 | 63 | def bundler_requirement 64 | @bundler_requirement ||= 65 | env_var_version || 66 | cli_arg_version || 67 | bundler_requirement_for(lockfile_version) 68 | end 69 | 70 | def bundler_requirement_for(version) 71 | return "#{Gem::Requirement.default}.a" unless version 72 | 73 | bundler_gem_version = Gem::Version.new(version) 74 | 75 | bundler_gem_version.approximate_recommendation 76 | end 77 | 78 | def load_bundler! 79 | ENV["BUNDLE_GEMFILE"] ||= gemfile 80 | 81 | activate_bundler 82 | end 83 | 84 | def activate_bundler 85 | gem_error = activation_error_handling do 86 | gem "bundler", bundler_requirement 87 | end 88 | return if gem_error.nil? 89 | require_error = activation_error_handling do 90 | require "bundler/version" 91 | end 92 | return if require_error.nil? && Gem::Requirement.new(bundler_requirement).satisfied_by?(Gem::Version.new(Bundler::VERSION)) 93 | warn "Activating bundler (#{bundler_requirement}) failed:\n#{gem_error.message}\n\nTo install the version of bundler this project requires, run `gem install bundler -v '#{bundler_requirement}'`" 94 | exit 42 95 | end 96 | 97 | def activation_error_handling 98 | yield 99 | nil 100 | rescue StandardError, LoadError => e 101 | e 102 | end 103 | end 104 | 105 | m.load_bundler! 106 | 107 | if m.invoked_as_script? 108 | load Gem.bin_path("bundler", "bundle") 109 | end 110 | -------------------------------------------------------------------------------- /bin/rails: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_PATH = File.expand_path("../config/application", __dir__) 3 | require_relative "../config/boot" 4 | require "rails/commands" 5 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require_relative "../config/boot" 3 | require "rake" 4 | Rake.application.run 5 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require "fileutils" 3 | 4 | # path to your application root. 5 | APP_ROOT = File.expand_path("..", __dir__) 6 | 7 | def system!(*args) 8 | system(*args) || abort("\n== Command #{args} failed ==") 9 | end 10 | 11 | FileUtils.chdir APP_ROOT do 12 | # This script is a way to set up or update your development environment automatically. 13 | # This script is idempotent, so that you can run it at any time and get an expectable outcome. 14 | # Add necessary setup steps to this file. 15 | 16 | puts "== Installing dependencies ==" 17 | system! "gem install bundler --conservative" 18 | system("bundle check") || system!("bundle install") 19 | 20 | # puts "\n== Copying sample files ==" 21 | # unless File.exist?("config/database.yml") 22 | # FileUtils.cp "config/database.yml.sample", "config/database.yml" 23 | # end 24 | 25 | puts "\n== Preparing database ==" 26 | system! "bin/rails db:prepare" 27 | 28 | puts "\n== Removing old logs and tempfiles ==" 29 | system! "bin/rails log:clear tmp:clear" 30 | 31 | puts "\n== Restarting application server ==" 32 | system! "bin/rails restart" 33 | end 34 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | # This file is used by Rack-based servers to start the application. 2 | 3 | require_relative "config/environment" 4 | 5 | run Rails.application 6 | Rails.application.load_server 7 | -------------------------------------------------------------------------------- /config/application.rb: -------------------------------------------------------------------------------- 1 | require_relative "boot" 2 | 3 | require "rails" 4 | # Pick the frameworks you want: 5 | require "active_model/railtie" 6 | # require "active_job/railtie" 7 | require "active_record/railtie" # Uncomment for ActiveRecord 8 | # require "active_storage/engine" 9 | require "action_controller/railtie" 10 | # require "action_view/railtie" 11 | # require "action_mailer/railtie" # Comment out for ActionMailer 12 | # require "action_mailbox/engine" # Comment out for ActionMailbox 13 | # require "action_text/engine" # Comment out for ActionText 14 | # require "action_cable/engine" # Comment out for ActionCable 15 | # require "sprockets/railtie" 16 | # require "rails/test_unit/railtie" # Comment out for TestUnit 17 | 18 | # Require the gems listed in Gemfile, including any gems 19 | # you've limited to :test, :development, or :production. 20 | Bundler.require(*Rails.groups) 21 | 22 | require './lib/tag_coder' 23 | require './lib/redis_queue' 24 | 25 | module Rinhabackend 26 | class Application < Rails::Application 27 | # Initialize configuration defaults for originally generated Rails version. 28 | config.load_defaults 7.0 29 | 30 | # Configuration for the application, engines, and railties goes here. 31 | # 32 | # These settings can be overridden in specific environments using the files 33 | # in config/environments, which are processed later. 34 | # 35 | # config.time_zone = "Central Time (US & Canada)" 36 | # config.eager_load_paths << Rails.root.join("extras") 37 | 38 | # Only loads a smaller set of middleware suitable for API only apps. 39 | # Middleware like session, flash, cookies can be added back manually. 40 | # Skip views, helpers and assets when generating a new resource. 41 | config.api_only = true 42 | 43 | config.active_record.schema_format = :sql 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /config/boot.rb: -------------------------------------------------------------------------------- 1 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) 2 | 3 | require "bundler/setup" # Set up gems listed in the Gemfile. 4 | require "bootsnap/setup" # Speed up boot time by caching expensive operations. 5 | -------------------------------------------------------------------------------- /config/credentials.yml.enc: -------------------------------------------------------------------------------- 1 | mMcBvbAXBC6JOdV9aE2X9+ocQzMPN17WjCcKheLHobb8OeUqIsW8wb4XSIdOZfbrihYJz35/gegk6vMfF5PFz+radB/9R5ZPs1rj1rd0Op5HGvXt+J5vdrU1gXjBuvFqEMQob1gF8lrJM3nVIw91Jm1zbbe8r0BPb3fr9vYl+ZNSt3khq4mtmptHbcx+kKRV2c7d6VqFRUXfUHQ4z3LrvW8/1Rp3FH3GqUdC6+ApyhkIbKATfrhmde4yZhkwDY3W2JlRRQrrNZezu61+jn0MkhRzp89i2MEvh9N9awjiFw/dMWLZ4Bs1CAA6vhtC6ELz6FJoN1371pmFYakMGEfuN/LtDSQ4GXNk/F4yfZMDkAbOAwp4XAgrlxBpA7+Fr0XAAB1BX5CaqePjaFuH/Eu57RYV7kOtzx0Yy8V3--rn+NYKoD4xIvxbga--45ClPLxtpkrQyKg5RIYgeQ== -------------------------------------------------------------------------------- /config/database.yml: -------------------------------------------------------------------------------- 1 | # SQLite. Versions 3.8.0 and up are supported. 2 | # gem install sqlite3 3 | # 4 | # Ensure the SQLite 3 gem is defined in your Gemfile 5 | # gem "sqlite3" 6 | # 7 | default: &default 8 | adapter: postgresql 9 | pool: <%= ENV.fetch("DB_POOL", 5) %> 10 | host: <%= ENV.fetch("DB_HOST", 'localhost') %> 11 | username: <%= ENV.fetch("DB_USERNAME", 'postgres') %> 12 | password: <%= ENV.fetch("DB_PASSWORD", 'password') %> 13 | timeout: <%= ENV.fetch("DB_TIMEOUT", 5000) %> 14 | 15 | development: 16 | <<: *default 17 | database: <%= ENV.fetch("DB_NAME_DEV", 'rinhabackend_dev') %> 18 | 19 | # Warning: The database defined as "test" will be erased and 20 | # re-generated from your development database when you run "rake". 21 | # Do not set this db to the same as development or production. 22 | test: 23 | <<: *default 24 | database: <%= ENV.fetch("DB_NAME_TEST", 'rinhabackend_test') %> 25 | 26 | production: 27 | <<: *default 28 | database: <%= ENV.fetch("DB_NAME_PROD", 'rinhabackend') %> 29 | -------------------------------------------------------------------------------- /config/environment.rb: -------------------------------------------------------------------------------- 1 | # Load the Rails application. 2 | require_relative "application" 3 | 4 | # Initialize the Rails application. 5 | Rails.application.initialize! 6 | -------------------------------------------------------------------------------- /config/environments/development.rb: -------------------------------------------------------------------------------- 1 | require "active_support/core_ext/integer/time" 2 | 3 | Rails.application.configure do 4 | # Settings specified here will take precedence over those in config/application.rb. 5 | 6 | # In the development environment your application's code is reloaded any time 7 | # it changes. This slows down response time but is perfect for development 8 | # since you don't have to restart the web server when you make code changes. 9 | config.cache_classes = false 10 | 11 | # Do not eager load code on boot. 12 | config.eager_load = false 13 | 14 | # Show full error reports. 15 | config.consider_all_requests_local = true 16 | 17 | # Enable server timing 18 | config.server_timing = true 19 | 20 | # Enable/disable caching. By default caching is disabled. 21 | # Run rails dev:cache to toggle caching. 22 | if Rails.root.join("tmp/caching-dev.txt").exist? 23 | config.cache_store = :memory_store 24 | config.public_file_server.headers = { 25 | "Cache-Control" => "public, max-age=#{2.days.to_i}" 26 | } 27 | else 28 | config.action_controller.perform_caching = false 29 | 30 | config.cache_store = :null_store 31 | end 32 | 33 | # Store uploaded files on the local file system (see config/storage.yml for options). 34 | # config.active_storage.service = :local 35 | 36 | # Don't care if the mailer can't send. 37 | # config.action_mailer.raise_delivery_errors = false 38 | 39 | # config.action_mailer.perform_caching = false 40 | 41 | # Print deprecation notices to the Rails logger. 42 | config.active_support.deprecation = :log 43 | 44 | # Raise exceptions for disallowed deprecations. 45 | config.active_support.disallowed_deprecation = :raise 46 | 47 | # Tell Active Support which deprecation messages to disallow. 48 | config.active_support.disallowed_deprecation_warnings = [] 49 | 50 | # Raise an error on page load if there are pending migrations. 51 | config.active_record.migration_error = :page_load 52 | 53 | # Highlight code that triggered database queries in logs. 54 | config.active_record.verbose_query_logs = true 55 | 56 | 57 | # Raises error for missing translations. 58 | # config.i18n.raise_on_missing_translations = true 59 | 60 | # Annotate rendered view with file names. 61 | # config.action_view.annotate_rendered_view_with_filenames = true 62 | 63 | # Uncomment if you wish to allow Action Cable access from any origin. 64 | # config.action_cable.disable_request_forgery_protection = true 65 | end 66 | -------------------------------------------------------------------------------- /config/environments/production.rb: -------------------------------------------------------------------------------- 1 | require "active_support/core_ext/integer/time" 2 | 3 | Rails.application.configure do 4 | # Settings specified here will take precedence over those in config/application.rb. 5 | 6 | # Code is not reloaded between requests. 7 | config.cache_classes = true 8 | 9 | # Eager load code on boot. This eager loads most of Rails and 10 | # your application in memory, allowing both threaded web servers 11 | # and those relying on copy on write to perform better. 12 | # Rake tasks automatically ignore this option for performance. 13 | config.eager_load = true 14 | 15 | # Full error reports are disabled and caching is turned on. 16 | config.consider_all_requests_local = false 17 | 18 | # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] 19 | # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). 20 | # config.require_master_key = true 21 | 22 | # Disable serving static files from the `/public` folder by default since 23 | # Apache or NGINX already handles this. 24 | config.public_file_server.enabled = ENV["RAILS_SERVE_STATIC_FILES"].present? 25 | 26 | # Enable serving of images, stylesheets, and JavaScripts from an asset server. 27 | # config.asset_host = "http://assets.example.com" 28 | 29 | # Specifies the header that your server uses for sending files. 30 | # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for Apache 31 | # config.action_dispatch.x_sendfile_header = "X-Accel-Redirect" # for NGINX 32 | 33 | # Store uploaded files on the local file system (see config/storage.yml for options). 34 | # config.active_storage.service = :local 35 | 36 | # Mount Action Cable outside main process or domain. 37 | # config.action_cable.mount_path = nil 38 | # config.action_cable.url = "wss://example.com/cable" 39 | # config.action_cable.allowed_request_origins = [ "http://example.com", /http:\/\/example.*/ ] 40 | 41 | # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. 42 | # config.force_ssl = true 43 | 44 | # Include generic and useful information about system operation, but avoid logging too much 45 | # information to avoid inadvertent exposure of personally identifiable information (PII). 46 | config.log_level = :warn 47 | 48 | # Prepend all log lines with the following tags. 49 | config.log_tags = [ :request_id ] 50 | 51 | # Use a different cache store in production. 52 | config.cache_store = :redis_cache_store, { url: "redis://#{ENV.fetch('REDIS_HOST', "localhost")}:6379/1", 53 | connect_timeout: 30, read_timeout: 0.5, write_timeout: 0.5, reconnect_attempts: 1, 54 | size: ENV.fetch('REDIS_POOL_SIZE', "5").to_i, timeout: ENV.fetch('REDIS_POOL_SIZE', "2").to_i } 55 | 56 | # Use a real queuing backend for Active Job (and separate queues per environment). 57 | # config.active_job.queue_adapter = :resque 58 | # config.active_job.queue_name_prefix = "rinhabackend_production" 59 | 60 | # config.action_mailer.perform_caching = false 61 | 62 | # Ignore bad email addresses and do not raise email delivery errors. 63 | # Set this to true and configure the email server for immediate delivery to raise delivery errors. 64 | # config.action_mailer.raise_delivery_errors = false 65 | 66 | # Enable locale fallbacks for I18n (makes lookups for any locale fall back to 67 | # the I18n.default_locale when a translation cannot be found). 68 | config.i18n.fallbacks = true 69 | 70 | # Don't log any deprecations. 71 | config.active_support.report_deprecations = false 72 | 73 | # Use default logging formatter so that PID and timestamp are not suppressed. 74 | config.log_formatter = ::Logger::Formatter.new 75 | 76 | # Use a different logger for distributed setups. 77 | # require "syslog/logger" 78 | # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new "app-name") 79 | 80 | if ENV["RAILS_LOG_TO_STDOUT"].present? 81 | logger = ActiveSupport::Logger.new(STDOUT) 82 | logger.formatter = config.log_formatter 83 | config.logger = ActiveSupport::TaggedLogging.new(logger) 84 | end 85 | 86 | # Do not dump schema after migrations. 87 | config.active_record.dump_schema_after_migration = false 88 | end 89 | -------------------------------------------------------------------------------- /config/environments/test.rb: -------------------------------------------------------------------------------- 1 | require "active_support/core_ext/integer/time" 2 | 3 | # The test environment is used exclusively to run your application's 4 | # test suite. You never need to work with it otherwise. Remember that 5 | # your test database is "scratch space" for the test suite and is wiped 6 | # and recreated between test runs. Don't rely on the data there! 7 | 8 | Rails.application.configure do 9 | # Settings specified here will take precedence over those in config/application.rb. 10 | 11 | # Turn false under Spring and add config.action_view.cache_template_loading = true. 12 | config.cache_classes = true 13 | 14 | # Eager loading loads your whole application. When running a single test locally, 15 | # this probably isn't necessary. It's a good idea to do in a continuous integration 16 | # system, or in some way before deploying your code. 17 | config.eager_load = ENV["CI"].present? 18 | 19 | # Configure public file server for tests with Cache-Control for performance. 20 | config.public_file_server.enabled = true 21 | config.public_file_server.headers = { 22 | "Cache-Control" => "public, max-age=#{1.hour.to_i}" 23 | } 24 | 25 | # Show full error reports and disable caching. 26 | config.consider_all_requests_local = true 27 | config.action_controller.perform_caching = true 28 | config.cache_store = :memory_store 29 | 30 | # Raise exceptions instead of rendering exception templates. 31 | config.action_dispatch.show_exceptions = false 32 | 33 | # Disable request forgery protection in test environment. 34 | config.action_controller.allow_forgery_protection = false 35 | 36 | # Store uploaded files on the local file system in a temporary directory. 37 | # config.active_storage.service = :test 38 | 39 | # config.action_mailer.perform_caching = false 40 | 41 | # Tell Action Mailer not to deliver emails to the real world. 42 | # The :test delivery method accumulates sent emails in the 43 | # ActionMailer::Base.deliveries array. 44 | # config.action_mailer.delivery_method = :test 45 | 46 | # Print deprecation notices to the stderr. 47 | config.active_support.deprecation = :stderr 48 | 49 | # Raise exceptions for disallowed deprecations. 50 | config.active_support.disallowed_deprecation = :raise 51 | 52 | # Tell Active Support which deprecation messages to disallow. 53 | config.active_support.disallowed_deprecation_warnings = [] 54 | 55 | # Raises error for missing translations. 56 | # config.i18n.raise_on_missing_translations = true 57 | 58 | # Annotate rendered view with file names. 59 | # config.action_view.annotate_rendered_view_with_filenames = true 60 | end 61 | -------------------------------------------------------------------------------- /config/initializers/cors.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Avoid CORS issues when API is called from the frontend app. 4 | # Handle Cross-Origin Resource Sharing (CORS) in order to accept cross-origin AJAX requests. 5 | 6 | # Read more: https://github.com/cyu/rack-cors 7 | 8 | # Rails.application.config.middleware.insert_before 0, Rack::Cors do 9 | # allow do 10 | # origins "example.com" 11 | # 12 | # resource "*", 13 | # headers: :any, 14 | # methods: [:get, :post, :put, :patch, :delete, :options, :head] 15 | # end 16 | # end 17 | -------------------------------------------------------------------------------- /config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Configure parameters to be filtered from the log file. Use this to limit dissemination of 4 | # sensitive information. See the ActiveSupport::ParameterFilter documentation for supported 5 | # notations and behaviors. 6 | Rails.application.config.filter_parameters += [ 7 | :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn 8 | ] 9 | -------------------------------------------------------------------------------- /config/initializers/inflections.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new inflection rules using the following format. Inflections 4 | # are locale specific, and you may define rules for as many different 5 | # locales as you wish. All of these examples are active by default: 6 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 7 | # inflect.plural /^(ox)$/i, "\\1en" 8 | # inflect.singular /^(ox)en/i, "\\1" 9 | # inflect.irregular "person", "people" 10 | # inflect.uncountable %w( fish sheep ) 11 | # end 12 | 13 | # These inflection rules are supported but not enabled by default: 14 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 15 | # inflect.acronym "RESTful" 16 | # end 17 | -------------------------------------------------------------------------------- /config/initializers/sidekiq.rb: -------------------------------------------------------------------------------- 1 | Sidekiq.configure_server do |config| 2 | config.logger.level = Logger::ERROR 3 | config.redis = { url: "redis://#{ENV['REDIS_HOST'] || "localhost"}:6379/0"} 4 | end 5 | 6 | Sidekiq.configure_client do |config| 7 | config.redis = { url: "redis://#{ENV['REDIS_HOST'] || "localhost"}:6379/0"} 8 | end 9 | 10 | Sidekiq.strict_args!(false) 11 | -------------------------------------------------------------------------------- /config/locales/en.yml: -------------------------------------------------------------------------------- 1 | # Files in the config/locales directory are used for internationalization 2 | # and are automatically loaded by Rails. If you want to use locales other 3 | # than English, add the necessary files in this directory. 4 | # 5 | # To use the locales, use `I18n.t`: 6 | # 7 | # I18n.t "hello" 8 | # 9 | # In views, this is aliased to just `t`: 10 | # 11 | # <%= t("hello") %> 12 | # 13 | # To use a different locale, set it with `I18n.locale`: 14 | # 15 | # I18n.locale = :es 16 | # 17 | # This would use the information in config/locales/es.yml. 18 | # 19 | # The following keys must be escaped otherwise they will not be retrieved by 20 | # the default I18n backend: 21 | # 22 | # true, false, on, off, yes, no 23 | # 24 | # Instead, surround them with single quotes. 25 | # 26 | # en: 27 | # "true": "foo" 28 | # 29 | # To learn more, please read the Rails Internationalization guide 30 | # available at https://guides.rubyonrails.org/i18n.html. 31 | 32 | en: 33 | hello: "Hello world" 34 | -------------------------------------------------------------------------------- /config/puma.rb: -------------------------------------------------------------------------------- 1 | # This configuration file will be evaluated by Puma. The top-level methods that 2 | # are invoked here are part of Puma's configuration DSL. For more information 3 | # about methods provided by the DSL, see https://puma.io/puma/Puma/DSL.html. 4 | 5 | # Puma can serve each request in a thread from an internal thread pool. 6 | # The `threads` method setting takes two numbers: a minimum and maximum. 7 | # Any libraries that use thread pools should be configured to match 8 | # the maximum value specified for Puma. Default is set to 5 threads for minimum 9 | # and maximum; this matches the default thread size of Active Record. 10 | max_threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 } 11 | min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count } 12 | threads min_threads_count, max_threads_count 13 | 14 | # Specifies that the worker count should equal the number of processors in production. 15 | if ENV["RAILS_ENV"] == "production" 16 | worker_count = Integer(ENV.fetch("WEB_CONCURRENCY") { Concurrent.physical_processor_count }) 17 | workers worker_count if worker_count > 1 18 | end 19 | 20 | # Specifies the `worker_timeout` threshold that Puma will use to wait before 21 | # terminating a worker in development environments. 22 | worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development" 23 | 24 | # Specifies the `port` that Puma will listen on to receive requests; default is 3000. 25 | port ENV.fetch("PORT") { 3000 } 26 | 27 | # Specifies the `environment` that Puma will run in. 28 | environment ENV.fetch("RAILS_ENV") { "development" } 29 | 30 | # Specifies the `pidfile` that Puma will use. 31 | pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } 32 | 33 | # Allow puma to be restarted by `bin/rails restart` command. 34 | plugin :tmp_restart 35 | -------------------------------------------------------------------------------- /config/routes.rb: -------------------------------------------------------------------------------- 1 | Rails.application.routes.draw do 2 | resources :pessoas 3 | # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html 4 | 5 | # Defines the root path route ("/") 6 | #root 'pessoas#index' 7 | 8 | get 'contagem-pessoas', to: 'pessoas#contagem' 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20230825134432_create_pessoas.rb: -------------------------------------------------------------------------------- 1 | class CreatePessoas < ActiveRecord::Migration[7.0] 2 | def up 3 | create_table :pessoas, id: :uuid do |t| 4 | t.string :apelido, limit: 32, null: false, index: { unique: true } 5 | t.string :nome, limit: 100, null: false 6 | t.datetime :nascimento 7 | t.string :stack 8 | t.timestamps 9 | end 10 | end 11 | 12 | def down 13 | drop_table :pessoas 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20230825151541_add_pg_trgm_extension.rb: -------------------------------------------------------------------------------- 1 | class AddPgTrgmExtension < ActiveRecord::Migration[7.0] 2 | def change 3 | enable_extension :pg_trgm 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20230827015102_add_index_pessoas_on_search.rb: -------------------------------------------------------------------------------- 1 | class AddIndexPessoasOnSearch < ActiveRecord::Migration[7.0] 2 | def change 3 | add_column :pessoas, :searchable, :virtual, type: :text, as: "nome || ' ' || apelido || ' ' || coalesce(stack, ' ')", stored: true 4 | add_index :pessoas, :searchable, using: :gist, opclass: :gist_trgm_ops 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/schema.rb: -------------------------------------------------------------------------------- 1 | # This file is auto-generated from the current state of the database. Instead 2 | # of editing this file, please use the migrations feature of Active Record to 3 | # incrementally modify your database, and then regenerate this schema definition. 4 | # 5 | # This file is the source Rails uses to define your schema when running `bin/rails 6 | # db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to 7 | # be faster and is potentially less error prone than running all of your 8 | # migrations from scratch. Old migrations may fail to apply correctly if those 9 | # migrations use external dependencies or application code. 10 | # 11 | # It's strongly recommended that you check this file into your version control system. 12 | 13 | ActiveRecord::Schema[7.0].define(version: 2023_08_27_015102) do 14 | # These are extensions that must be enabled in order to support this database 15 | enable_extension "pg_trgm" 16 | enable_extension "plpgsql" 17 | 18 | create_table "pessoas", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 19 | t.string "apelido", limit: 32, null: false 20 | t.string "nome", limit: 100, null: false 21 | t.datetime "nascimento" 22 | t.string "stack" 23 | t.datetime "created_at", null: false 24 | t.datetime "updated_at", null: false 25 | t.virtual "searchable", type: :text, as: "(((((nome)::text || ' '::text) || (apelido)::text) || ' '::text) || (COALESCE(stack, ' '::character varying))::text)", stored: true 26 | t.index ["apelido"], name: "index_pessoas_on_apelido", unique: true 27 | t.index ["searchable"], name: "index_pessoas_on_searchable", opclass: :gist_trgm_ops, using: :gist 28 | end 29 | 30 | end 31 | -------------------------------------------------------------------------------- /db/seeds.rb: -------------------------------------------------------------------------------- 1 | # This file should contain all the record creation needed to seed the database with its default values. 2 | # The data can then be loaded with the bin/rails db:seed command (or created alongside the database with db:setup). 3 | # 4 | # Examples: 5 | # 6 | # movies = Movie.create([{ name: "Star Wars" }, { name: "Lord of the Rings" }]) 7 | # Character.create(name: "Luke", movie: movies.first) 8 | -------------------------------------------------------------------------------- /db/structure.sql: -------------------------------------------------------------------------------- 1 | SET statement_timeout = 0; 2 | SET lock_timeout = 0; 3 | SET idle_in_transaction_session_timeout = 0; 4 | SET client_encoding = 'UTF8'; 5 | SET standard_conforming_strings = on; 6 | SELECT pg_catalog.set_config('search_path', '', false); 7 | SET check_function_bodies = false; 8 | SET xmloption = content; 9 | SET client_min_messages = warning; 10 | SET row_security = off; 11 | 12 | -- 13 | -- Name: pg_trgm; Type: EXTENSION; Schema: -; Owner: - 14 | -- 15 | 16 | CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; 17 | 18 | 19 | -- 20 | -- Name: EXTENSION pg_trgm; Type: COMMENT; Schema: -; Owner: - 21 | -- 22 | 23 | COMMENT ON EXTENSION pg_trgm IS 'text similarity measurement and index searching based on trigrams'; 24 | 25 | 26 | SET default_tablespace = ''; 27 | 28 | SET default_table_access_method = heap; 29 | 30 | -- 31 | -- Name: ar_internal_metadata; Type: TABLE; Schema: public; Owner: - 32 | -- 33 | 34 | CREATE TABLE public.ar_internal_metadata ( 35 | key character varying NOT NULL, 36 | value character varying, 37 | created_at timestamp(6) without time zone NOT NULL, 38 | updated_at timestamp(6) without time zone NOT NULL 39 | ); 40 | 41 | 42 | -- 43 | -- Name: pessoas; Type: TABLE; Schema: public; Owner: - 44 | -- 45 | 46 | CREATE TABLE public.pessoas ( 47 | id uuid DEFAULT gen_random_uuid() NOT NULL, 48 | apelido character varying(32) NOT NULL, 49 | nome character varying(100) NOT NULL, 50 | nascimento timestamp(6) without time zone, 51 | stack character varying, 52 | created_at timestamp(6) without time zone NOT NULL, 53 | updated_at timestamp(6) without time zone NOT NULL, 54 | searchable text GENERATED ALWAYS AS ((((((nome)::text || ' '::text) || (apelido)::text) || ' '::text) || (COALESCE(stack, ' '::character varying))::text)) STORED 55 | ); 56 | 57 | 58 | -- 59 | -- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - 60 | -- 61 | 62 | CREATE TABLE public.schema_migrations ( 63 | version character varying NOT NULL 64 | ); 65 | 66 | 67 | -- 68 | -- Name: ar_internal_metadata ar_internal_metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: - 69 | -- 70 | 71 | ALTER TABLE ONLY public.ar_internal_metadata 72 | ADD CONSTRAINT ar_internal_metadata_pkey PRIMARY KEY (key); 73 | 74 | 75 | -- 76 | -- Name: pessoas pessoas_pkey; Type: CONSTRAINT; Schema: public; Owner: - 77 | -- 78 | 79 | ALTER TABLE ONLY public.pessoas 80 | ADD CONSTRAINT pessoas_pkey PRIMARY KEY (id); 81 | 82 | 83 | -- 84 | -- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - 85 | -- 86 | 87 | ALTER TABLE ONLY public.schema_migrations 88 | ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); 89 | 90 | 91 | -- 92 | -- Name: index_pessoas_on_apelido; Type: INDEX; Schema: public; Owner: - 93 | -- 94 | 95 | CREATE UNIQUE INDEX index_pessoas_on_apelido ON public.pessoas USING btree (apelido); 96 | 97 | 98 | -- 99 | -- Name: index_pessoas_on_searchable; Type: INDEX; Schema: public; Owner: - 100 | -- 101 | 102 | CREATE INDEX index_pessoas_on_searchable ON public.pessoas USING gist (searchable public.gist_trgm_ops); 103 | 104 | 105 | -- 106 | -- PostgreSQL database dump complete 107 | -- 108 | 109 | SET search_path TO "$user", public; 110 | 111 | INSERT INTO "schema_migrations" (version) VALUES 112 | ('20230825134432'), 113 | ('20230825151541'), 114 | ('20230827015102'); 115 | 116 | 117 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | services: 3 | api1: &api 4 | #image: docker.io/akitaonrails/rinhabackendapi:latest 5 | build: ./ 6 | environment: 7 | PORT: 3000 8 | DB_HOST: localhost 9 | DB_POOL: 80 10 | RAILS_MAX_THREADS: 40 11 | WEB_CONCURRENCY: 2 12 | RAILS_LOG_LEVEL: warn 13 | RAILS_ENV: production 14 | REDIS_HOST: localhost 15 | REDIS_POOL_SIZE: 50 16 | JOB_BATCH_SIZE: 100 17 | CACHE_EXPIRES_SECONDS: 60 18 | RAILS_LOG_TO_STDOUT: 'true' 19 | RUBY_YJIT_ENABLE: 1 20 | RAILS_MASTER_KEY: 84ec93b5f81d27f8fdf7fc71d7b28e15 21 | hostname: api1 22 | depends_on: 23 | - postgres 24 | - redis 25 | network_mode: host 26 | deploy: 27 | resources: 28 | limits: 29 | cpus: '0.45' 30 | memory: '0.5GB' 31 | 32 | api2: 33 | <<: *api 34 | hostname: api2 35 | environment: 36 | PORT: 3001 37 | DB_HOST: localhost 38 | DB_POOL: 80 39 | RAILS_MAX_THREADS: 40 40 | WEB_CONCURRENCY: 2 41 | RAILS_LOG_LEVEL: warn 42 | RAILS_ENV: production 43 | REDIS_HOST: localhost 44 | REDIS_POOL_SIZE: 50 45 | JOB_BATCH_SIZE: 100 46 | CACHE_EXPIRES_SECONDS: 60 47 | RAILS_LOG_TO_STDOUT: 'true' 48 | RUBY_YJIT_ENABLE: 1 49 | RAILS_MASTER_KEY: 84ec93b5f81d27f8fdf7fc71d7b28e15 50 | 51 | sidekiq1: &sidekiq 52 | <<: *api 53 | hostname: sidekiq 54 | environment: 55 | DB_HOST: localhost 56 | DB_POOL: 3 57 | RAILS_MAX_THREADS: 1 58 | WEB_CONCURRENCY: 1 59 | RAILS_LOG_LEVEL: warn 60 | RAILS_ENV: production 61 | REDIS_HOST: localhost 62 | REDIS_POOL_SIZE: 1 63 | JOB_BATCH_SIZE: 40 64 | JOB_FLUSH_TIMEOUT: 30 65 | CACHE_EXPIRES_SECONDS: 60 66 | RAILS_LOG_TO_STDOUT: 'true' 67 | RUBY_YJIT_ENABLE: 1 68 | RAILS_MASTER_KEY: 84ec93b5f81d27f8fdf7fc71d7b28e15 69 | command: /rails/bin/bundle exec sidekiq -c 1 -q insert_buffer 70 | deploy: 71 | resources: 72 | limits: 73 | cpus: '0.05' 74 | memory: '0.15GB' 75 | 76 | sidekiq2: 77 | <<: *sidekiq 78 | command: /rails/bin/bundle exec sidekiq -c 1 -q flush 79 | 80 | nginx: # Load Balancer 81 | image: docker.io/nginx:latest 82 | command: ["nginx", "-g", "daemon off;"] 83 | volumes: 84 | - ./nginx.conf:/etc/nginx/nginx.conf:ro 85 | depends_on: 86 | - api1 87 | - api2 88 | ulimits: 89 | nproc: 1000000 90 | nofile: 91 | soft: 1000000 92 | hard: 1000000 93 | network_mode: host 94 | deploy: 95 | resources: 96 | limits: 97 | cpus: '0.15' 98 | memory: '0.3GB' 99 | 100 | postgres: # Banco de dados 101 | image: docker.io/postgres 102 | hostname: postgres 103 | environment: 104 | POSTGRES_USERNAME: postgres 105 | POSTGRES_PASSWORD: password 106 | command: postgres -c 'max_connections=450' 107 | volumes: 108 | - ./postgresql.conf:/docker-entrypoint-initdb.d/postgresql.conf 109 | healthcheck: 110 | test: ["CMD-SHELL", "pg_isready"] 111 | interval: 5s 112 | timeout: 5s 113 | retries: 20 114 | start_period: 10s 115 | network_mode: host 116 | deploy: 117 | resources: 118 | limits: 119 | cpus: '0.35' 120 | memory: '1.3GB' 121 | 122 | redis: 123 | image: docker.io/redis:latest 124 | hostname: redis 125 | command: redis-server --save "" --appendonly no --maxclients 20000 126 | network_mode: host 127 | deploy: 128 | resources: 129 | limits: 130 | cpus: '0.05' 131 | memory: '0.1GB' 132 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cd /rails 3 | sleep 5 # just to make sure postgres is up 4 | echo "run db:reset" 5 | DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bin/rails db:reset db:create db:migrate 6 | echo "run Puma" 7 | bin/bundle exec puma -C config/puma.rb 8 | -------------------------------------------------------------------------------- /imgs/graphs1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/imgs/graphs1.png -------------------------------------------------------------------------------- /imgs/graphs2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/imgs/graphs2.png -------------------------------------------------------------------------------- /lib/redis_queue.rb: -------------------------------------------------------------------------------- 1 | require 'redis' 2 | require 'connection_pool' 3 | require 'mock_redis' unless Rails.env.production? 4 | 5 | # Redis based Queue to register params for the inserts 6 | # when there's enough items we can fetch_batch to perform an insert_all 7 | class RedisQueue 8 | REDIS_URL = 'redis://%s:6379/5'.freeze 9 | 10 | def initialize(queue) 11 | @queue = queue 12 | @key_counter = "#{queue}-counter" 13 | @lock_key = "#{@queue}-lock" 14 | @redis_pool = ConnectionPool.new( 15 | size: ENV.fetch('REDIS_POOL_SIZE', '5').to_i, 16 | timeout: ENV.fetch('REDIS_TIMEOUT', '5').to_i 17 | ) do 18 | Rails.env.production? ? Redis.new(url: redis_url) : MockRedis.new 19 | end 20 | end 21 | 22 | def push(value) 23 | return if @queue.nil? || value.nil? 24 | 25 | @redis_pool.with do |conn| 26 | conn.rpush(@queue, value.to_json) 27 | conn.incr(@key_counter) 28 | end 29 | end 30 | 31 | def size 32 | return -1 unless @queue 33 | 34 | @redis_pool.with do |conn| 35 | conn.llen(@queue) 36 | end 37 | end 38 | 39 | def fetch_batch(batch_size = 10) 40 | @redis_pool.with do |conn| 41 | acquire_lock(conn, @lock_key) 42 | items = [] 43 | begin 44 | buffer_size = conn.llen(@queue) 45 | end_index = [batch_size - 1, buffer_size - 1].min 46 | items = conn.lrange(@queue, 0, end_index).map { |item| JSON.parse(item || 'null') } 47 | conn.ltrim(@queue, end_index + 1, -1) 48 | ensure 49 | release_lock(conn, @lock_key) 50 | end 51 | items 52 | end 53 | end 54 | 55 | def clear! 56 | @redis_pool.with do |conn| 57 | conn.del(@key_counter) 58 | conn.del(@queue) 59 | end 60 | end 61 | 62 | def counter 63 | return 0 unless @queue 64 | 65 | @redis_pool.with do |conn| 66 | conn.get(@key_counter).to_i.tap do |value| 67 | Rails.cache.write(@key_counter, value) 68 | end 69 | end 70 | end 71 | 72 | private 73 | 74 | def redis_url 75 | @redis_url ||= REDIS_URL % ENV.fetch('REDIS_HOST', 'localhost') 76 | end 77 | 78 | def acquire_lock(redis, lock_key) 79 | # Retry acquiring the lock until successful 80 | loop do 81 | break if redis.set(lock_key, 'LOCKED', nx: true, ex: 10) # Set a 10-second timeout for the lock 82 | sleep 0.1 # Sleep for a small interval before trying again 83 | end 84 | end 85 | 86 | def release_lock(redis, lock_key) 87 | redis.del(lock_key) 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/tag_coder.rb: -------------------------------------------------------------------------------- 1 | class TagCoder 2 | def self.dump(array) 3 | array.join(" ") 4 | end 5 | 6 | def self.load(string) 7 | string.split(" ") unless string.blank? 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/tasks/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/lib/tasks/.keep -------------------------------------------------------------------------------- /log/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/log/.keep -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes auto; 2 | worker_rlimit_nofile 500000; 3 | 4 | events { 5 | use epoll; 6 | worker_connections 512; 7 | } 8 | 9 | http { 10 | access_log off; 11 | error_log /dev/null emerg; 12 | 13 | upstream api { 14 | server localhost:3000; 15 | server localhost:3001; 16 | keepalive 500; 17 | } 18 | server { 19 | listen 9999; 20 | 21 | location / { 22 | proxy_buffering off; 23 | proxy_set_header Connection ""; 24 | proxy_http_version 1.1; 25 | proxy_set_header Keep-Alive ""; 26 | proxy_set_header Proxy-Connection "keep-alive"; 27 | proxy_pass http://api; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /postgresql.conf: -------------------------------------------------------------------------------- 1 | listen_addresses = '*' 2 | max_connections = 500 3 | superuser_reserved_connections = 3 4 | unix_socket_directories = '/var/run/postgresql' 5 | shared_buffers = 512MB 6 | work_mem = 4MB 7 | maintenance_work_mem = 256MB 8 | effective_cache_size = 1GB 9 | wal_buffers = 64MB 10 | checkpoint_timeout = 10min 11 | checkpoint_completion_target = 0.9 12 | random_page_cost = 4.0 13 | effective_io_concurrency = 2 14 | autovacuum = on 15 | log_statement = 'none' 16 | log_duration = off 17 | log_lock_waits = on 18 | log_error_verbosity = terse 19 | log_min_messages = fatal 20 | log_min_error_statement = fatal 21 | -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | -------------------------------------------------------------------------------- /storage/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/storage/.keep -------------------------------------------------------------------------------- /test/controllers/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/test/controllers/.keep -------------------------------------------------------------------------------- /test/controllers/pessoas_controller_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | 3 | class PessoasControllerTest < ActionDispatch::IntegrationTest 4 | setup do 5 | Rails.cache.clear 6 | @pessoa = pessoas(:one) 7 | @pessoa_two = pessoas(:two) 8 | end 9 | 10 | test "should get index and return unauthorized" do 11 | get pessoas_url, as: :json 12 | assert_response :bad_request 13 | end 14 | 15 | test 'should only bring search results' do 16 | get pessoas_url(t: 'berto'), as: :json 17 | assert_response :success 18 | 19 | returned = JSON.parse(response.body)&.map { |pessoa| pessoa['nome'] } 20 | assert_includes returned, @pessoa.nome 21 | end 22 | 23 | test 'should return count of pessoas' do 24 | get contagem_pessoas_url, as: :plain 25 | assert_response :success 26 | 27 | assert_includes response.body, 'total: 2' 28 | end 29 | 30 | test "should create pessoa" do 31 | # can't assert difference because it's async 32 | #assert_difference("Pessoa.count") do 33 | post pessoas_url, params: { pessoa: { apelido: 'foo', nascimento: @pessoa.nascimento, nome: 'foo foo', stack: @pessoa.stack } }, as: :json 34 | #end 35 | 36 | assert_response :created 37 | end 38 | 39 | test "should not allow apelido not being a string" do 40 | post pessoas_url, params: { pessoa: { apelido: 1, nascimento: @pessoa.nascimento, nome: 'foo foo', stack: @pessoa.stack } }, as: :json 41 | 42 | assert_response :bad_request 43 | end 44 | 45 | test "should not allow nome not being a string" do 46 | post pessoas_url, params: { pessoa: { apelido: 'foo', nascimento: @pessoa.nascimento, nome: 2, stack: @pessoa.stack } }, as: :json 47 | 48 | assert_response :bad_request 49 | end 50 | 51 | test "should not allow nascimento not being a date" do 52 | post pessoas_url, params: { pessoa: { apelido: 'foo', nascimento: 'foo', nome: 'foo foo', stack: @pessoa.stack } }, as: :json 53 | 54 | assert_response :bad_request 55 | end 56 | 57 | test "should not allow stack having element not being a string" do 58 | post pessoas_url, params: { pessoa: { apelido: 'foo', nascimento: @pessoa.nascimento, nome: 'foo foo', 59 | stack: ['ruby', 3] } }, as: :json 60 | 61 | assert_response :bad_request 62 | end 63 | 64 | test "should not allow duplicate pessoa" do 65 | # has to run this once to pre-heat the cache 66 | post pessoas_url, params: { pessoa: { apelido: @pessoa.apelido, nascimento: @pessoa.nascimento, nome: @pessoa.nome, 67 | stack: @pessoa.stack } }, as: :json 68 | assert_response :created 69 | 70 | post pessoas_url, params: { pessoa: { apelido: @pessoa.apelido, nascimento: @pessoa.nascimento, nome: @pessoa.nome, 71 | stack: @pessoa.stack } }, as: :json 72 | assert_response :unprocessable_entity 73 | end 74 | 75 | test "should show pessoa" do 76 | get pessoa_url(@pessoa), as: :json 77 | assert_response :success 78 | end 79 | 80 | test "should return not found pessoa" do 81 | get pessoa_url("unknown"), as: :json 82 | assert_response :not_found 83 | end 84 | 85 | test "should update pessoa" do 86 | patch pessoa_url(@pessoa), params: { pessoa: { apelido: 'bar', nascimento: @pessoa.nascimento, nome: 'bar bar', stack: @pessoa.stack } }, as: :json 87 | assert_response :success 88 | end 89 | 90 | test "should destroy pessoa" do 91 | assert_difference("Pessoa.count", -1) do 92 | delete pessoa_url(@pessoa), as: :json 93 | end 94 | 95 | assert_response :no_content 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /test/fixtures/files/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/test/fixtures/files/.keep -------------------------------------------------------------------------------- /test/fixtures/pessoas.yml: -------------------------------------------------------------------------------- 1 | # Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html 2 | 3 | one: 4 | apelido: josé 5 | nome: José Roberto 6 | nascimento: 2000-10-01 7 | stack: ["C#", "Node", "Oracle"] 8 | 9 | two: 10 | apelido: ana 11 | nome: Ana Barbosa 12 | nascimento: 1985-09-23 13 | stack: 14 | -------------------------------------------------------------------------------- /test/integration/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/test/integration/.keep -------------------------------------------------------------------------------- /test/jobs/pessoa_flush_job_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | require 'sidekiq/testing' 3 | require 'mock_redis' 4 | require 'minitest/autorun' 5 | 6 | class PessoaFlushJob 7 | # to share the same MockRedis from PessoaJob 8 | def set_buffer(buffer) 9 | @@buffer = buffer 10 | end 11 | end 12 | 13 | class PessoaFlushJobTest < ActiveSupport::TestCase 14 | setup do 15 | Sidekiq::Testing.fake! 16 | @generator = ->(i) { { apelido: "hello #{i}", nome: 'world' } } 17 | 18 | @pessoa_job = PessoaJob.new 19 | @pessoa_job.buffer.clear! 20 | 21 | @flush_job = PessoaFlushJob.new 22 | @flush_job.set_buffer(@pessoa_job.buffer) 23 | 24 | 2.times do |i| 25 | @pessoa_job.perform('create', @generator.call(i)) 26 | end 27 | 28 | @insert_mock = Minitest::Mock.new 29 | @insert_mock.expect(:insert_all, nil) 30 | end 31 | 32 | test 'flush batch' do 33 | PessoaJob::BUFFER_SIZE.times do |i| 34 | @pessoa_job.perform('create', @generator.call(i)) 35 | end 36 | assert_equal 12, @pessoa_job.buffer.size 37 | 38 | Pessoa.stub(:insert_all, @insert_mock) do 39 | @flush_job.perform 40 | end 41 | assert_equal 2, @pessoa_job.buffer.size 42 | end 43 | 44 | test 'flushes remaining items' do 45 | assert PessoaJob::BUFFER_SIZE > 2 46 | assert_equal 2, @pessoa_job.buffer.size 47 | 48 | Pessoa.stub(:insert_all, @insert_mock) do 49 | @flush_job.perform 50 | end 51 | assert_equal 0, @pessoa_job.buffer.size 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/jobs/pessoa_job_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | require 'sidekiq/testing' 3 | require 'mock_redis' 4 | require 'minitest/autorun' 5 | 6 | class PessoaJobTest < ActiveSupport::TestCase 7 | setup do 8 | Sidekiq::Testing.fake! 9 | @generator = ->(i) { { apelido: "hello #{i}", nome: 'world' } } 10 | end 11 | 12 | test 'should push new element to queue' do 13 | job = PessoaJob.new 14 | job.buffer.clear! 15 | 16 | job.perform('create', { apelido: "hello", nome: "world"} ) 17 | assert_equal 1, job.buffer.size 18 | end 19 | 20 | test 'should send to flush queue' do 21 | job = PessoaJob.new 22 | job.buffer.clear! 23 | 24 | mock = Minitest::Mock.new 25 | mock.expect(:perform_async, nil) 26 | 27 | PessoaFlushJob.stub(:perform_async, mock) do 28 | (PessoaJob::BUFFER_SIZE + 2).times do |i| 29 | job.perform('create', @generator.call(i)) 30 | end 31 | end 32 | end 33 | 34 | test 'flushes remaining queue items' do 35 | job = PessoaJob.new 36 | job.buffer.clear! 37 | 38 | assert PessoaJob::BUFFER_SIZE > 2 39 | 40 | mock = Minitest::Mock.new 41 | mock.expect(:perform_async, nil) 42 | 43 | PessoaFlushJob.stub(:perform_async, mock) do 44 | 2.times do |i| 45 | job.perform('create', @generator.call(i)) 46 | end 47 | end 48 | job.perform('flush', nil) 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /test/lib/redis_queue_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | require 'mock_redis' 3 | 4 | class RedisQueueTest < ActiveSupport::TestCase 5 | setup do 6 | @queue = RedisQueue.new("test") 7 | 8 | @queue.push({ a: 'foo' }) 9 | @queue.push({ a: 'bar' }) 10 | @queue.push({ a: 'hello' }) 11 | @queue.push({ a: 'world' }) 12 | end 13 | 14 | test 'should push new item and set timestamp and counter' do 15 | assert_equal 4, @queue.size() 16 | end 17 | 18 | test 'should check if list is being correctly appended' do 19 | values = @queue.fetch_batch(3).map { |item| item['a'] } 20 | assert_equal ['foo', 'bar', 'hello'], values 21 | 22 | values = @queue.fetch_batch(3).map { |item| item['a'] } 23 | assert_equal ['world'], values 24 | end 25 | 26 | test 'should have correct counter' do 27 | assert_equal 4, @queue.counter() 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /test/mailers/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/test/mailers/.keep -------------------------------------------------------------------------------- /test/models/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/test/models/.keep -------------------------------------------------------------------------------- /test/models/pessoa_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | 3 | class PessoaTest < ActiveSupport::TestCase 4 | test 'is validating apelido size limit' do 5 | pessoa = Pessoa.new apelido: 'abcdefghijklmnopqrstuvwxyz01234567890', nome: 'Valid name' 6 | assert_not pessoa.valid? 7 | end 8 | 9 | test 'is validating nome size limit' do 10 | pessoa = Pessoa.new apelido: 'valid apelido', nome: 'abcdefghijklmnopqrstuvwxyz01234567890abcdefghijklmnopqrstuvwxyz01234567890abcdefghijklmnopqrstuvwxyz01234567890abcdefghijklmnopqrstuvwxyz01234567890abcdefghijklmnopqrstuvwxyz01234567890abcdefghijklmnopqrstuvwxyz01234567890abcdefghijklmnopqrstuvwxyz01234567890abcdefghijklmnopqrstuvwxyz01234567890' 11 | assert_not pessoa.valid? 12 | end 13 | 14 | test 'cannot duplicate apelido' do 15 | fields = { apelido: 'valid apelido', nome: 'valid nome', nascimento: Date.current, stack: nil } 16 | assert Pessoa.create fields 17 | assert_raise(ActiveRecord::RecordNotUnique) { Pessoa.create fields.merge(nome: 'some other nome') } 18 | end 19 | 20 | test 'cannot duplicate nome' do 21 | fields = { apelido: 'valid apelido', nome: 'valid nome', nascimento: Date.current, stack: nil } 22 | assert Pessoa.create fields 23 | assert_raise(ActiveRecord::RecordNotUnique) { Pessoa.create fields.merge(apelido: 'some other apelido') } 24 | end 25 | 26 | test 'can find specific stack in stack list' do 27 | Pessoa.create apelido: 'foo', nome: 'foo foo', nascimento: Date.current, stack: %w[java ruby php bash] 28 | Pessoa.create apelido: 'bar', nome: 'bar bar', nascimento: Date.current, stack: %w[python perl bash] 29 | 30 | pessoa = Pessoa.search('python') 31 | assert_equal pessoa.count, 1 32 | assert_equal pessoa.first.apelido, 'bar' 33 | 34 | pessoa = Pessoa.search('ruby') 35 | assert_equal pessoa.count,1 36 | assert_equal pessoa.first.apelido, 'foo' 37 | 38 | pessoa = Pessoa.search('bash') 39 | assert_equal pessoa.count, 2 40 | end 41 | 42 | test 'search' do 43 | one = pessoas(:one) 44 | two = pessoas(:two) 45 | 46 | result = Pessoa.search('berto').first 47 | assert_equal result.nome, one.nome 48 | 49 | result = Pessoa.search('bosa').first 50 | assert_equal result.nome, two.nome 51 | 52 | result = Pessoa.search('Node').first 53 | assert_equal result.nome, one.nome 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | ENV["RAILS_ENV"] ||= "test" 2 | require_relative "../config/environment" 3 | require "rails/test_help" 4 | 5 | class ActiveSupport::TestCase 6 | # Run tests in parallel with specified workers 7 | parallelize(workers: :number_of_processors) 8 | 9 | # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. 10 | fixtures :all 11 | 12 | setup do 13 | Rails.cache.clear 14 | end 15 | # Add more helper methods to be used by all tests here... 16 | end 17 | -------------------------------------------------------------------------------- /tmp/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/tmp/.keep -------------------------------------------------------------------------------- /tmp/pids/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/tmp/pids/.keep -------------------------------------------------------------------------------- /tmp/storage/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/tmp/storage/.keep -------------------------------------------------------------------------------- /tmuxp-monitor.yaml: -------------------------------------------------------------------------------- 1 | session_name: my-tmux-session 2 | windows: 3 | - window_name: my-window 4 | panes: 5 | - htop -F puma 6 | - htop -F postgres 7 | - htop -F nginx 8 | - htop -F redis 9 | layout: tiled 10 | -------------------------------------------------------------------------------- /vendor/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akitaonrails/rinhabackend-rails-api/08e3f56f2d9058b1997e79b6c375c8d77509ec5e/vendor/.keep --------------------------------------------------------------------------------