├── vendor └── .keep ├── lib ├── assets │ └── .keep └── tasks │ ├── .keep │ ├── rap_sheet.rake │ └── process.rake ├── public ├── favicon.ico ├── apple-touch-icon.png ├── apple-touch-icon-precomposed.png ├── robots.txt ├── 500.html ├── 422.html └── 404.html ├── .ruby-version ├── app ├── assets │ ├── images │ │ └── .keep │ ├── javascripts │ │ ├── channels │ │ │ └── .keep │ │ ├── uploads_new.js │ │ ├── cable.js │ │ └── application.js │ ├── motions │ │ └── prop64.pdf │ ├── config │ │ └── manifest.js │ ├── petitions │ │ └── prop64_misdemeanors.pdf │ └── stylesheets │ │ ├── autoclearance.scss │ │ └── application.scss ├── models │ ├── concerns │ │ └── .keep │ ├── application_record.rb │ ├── eligibility_estimate.rb │ ├── anon_disposition.rb │ ├── anon_cycle.rb │ ├── rap_sheet_properties.rb │ ├── event_properties.rb │ ├── count_properties.rb │ ├── anon_count.rb │ ├── anon_event.rb │ └── anon_rap_sheet.rb ├── controllers │ ├── concerns │ │ └── .keep │ ├── uploads_controller.rb │ ├── application_controller.rb │ └── jobs_controller.rb ├── views │ ├── layouts │ │ ├── mailer.text.erb │ │ ├── mailer.html.erb │ │ └── application.html.erb │ ├── jobs │ │ └── index.html.erb │ └── uploads │ │ └── new.html.erb ├── helpers │ ├── application_helper.rb │ ├── code_sections.rb │ ├── pdf_reader.rb │ ├── counties.rb │ ├── rap_sheet_deidentifier.rb │ └── rap_sheet_processor.rb ├── jobs │ ├── application_job.rb │ └── process_rap_sheets_job.rb ├── channels │ └── application_cable │ │ ├── channel.rb │ │ └── connection.rb ├── mailers │ └── application_mailer.rb ├── domain │ ├── event_with_eligibility.rb │ ├── rap_sheet_with_eligibility.rb │ ├── prop64_classifier.rb │ └── summary_csv.rb ├── classifiers │ └── plea_bargain_classifier.rb └── lib │ ├── fill_prop64_motion.rb │ └── fill_misdemeanor_petitions.rb ├── terraform ├── staging │ ├── backend-config.example │ ├── varfile.example │ ├── variables.tf │ └── staging.tf ├── production │ ├── backend-config.example │ ├── varfile.example │ ├── production.tf │ └── variables.tf ├── main │ └── variables.tf ├── policies │ └── mfa_policy.json └── metabase │ └── metabase.tf ├── scripts ├── bastion_awscli.conf ├── bastion_awslogs.conf ├── setup_eb.sh ├── sync_analysis_db.sh ├── bastion_setup.sh ├── create_postgres_user.sh └── adduser.sh ├── bin ├── rake ├── bundle ├── rails ├── yarn ├── spring ├── update └── setup ├── .ebextensions ├── 03_set_up_users.config ├── 02_security_update.config ├── 05_db_migrations.config ├── 06_pdftk.config ├── 04_ruby.config └── 01_awslogs.config ├── package.json ├── spec ├── fixtures │ ├── chewbacca_rap_sheet.pdf │ ├── skywalker_rap_sheet.pdf │ ├── not_a_valid_rap_sheet.pdf │ ├── summary.csv │ ├── skywalker_rap_sheet.txt │ └── chewbacca_rap_sheet.txt ├── support │ └── pdf_helpers.rb ├── features │ └── upload_rap_sheet_spec.rb ├── models │ ├── eligibility_estimate_spec.rb │ ├── anon_cycle_spec.rb │ ├── event_properties_spec.rb │ ├── anon_disposition_spec.rb │ ├── rap_sheet_properties_spec.rb │ ├── anon_event_spec.rb │ ├── count_properties_spec.rb │ └── anon_count_spec.rb ├── lib │ ├── fill_misdemeanor_petitions_spec.rb │ └── fill_prop64_motion_spec.rb ├── rails_helper.rb ├── helpers │ ├── pdf_reader_spec.rb │ ├── rap_sheet_processor_spec.rb │ └── rap_sheet_deidentifier_spec.rb ├── domain │ ├── summary_csv_spec.rb │ ├── event_with_eligibility_spec.rb │ ├── rap_sheet_with_eligibility_spec.rb │ └── prop64_classifier_spec.rb └── spec_helper.rb ├── config ├── spring.rb ├── environment.rb ├── initializers │ ├── mime_types.rb │ ├── filter_parameter_logging.rb │ ├── application_controller_renderer.rb │ ├── cookies_serializer.rb │ ├── backtrace_silencers.rb │ ├── wrap_parameters.rb │ ├── assets.rb │ ├── inflections.rb │ ├── content_security_policy.rb │ └── new_framework_defaults_5_2.rb ├── boot.rb ├── cable.yml ├── routes.rb ├── database.yml ├── application.rb ├── locales │ └── en.yml ├── storage.yml ├── secrets.yml ├── puma.rb └── environments │ ├── test.rb │ ├── development.rb │ └── production.rb ├── config.ru ├── db ├── migrate │ ├── 20180816211915_add_sex_to_anon_rap_sheet.rb │ ├── 20180816220855_add_race_to_anon_rap_sheet.rb │ ├── 20180821003637_add_text_to_anon_dispositions.rb │ ├── 20180809175425_add_county_to_anon_rap_sheets.rb │ ├── 20180814010846_add_index_to_rap_sheet_checksum.rb │ ├── 20180816203328_add_year_of_birth_to_anon_rap_sheets.rb │ ├── 20180817184950_add_person_unique_id_to_anon_rap_sheets.rb │ ├── 20180808174608_create_anon_rap_sheets.rb │ ├── 20180918213324_create_eligibility_estimates.rb │ ├── 20180814232139_create_anon_dispositions.rb │ ├── 20180809215249_add_anon_events_table.rb │ ├── 20180823225331_move_severity_from_count_to_disposition.rb │ ├── 20180809233633_add_anon_counts_table.rb │ ├── 20180917172255_create_count_properties.rb │ ├── 20180913184908_create_rap_sheet_properties.rb │ ├── 20180815170521_add_count_number_to_anon_counts.rb │ ├── 20180913235743_create_event_properties.rb │ └── 20180814195919_create_anon_cycles.rb ├── seeds.rb └── schema.rb ├── .elasticbeanstalk └── config.yml ├── yarn.lock ├── Brewfile ├── .rubocop.yml ├── Rakefile ├── .pgsync.yml ├── .gitignore ├── .circleci └── config.yml ├── Gemfile ├── README.md └── .rubocop_todo.yml /vendor/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/assets/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/tasks/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.5.1 2 | -------------------------------------------------------------------------------- /app/assets/images/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/models/concerns/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/apple-touch-icon.png: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/controllers/concerns/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/assets/javascripts/channels/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/apple-touch-icon-precomposed.png: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/views/layouts/mailer.text.erb: -------------------------------------------------------------------------------- 1 | <%= yield %> 2 | -------------------------------------------------------------------------------- /app/helpers/application_helper.rb: -------------------------------------------------------------------------------- 1 | module ApplicationHelper 2 | end 3 | -------------------------------------------------------------------------------- /app/jobs/application_job.rb: -------------------------------------------------------------------------------- 1 | class ApplicationJob < ActiveJob::Base 2 | end 3 | -------------------------------------------------------------------------------- /terraform/staging/backend-config.example: -------------------------------------------------------------------------------- 1 | access_key = 2 | secret_key = 3 | bucket = 4 | -------------------------------------------------------------------------------- /terraform/production/backend-config.example: -------------------------------------------------------------------------------- 1 | access_key = 2 | secret_key = 3 | bucket = 4 | -------------------------------------------------------------------------------- /app/views/jobs/index.html.erb: -------------------------------------------------------------------------------- 1 |
2 | <%= button_to('go go go') %> 3 |
4 | -------------------------------------------------------------------------------- /scripts/bastion_awscli.conf: -------------------------------------------------------------------------------- 1 | [plugins] 2 | cwlogs = cwlogs 3 | [default] 4 | region = us-gov-west-1 5 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require_relative '../config/boot' 3 | require 'rake' 4 | Rake.application.run 5 | -------------------------------------------------------------------------------- /app/models/application_record.rb: -------------------------------------------------------------------------------- 1 | class ApplicationRecord < ActiveRecord::Base 2 | self.abstract_class = true 3 | end 4 | -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | -------------------------------------------------------------------------------- /.ebextensions/03_set_up_users.config: -------------------------------------------------------------------------------- 1 | container_commands: 2 | 01-add-user-script: 3 | command: "sh scripts/adduser.sh" 4 | -------------------------------------------------------------------------------- /app/assets/motions/prop64.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codeforamerica/autoclearance/master/app/assets/motions/prop64.pdf -------------------------------------------------------------------------------- /.ebextensions/02_security_update.config: -------------------------------------------------------------------------------- 1 | commands: 2 | 01-install-security-updates: 3 | command: "sudo yum -y update --security" 4 | -------------------------------------------------------------------------------- /app/channels/application_cable/channel.rb: -------------------------------------------------------------------------------- 1 | module ApplicationCable 2 | class Channel < ActionCable::Channel::Base 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "autoclearance", 3 | "private": true, 4 | "dependencies": { 5 | "vue": "^2.5.17" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /app/assets/config/manifest.js: -------------------------------------------------------------------------------- 1 | //= link_tree ../images 2 | //= link_directory ../javascripts .js 3 | //= link_directory ../stylesheets .css 4 | -------------------------------------------------------------------------------- /app/controllers/uploads_controller.rb: -------------------------------------------------------------------------------- 1 | class UploadsController < ApplicationController 2 | protect_from_forgery with: :exception 3 | end 4 | -------------------------------------------------------------------------------- /spec/fixtures/chewbacca_rap_sheet.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codeforamerica/autoclearance/master/spec/fixtures/chewbacca_rap_sheet.pdf -------------------------------------------------------------------------------- /spec/fixtures/skywalker_rap_sheet.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codeforamerica/autoclearance/master/spec/fixtures/skywalker_rap_sheet.pdf -------------------------------------------------------------------------------- /app/channels/application_cable/connection.rb: -------------------------------------------------------------------------------- 1 | module ApplicationCable 2 | class Connection < ActionCable::Connection::Base 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /bin/bundle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) 3 | load Gem.bin_path('bundler', 'bundle') 4 | -------------------------------------------------------------------------------- /spec/fixtures/not_a_valid_rap_sheet.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codeforamerica/autoclearance/master/spec/fixtures/not_a_valid_rap_sheet.pdf -------------------------------------------------------------------------------- /.ebextensions/05_db_migrations.config: -------------------------------------------------------------------------------- 1 | container_commands: 2 | migrate: 3 | command: "bundle exec rake db:migrate" 4 | leader_only: true 5 | -------------------------------------------------------------------------------- /app/mailers/application_mailer.rb: -------------------------------------------------------------------------------- 1 | class ApplicationMailer < ActionMailer::Base 2 | default from: 'from@example.com' 3 | layout 'mailer' 4 | end 5 | -------------------------------------------------------------------------------- /config/spring.rb: -------------------------------------------------------------------------------- 1 | %w[ 2 | .ruby-version 3 | .rbenv-vars 4 | tmp/restart.txt 5 | tmp/caching-dev.txt 6 | ].each { |path| Spring.watch(path) } 7 | -------------------------------------------------------------------------------- /app/assets/petitions/prop64_misdemeanors.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codeforamerica/autoclearance/master/app/assets/petitions/prop64_misdemeanors.pdf -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | # This file is used by Rack-based servers to start the application. 2 | 3 | require_relative 'config/environment' 4 | 5 | run Rails.application 6 | -------------------------------------------------------------------------------- /bin/rails: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_PATH = File.expand_path('../config/application', __dir__) 3 | require_relative '../config/boot' 4 | require 'rails/commands' 5 | -------------------------------------------------------------------------------- /config/environment.rb: -------------------------------------------------------------------------------- 1 | # Load the Rails application. 2 | require_relative 'application' 3 | 4 | # Initialize the Rails application. 5 | Rails.application.initialize! 6 | -------------------------------------------------------------------------------- /db/migrate/20180816211915_add_sex_to_anon_rap_sheet.rb: -------------------------------------------------------------------------------- 1 | class AddSexToAnonRapSheet < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :anon_rap_sheets, :sex, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /terraform/production/varfile.example: -------------------------------------------------------------------------------- 1 | aws_access_key = 2 | aws_secret_key = 3 | key_name = 4 | public_key = 5 | rails_secret_key_base = 6 | environment = 7 | rds_username = 8 | analysis_rds_username = 9 | -------------------------------------------------------------------------------- /terraform/staging/varfile.example: -------------------------------------------------------------------------------- 1 | aws_access_key = 2 | aws_secret_key = 3 | key_name = 4 | public_key = 5 | rails_secret_key_base = 6 | environment = 7 | rds_username = 8 | analysis_rds_username = 9 | -------------------------------------------------------------------------------- /app/jobs/process_rap_sheets_job.rb: -------------------------------------------------------------------------------- 1 | class ProcessRapSheetsJob < ApplicationJob 2 | queue_as :default 3 | 4 | def perform(courthouses) 5 | RapSheetProcessor.new(courthouses).run 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /config/initializers/mime_types.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new mime types for use in respond_to blocks: 4 | # Mime::Type.register "text/richtext", :rtf 5 | -------------------------------------------------------------------------------- /db/migrate/20180816220855_add_race_to_anon_rap_sheet.rb: -------------------------------------------------------------------------------- 1 | class AddRaceToAnonRapSheet < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :anon_rap_sheets, :race, :string 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180821003637_add_text_to_anon_dispositions.rb: -------------------------------------------------------------------------------- 1 | class AddTextToAnonDispositions < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :anon_dispositions, :text, :text 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- 1 | class ApplicationController < ActionController::Base 2 | def not_found 3 | render :file => 'public/404.html', :status => :not_found, :layout => false 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180809175425_add_county_to_anon_rap_sheets.rb: -------------------------------------------------------------------------------- 1 | class AddCountyToAnonRapSheets < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :anon_rap_sheets, :county, :string, null: false 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180814010846_add_index_to_rap_sheet_checksum.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToRapSheetChecksum < ActiveRecord::Migration[5.2] 2 | def change 3 | add_index :anon_rap_sheets, :checksum, :unique => true 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /db/migrate/20180816203328_add_year_of_birth_to_anon_rap_sheets.rb: -------------------------------------------------------------------------------- 1 | class AddYearOfBirthToAnonRapSheets < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :anon_rap_sheets, :year_of_birth, :integer 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /app/controllers/jobs_controller.rb: -------------------------------------------------------------------------------- 1 | class JobsController < ApplicationController 2 | protect_from_forgery with: :exception 3 | 4 | def create 5 | ProcessRapSheetsJob.perform_later(Counties::SAN_FRANCISCO) 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /config/boot.rb: -------------------------------------------------------------------------------- 1 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) 2 | 3 | require 'bundler/setup' # Set up gems listed in the Gemfile. 4 | require 'bootsnap/setup' # Speed up boot time by caching expensive operations. 5 | -------------------------------------------------------------------------------- /lib/tasks/rap_sheet.rake: -------------------------------------------------------------------------------- 1 | namespace :rap_sheet do 2 | desc "This task will strip the identifiable information from a RAP sheet and save the text" 3 | task deidentify: :environment do 4 | RapSheetDeidentifier.new.run 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /db/migrate/20180817184950_add_person_unique_id_to_anon_rap_sheets.rb: -------------------------------------------------------------------------------- 1 | class AddPersonUniqueIdToAnonRapSheets < ActiveRecord::Migration[5.2] 2 | def change 3 | add_column :anon_rap_sheets, :person_unique_id, :binary, limit: 32 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Configure sensitive parameters which will be filtered from the log file. 4 | Rails.application.config.filter_parameters += [:password] 5 | -------------------------------------------------------------------------------- /config/cable.yml: -------------------------------------------------------------------------------- 1 | development: 2 | adapter: async 3 | 4 | test: 5 | adapter: async 6 | 7 | production: 8 | adapter: redis 9 | url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> 10 | channel_prefix: autoclearance_production 11 | -------------------------------------------------------------------------------- /config/routes.rb: -------------------------------------------------------------------------------- 1 | Rails.application.routes.draw do 2 | root to: 'uploads#new' 3 | 4 | resources :jobs, only: [:index, :create] 5 | 6 | mount Cfa::Styleguide::Engine => '/cfa' 7 | 8 | # keep last 9 | match '*path', via: [:all], to: 'application#not_found' 10 | end 11 | -------------------------------------------------------------------------------- /config/initializers/application_controller_renderer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # ActiveSupport::Reloader.to_prepare do 4 | # ApplicationController.renderer.defaults.merge!( 5 | # http_host: 'example.org', 6 | # https: false 7 | # ) 8 | # end 9 | -------------------------------------------------------------------------------- /app/models/eligibility_estimate.rb: -------------------------------------------------------------------------------- 1 | class EligibilityEstimate < ApplicationRecord 2 | belongs_to :count_properties 3 | 4 | def self.build(prop64_classifier) 5 | prop_64_eligible = prop64_classifier.eligible? 6 | EligibilityEstimate.new(prop_64_eligible: prop_64_eligible) 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /config/initializers/cookies_serializer.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Specify a serializer for the signed and encrypted cookie jars. 4 | # Valid options are :json, :marshal, and :hybrid. 5 | Rails.application.config.action_dispatch.cookies_serializer = :json 6 | -------------------------------------------------------------------------------- /scripts/bastion_awslogs.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | state_file = /var/lib/awslogs/agent-state 3 | 4 | [bastion-ssh] 5 | datetime_format = %b %d %H:%M:%S 6 | file = /var/log/secure 7 | buffer_duration = 5000 8 | log_stream_name = {instance_id} 9 | initial_position = start_of_file 10 | log_group_name = bastion-ssh 11 | -------------------------------------------------------------------------------- /app/views/layouts/mailer.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 8 | 9 | 10 | 11 | <%= yield %> 12 | 13 | 14 | -------------------------------------------------------------------------------- /scripts/setup_eb.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | mkdir ~/.aws 6 | touch ~/.aws/config 7 | chmod 600 ~/.aws/config 8 | echo "[profile autoclearance]" > ~/.aws/config 9 | echo "aws_access_key_id=$AWS_ACCESS_KEY_ID" >> ~/.aws/config 10 | echo "aws_secret_access_key=$AWS_SECRET_KEY" >> ~/.aws/config 11 | -------------------------------------------------------------------------------- /db/migrate/20180808174608_create_anon_rap_sheets.rb: -------------------------------------------------------------------------------- 1 | class CreateAnonRapSheets < ActiveRecord::Migration[5.2] 2 | def change 3 | enable_extension 'pgcrypto' 4 | create_table :anon_rap_sheets, id: :uuid do |t| 5 | t.binary :checksum, null: false, limit: 32 6 | t.timestamps 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /.elasticbeanstalk/config.yml: -------------------------------------------------------------------------------- 1 | branch-defaults: 2 | master: 3 | environment: autoclearance-prod 4 | global: 5 | application_name: Autoclearance 6 | default_platform: Ruby 2.5 (Puma) 7 | default_region: us-gov-west-1 8 | include_git_submodules: true 9 | profile: autoclearance 10 | sc: git 11 | workspace_type: Application 12 | -------------------------------------------------------------------------------- /bin/yarn: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | APP_ROOT = File.expand_path('..', __dir__) 3 | Dir.chdir(APP_ROOT) do 4 | begin 5 | exec "yarnpkg", *ARGV 6 | rescue Errno::ENOENT 7 | $stderr.puts "Yarn executable was not detected in the system." 8 | $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install" 9 | exit 1 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/support/pdf_helpers.rb: -------------------------------------------------------------------------------- 1 | module PdfHelpers 2 | def get_fields_from_pdf(tempfile) 3 | pdftk = PdfForms.new(Cliver.detect('pdftk')) 4 | fields = pdftk.get_fields tempfile.path 5 | 6 | {}.tap do |fields_dict| 7 | fields.each do |field| 8 | fields_dict[field.name] = field.value 9 | end 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | vue@^2.5.17: 6 | version "2.5.17" 7 | resolved "https://registry.yarnpkg.com/vue/-/vue-2.5.17.tgz#0f8789ad718be68ca1872629832ed533589c6ada" 8 | integrity sha512-mFbcWoDIJi0w0Za4emyLiW72Jae0yjANHbCVquMKijcavBGypqlF7zHRgMa5k4sesdv7hv2rB4JPdZfR+TPfhQ== 9 | -------------------------------------------------------------------------------- /Brewfile: -------------------------------------------------------------------------------- 1 | brew "terraform" 2 | brew "chruby" 3 | brew "ruby-install" 4 | brew "python3" 5 | tap "git-duet/tap" 6 | brew "git-duet/tap/git-duet" 7 | cask "shiftit" 8 | cask "iterm2" 9 | cask "firefox" 10 | brew "awsebcli" 11 | cask "flycut" 12 | brew "enscript" 13 | brew "ghostscript" 14 | brew "node" 15 | brew "yarn" 16 | tap "homebrew/cask" 17 | cask "chromedriver" 18 | -------------------------------------------------------------------------------- /.ebextensions/06_pdftk.config: -------------------------------------------------------------------------------- 1 | commands: 2 | 01-libgcj: 3 | command: "sudo wget -O /usr/lib64/libgcj.so.10 https://github.com/lob/lambda-pdftk-example/raw/master/bin/libgcj.so.10" 4 | 02-pdftk: 5 | command: "sudo wget -O /usr/bin/pdftk https://github.com/lob/lambda-pdftk-example/raw/master/bin/pdftk" 6 | 03-pdftk-permissions: 7 | command: "chmod a+x /usr/bin/pdftk" 8 | -------------------------------------------------------------------------------- /app/models/anon_disposition.rb: -------------------------------------------------------------------------------- 1 | class AnonDisposition < ApplicationRecord 2 | belongs_to :anon_count 3 | 4 | def self.build_from_parser(disposition) 5 | AnonDisposition.new( 6 | disposition_type: disposition.type, 7 | sentence: disposition.sentence, 8 | text: disposition.text, 9 | severity: disposition.severity 10 | ) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /db/migrate/20180918213324_create_eligibility_estimates.rb: -------------------------------------------------------------------------------- 1 | class CreateEligibilityEstimates < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :eligibility_estimates, id: :uuid do |t| 4 | t.timestamps 5 | t.references :count_properties, type: :uuid, foreign_key: true, null: false 6 | t.string :prop_64_eligible, null: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /db/seeds.rb: -------------------------------------------------------------------------------- 1 | # This file should contain all the record creation needed to seed the database with its default values. 2 | # The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup). 3 | # 4 | # Examples: 5 | # 6 | # movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }]) 7 | # Character.create(name: 'Luke', movie: movies.first) 8 | -------------------------------------------------------------------------------- /app/models/anon_cycle.rb: -------------------------------------------------------------------------------- 1 | class AnonCycle < ApplicationRecord 2 | belongs_to :anon_rap_sheet 3 | has_many :anon_events, dependent: :destroy 4 | 5 | def self.build_from_parser(cycle:, rap_sheet:) 6 | events = cycle.events.map do |event| 7 | AnonEvent.build_from_parser(event: event, rap_sheet: rap_sheet) 8 | end 9 | 10 | AnonCycle.new(anon_events: events) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /app/models/rap_sheet_properties.rb: -------------------------------------------------------------------------------- 1 | class RapSheetProperties < ApplicationRecord 2 | belongs_to :anon_rap_sheet 3 | 4 | def self.build(rap_sheet:) 5 | RapSheetProperties.new( 6 | has_superstrikes: rap_sheet.superstrikes.present?, 7 | has_sex_offender_registration: rap_sheet.sex_offender_registration?, 8 | deceased: rap_sheet.deceased_events.any? 9 | ) 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20180814232139_create_anon_dispositions.rb: -------------------------------------------------------------------------------- 1 | class CreateAnonDispositions < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :anon_dispositions, id: :uuid do |t| 4 | t.timestamps 5 | t.references :anon_count, type: :uuid, foreign_key: true, null: false 6 | t.string :disposition_type, null: false 7 | t.string :sentence 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /db/migrate/20180809215249_add_anon_events_table.rb: -------------------------------------------------------------------------------- 1 | class AddAnonEventsTable < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :anon_events, id: :uuid do |t| 4 | t.string :agency 5 | t.string :event_type, null: false 6 | t.date :date 7 | t.timestamps 8 | t.references :anon_rap_sheet, type: :uuid, foreign_key: true, null: false 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20180823225331_move_severity_from_count_to_disposition.rb: -------------------------------------------------------------------------------- 1 | class MoveSeverityFromCountToDisposition < ActiveRecord::Migration[5.2] 2 | def up 3 | remove_column :anon_counts, :severity 4 | add_column :anon_dispositions, :severity, :string 5 | end 6 | 7 | def down 8 | add_column :anon_counts, :severity, :string 9 | remove_column :anon_dispositions, :severity 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /app/assets/javascripts/uploads_new.js: -------------------------------------------------------------------------------- 1 | var store = { 2 | selectedFiles: null 3 | }; 4 | 5 | document.addEventListener('DOMContentLoaded', function () { 6 | new Vue({ 7 | el: '#vue-app-div', 8 | data: store, 9 | methods: { 10 | fileInputChanged: function (event) { 11 | store.selectedFiles = event.target.files 12 | } 13 | } 14 | }); 15 | }); -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_from: .rubocop_todo.yml 2 | 3 | AllCops: 4 | Exclude: 5 | - 'db/schema.rb' 6 | - 'vendor/**/*' 7 | - 'bin/*' 8 | 9 | Rails: 10 | Enabled: true 11 | 12 | Rails/NotNullColumn: 13 | Enabled: false 14 | 15 | Naming/VariableNumber: 16 | Enabled: false 17 | 18 | Metrics/BlockLength: 19 | Exclude: 20 | - 'spec/**/*' 21 | 22 | Naming/PredicateName: 23 | NamePrefix: ['is_'] 24 | -------------------------------------------------------------------------------- /db/migrate/20180809233633_add_anon_counts_table.rb: -------------------------------------------------------------------------------- 1 | class AddAnonCountsTable < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :anon_counts, id: :uuid do |t| 4 | t.string :code 5 | t.string :section 6 | t.string :description 7 | t.string :severity 8 | t.timestamps 9 | t.references :anon_event, type: :uuid, foreign_key: true, null: false 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /scripts/sync_analysis_db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Provide environment as first argument (i.e: staging, production) 5 | ENV=$1 6 | 7 | # Need to run with the debug flag enabled because foreign key constraints fail 8 | # if tables are copied in parallel 9 | DATABASE_URL=$(cd terraform/${ENV}; terraform output analysis_database_url) 10 | 11 | DATABASE_URL=${DATABASE_URL} bundle exec pgsync all --debug #--schema-first 12 | -------------------------------------------------------------------------------- /app/assets/javascripts/cable.js: -------------------------------------------------------------------------------- 1 | // Action Cable provides the framework to deal with WebSockets in Rails. 2 | // You can generate new channels where WebSocket features live using the `rails generate channel` command. 3 | // 4 | //= require action_cable 5 | //= require_self 6 | //= require_tree ./channels 7 | 8 | (function() { 9 | this.App || (this.App = {}); 10 | 11 | App.cable = ActionCable.createConsumer(); 12 | 13 | }).call(this); 14 | -------------------------------------------------------------------------------- /app/assets/stylesheets/autoclearance.scss: -------------------------------------------------------------------------------- 1 | .main-footer { 2 | height: 117px 3 | } 4 | 5 | .main-footer__cfa-logo { 6 | margin-top: 0em; 7 | } 8 | 9 | .file-list { 10 | &:last-child { 11 | padding-bottom: -20px; 12 | } 13 | 14 | li { 15 | border-top: 2px solid #EAEAE9; 16 | padding: 10px; 17 | .truncated-filename { 18 | max-width: 25ch; 19 | overflow: hidden; 20 | text-overflow: ellipsis; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /app/helpers/code_sections.rb: -------------------------------------------------------------------------------- 1 | module CodeSections 2 | PROP64_DISMISSIBLE = 3 | [ 4 | 'HS 11357', # simple possession 5 | 'HS 11358', # cultivation 6 | 'HS 11359', # possession for sale 7 | 'HS 11360', # transportation for sale 8 | ].freeze 9 | 10 | PROP64_INELIGIBLE = 11 | [ 12 | 'HS 11359(c)(3)', 13 | 'HS 11359(d)', 14 | 'HS 11360(a)(3)(c)', 15 | 'HS 11360(a)(3)(d)' 16 | ].freeze 17 | end 18 | -------------------------------------------------------------------------------- /config/initializers/backtrace_silencers.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. 4 | # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } 5 | 6 | # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. 7 | # Rails.backtrace_cleaner.remove_silencers! 8 | -------------------------------------------------------------------------------- /db/migrate/20180917172255_create_count_properties.rb: -------------------------------------------------------------------------------- 1 | class CreateCountProperties < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :count_properties, id: :uuid do |t| 4 | t.timestamps 5 | t.references :anon_count, type: :uuid, foreign_key: true, null: false 6 | t.boolean :has_prop_64_code, null: false 7 | t.boolean :has_two_prop_64_priors, null: false 8 | t.string :prop_64_plea_bargain, null: false 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /db/migrate/20180913184908_create_rap_sheet_properties.rb: -------------------------------------------------------------------------------- 1 | class CreateRapSheetProperties < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :rap_sheet_properties, id: :uuid do |t| 4 | t.timestamps 5 | t.references :anon_rap_sheet, type: :uuid, foreign_key: true, null: false 6 | t.boolean :has_superstrikes, null: false 7 | t.boolean :has_sex_offender_registration, null: false 8 | t.boolean :deceased, null: false 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /app/models/event_properties.rb: -------------------------------------------------------------------------------- 1 | class EventProperties < ApplicationRecord 2 | belongs_to :anon_event 3 | 4 | def self.build(event:, rap_sheet:) 5 | EventProperties.new( 6 | has_felonies: event.severity == 'F', 7 | has_probation: event.has_sentence_with?(:probation), 8 | has_prison: event.has_sentence_with?(:prison), 9 | has_probation_violations: event.probation_violated?(rap_sheet), 10 | dismissed_by_pc1203: event.dismissed_by_pc1203? 11 | ) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # Add your own tasks in files placed in lib/tasks ending in .rake, 2 | # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. 3 | 4 | require_relative 'config/application' 5 | 6 | Rails.application.load_tasks 7 | 8 | begin 9 | require 'bundler/audit/task' 10 | Bundler::Audit::Task.new 11 | 12 | require 'rubocop/rake_task' 13 | RuboCop::RakeTask.new(:rubocop) 14 | 15 | task default: %w[bundle:audit rubocop] 16 | rescue NameError, LoadError 17 | # bundler-audit is not available 18 | end 19 | -------------------------------------------------------------------------------- /db/migrate/20180815170521_add_count_number_to_anon_counts.rb: -------------------------------------------------------------------------------- 1 | class AddCountNumberToAnonCounts < ActiveRecord::Migration[5.2] 2 | class AnonCount < ApplicationRecord 3 | end 4 | 5 | def change 6 | add_column :anon_counts, :count_number, :integer 7 | 8 | AnonCount.reset_column_information 9 | # rubocop:disable Rails/SkipsModelValidations 10 | AnonCount.where(count_number: nil).update_all(count_number: 0) 11 | # rubocop:enable Rails/SkipsModelValidations 12 | 13 | change_column_null(:anon_counts, :count_number, false) 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /db/migrate/20180913235743_create_event_properties.rb: -------------------------------------------------------------------------------- 1 | class CreateEventProperties < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :event_properties, id: :uuid do |t| 4 | t.timestamps 5 | t.references :anon_event, type: :uuid, foreign_key: true, null: false 6 | t.boolean :has_felonies, null: false 7 | t.boolean :has_probation, null: false 8 | t.boolean :has_probation_violations, null: false 9 | t.boolean :has_prison, null: false 10 | t.boolean :dismissed_by_pc1203, null: false 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /config/initializers/wrap_parameters.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # This file contains settings for ActionController::ParamsWrapper which 4 | # is enabled by default. 5 | 6 | # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array. 7 | ActiveSupport.on_load(:action_controller) do 8 | wrap_parameters format: [:json] 9 | end 10 | 11 | # To enable root element in JSON for ActiveRecord objects. 12 | # ActiveSupport.on_load(:active_record) do 13 | # self.include_root_in_json = true 14 | # end 15 | -------------------------------------------------------------------------------- /bin/spring: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # This file loads spring without using Bundler, in order to be fast. 4 | # It gets overwritten when you run the `spring binstub` command. 5 | 6 | unless defined?(Spring) 7 | require 'rubygems' 8 | require 'bundler' 9 | 10 | lockfile = Bundler::LockfileParser.new(Bundler.default_lockfile.read) 11 | spring = lockfile.specs.detect { |spec| spec.name == "spring" } 12 | if spring 13 | Gem.use_paths Gem.dir, Bundler.bundle_path.to_s, *Gem.path 14 | gem 'spring', spring.version 15 | require 'spring/binstub' 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /.ebextensions/04_ruby.config: -------------------------------------------------------------------------------- 1 | option_settings: 2 | - option_name: BUNDLE_DISABLE_SHARED_GEMS 3 | value: "1" 4 | - option_name: BUNDLE_PATH 5 | value: "vendor/bundle" 6 | 7 | packages: 8 | yum: 9 | git: [] 10 | 11 | commands: 12 | 01-install-nodesource-repo: 13 | command: "curl --silent --location https://rpm.nodesource.com/setup_8.x | bash -" 14 | 02-install-nodejs: 15 | command: "yum -y install nodejs" 16 | 03-install-yarn-source: 17 | command: "wget https://dl.yarnpkg.com/rpm/yarn.repo -O /etc/yum.repos.d/yarn.repo" 18 | 04-install-yarn: 19 | command: "yum -y install yarn" -------------------------------------------------------------------------------- /lib/tasks/process.rake: -------------------------------------------------------------------------------- 1 | namespace :process do 2 | # abbreviations from http://sv08data.dot.ca.gov/contractcost/map.html 3 | task la: :environment do 4 | RapSheetProcessor.new(Counties::LOS_ANGELES).run 5 | end 6 | 7 | task sf: :environment do 8 | RapSheetProcessor.new(Counties::SAN_FRANCISCO).run 9 | end 10 | 11 | task sj: :environment do 12 | RapSheetProcessor.new(Counties::SAN_JOAQUIN).run 13 | end 14 | 15 | task cc: :environment do 16 | RapSheetProcessor.new(Counties::CONTRA_COSTA).run 17 | end 18 | 19 | task sac: :environment do 20 | RapSheetProcessor.new(Counties::SACRAMENTO).run 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /scripts/bastion_setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | IP=$1 5 | SSH_KEY=$2 6 | 7 | scp -i "${SSH_KEY}" scripts/adduser.sh ec2-user@${IP}: 8 | ssh ec2-user@${IP} -i "${SSH_KEY}" -C "chmod a+x ./adduser.sh && sudo ./adduser.sh" 9 | 10 | ssh ec2-user@${IP} -i "${SSH_KEY}" -C "sudo yum update -y && sudo yum install -y awslogs" 11 | scp -i "${SSH_KEY}" scripts/bastion_awslogs.conf ec2-user@${IP}:awslogs.conf 12 | scp -i "${SSH_KEY}" scripts/bastion_awscli.conf ec2-user@${IP}:awscli.conf 13 | ssh ec2-user@${IP} -i "${SSH_KEY}" -C "sudo mv awslogs.conf /etc/awslogs/awslogs.conf && sudo mv awscli.conf /etc/awslogs/awscli.conf && sudo service awslogs restart" 14 | -------------------------------------------------------------------------------- /config/initializers/assets.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Version of your assets, change this if you want to expire all your assets. 4 | Rails.application.config.assets.version = '1.0' 5 | 6 | # Add additional assets to the asset load path. 7 | # Rails.application.config.assets.paths << Emoji.images_path 8 | # Add Yarn node_modules folder to the asset load path. 9 | Rails.application.config.assets.paths << Rails.root.join('node_modules') 10 | 11 | # Precompile additional assets. 12 | # application.js, application.css, and all non-JS/CSS in the app/assets 13 | # folder are already added. 14 | # Rails.application.config.assets.precompile += %w( admin.js admin.css ) 15 | -------------------------------------------------------------------------------- /terraform/production/production.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "s3" { 3 | key = "terraform_state" 4 | region = "us-gov-west-1" 5 | } 6 | } 7 | 8 | # Specify the provider and access details 9 | provider "aws" { 10 | access_key = "${var.aws_access_key}" 11 | secret_key = "${var.aws_secret_key}" 12 | region = "${var.aws_region}" 13 | } 14 | 15 | module "main" { 16 | source = "../main" 17 | 18 | aws_az1 = "${var.aws_az1}" 19 | aws_az2 = "${var.aws_az2}" 20 | 21 | rds_username = "${var.rds_username}" 22 | 23 | environment = "${var.environment}" 24 | rails_secret_key_base = "${var.rails_secret_key_base}" 25 | 26 | key_name = "${var.key_name}" 27 | public_key = "${var.public_key}" 28 | } 29 | -------------------------------------------------------------------------------- /config/initializers/inflections.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Add new inflection rules using the following format. Inflections 4 | # are locale specific, and you may define rules for as many different 5 | # locales as you wish. All of these examples are active by default: 6 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 7 | # inflect.plural /^(ox)$/i, '\1en' 8 | # inflect.singular /^(ox)en/i, '\1' 9 | # inflect.irregular 'person', 'people' 10 | # inflect.uncountable %w( fish sheep ) 11 | # end 12 | 13 | # These inflection rules are supported but not enabled by default: 14 | # ActiveSupport::Inflector.inflections(:en) do |inflect| 15 | # inflect.acronym 'RESTful' 16 | # end 17 | -------------------------------------------------------------------------------- /config/database.yml: -------------------------------------------------------------------------------- 1 | default: &default 2 | adapter: postgresql 3 | pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> 4 | timeout: 5000 5 | host: localhost 6 | 7 | development: 8 | <<: *default 9 | database: autoclearance_development 10 | 11 | # Warning: The database defined as "test" will be erased and 12 | # re-generated from your development database when you run "rake". 13 | # Do not set this db to the same as development or production. 14 | test: 15 | <<: *default 16 | database: autoclearance_test 17 | 18 | production: 19 | <<: *default 20 | database: autoclearance 21 | host: <%= ENV.fetch("RDS_HOST", nil) %> 22 | username: <%= ENV.fetch("RDS_USERNAME", nil) %> 23 | password: <%= ENV.fetch("RDS_PASSWORD", nil) %> 24 | -------------------------------------------------------------------------------- /spec/features/upload_rap_sheet_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | feature 'uploading rap sheets' do 4 | scenario 'user uploads a few rap sheets', js: true do 5 | visit root_path 6 | expect(page).to have_content 'Upload RAP sheets' 7 | expect(page).to have_button('Upload', disabled: true) 8 | 9 | files = [Rails.root.join('spec', 'fixtures', 'chewbacca_rap_sheet.pdf'), Rails.root.join('spec', 'fixtures', 'skywalker_rap_sheet.pdf')] 10 | 11 | attach_file 'Select files', files, multiple: true, visible: false 12 | 13 | expect(page).to have_content 'chewbacca_rap_sheet.pdf' 14 | expect(page).to have_content 'skywalker_rap_sheet.pdf' 15 | 16 | expect(page).to have_button('Upload', disabled: false) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /.pgsync.yml: -------------------------------------------------------------------------------- 1 | # source database URL 2 | # database URLs take the format of: 3 | # postgres://user:password@host:port/dbname 4 | # we recommend a command which outputs a database URL 5 | # so sensitive information is not included in this file 6 | from: postgres://localhost:5432/autoclearance_development 7 | 8 | # destination database URL 9 | 10 | to: $(echo $DATABASE_URL) 11 | to_safe: true 12 | 13 | # define groups 14 | groups: 15 | all: 16 | - anon_rap_sheets 17 | - anon_cycles 18 | - anon_events 19 | - anon_counts 20 | - anon_dispositions 21 | - rap_sheet_properties 22 | - event_properties 23 | - count_properties 24 | - eligibility_estimates 25 | 26 | # exclude tables 27 | exclude: 28 | - schema_migrations 29 | - ar_internal_metadata 30 | -------------------------------------------------------------------------------- /app/helpers/pdf_reader.rb: -------------------------------------------------------------------------------- 1 | class PDFReader 2 | def initialize(file_contents) 3 | @file_contents = file_contents 4 | end 5 | 6 | def text 7 | reader = PDF::Reader.new(StringIO.new(file_contents)) 8 | reader.pages.map do |page| 9 | PDFReader.strip_page_header_and_footer(page.text) 10 | end.join 11 | end 12 | 13 | def self.strip_page_header_and_footer(page_text) 14 | lines = page_text.split("\n") 15 | lines.pop while lines.any? && (lines.last.strip.empty? || lines.last.lstrip.start_with?('http://', 'https://', 'file://', 'file://')) 16 | lines.shift while lines.any? && (lines.first.strip.empty? || lines.first.lstrip.start_with?('Page', 'Route')) 17 | lines.join("\n") + "\n" 18 | end 19 | 20 | attr_reader :file_contents 21 | end 22 | -------------------------------------------------------------------------------- /app/assets/javascripts/application.js: -------------------------------------------------------------------------------- 1 | // This is a manifest file that'll be compiled into application.js, which will include all the files 2 | // listed below. 3 | // 4 | // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, or any plugin's 5 | // vendor/assets/javascripts directory can be referenced here using a relative path. 6 | // 7 | // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the 8 | // compiled file. JavaScript code in this file should be added after the last require_* statement. 9 | // 10 | // Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details 11 | // about supported directives. 12 | // 13 | //= require rails-ujs 14 | //= require turbolinks 15 | //= require cfa_styleguide_main 16 | //= require vue/dist/vue 17 | 18 | //= require_tree . 19 | -------------------------------------------------------------------------------- /spec/models/eligibility_estimate_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe EligibilityEstimate do 4 | it 'populates fields' do 5 | count = build_count(code: 'HS', section: '11357') 6 | event = build_court_event( 7 | date: Date.new(2001, 1, 5), 8 | counts: [count] 9 | ) 10 | rap_sheet = build_rap_sheet(events: [event]) 11 | rap_sheet_with_eligibility = build_rap_sheet_with_eligibility(rap_sheet: rap_sheet) 12 | event_with_eligibility = EventWithEligibility.new(event: event, eligibility: rap_sheet_with_eligibility) 13 | prop64_classifier = Prop64Classifier.new(count: count, event: event_with_eligibility, eligibility: rap_sheet_with_eligibility) 14 | 15 | eligibility_estimate = described_class.build(prop64_classifier) 16 | 17 | expect(eligibility_estimate.prop_64_eligible).to eq 'yes' 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /config/application.rb: -------------------------------------------------------------------------------- 1 | require_relative 'boot' 2 | 3 | require 'rails/all' 4 | 5 | # Require the gems listed in Gemfile, including any gems 6 | # you've limited to :test, :development, or :production. 7 | Bundler.require(*Rails.groups) 8 | 9 | module Autoclearance 10 | class Application < Rails::Application 11 | # Initialize configuration defaults for originally generated Rails version. 12 | config.load_defaults 5.1 13 | 14 | # Settings in config/environments/* take precedence over those specified here. 15 | # Application configuration can go into files in config/initializers 16 | # -- all .rb files in that directory are automatically loaded after loading 17 | # the framework and any gems in your application. 18 | 19 | config.generators do |g| 20 | g.orm :active_record, primary_key_type: :uuid 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /.ebextensions/01_awslogs.config: -------------------------------------------------------------------------------- 1 | packages: 2 | yum: 3 | awslogs: [] 4 | 5 | files: 6 | "/etc/awslogs/awslogs.conf": 7 | mode: "000644" 8 | owner: root 9 | group: root 10 | content: | 11 | [general] 12 | state_file = /var/lib/awslogs/agent-state 13 | 14 | [/var/log/secure] 15 | datetime_format = %b %d %H:%M:%S 16 | file = /var/log/secure 17 | buffer_duration = 5000 18 | log_stream_name = {instance_id} 19 | initial_position = start_of_file 20 | log_group_name = /var/log/secure 21 | 22 | [rails] 23 | log_group_name = rails 24 | log_stream_name = {instance_id} 25 | datetime_format = %Y-%m-%dT%H:%M:%S.%f 26 | file = /var/app/containerfiles/logs/production.log* 27 | 28 | commands: 29 | 01-restart-service: 30 | command: "sudo service awslogs restart" 31 | 32 | -------------------------------------------------------------------------------- /app/assets/stylesheets/application.scss: -------------------------------------------------------------------------------- 1 | /* 2 | * This is a manifest file that'll be compiled into application.css, which will include all the files 3 | * listed below. 4 | * 5 | * Any CSS and SCSS file within this directory, lib/assets/stylesheets, or any plugin's 6 | * vendor/assets/stylesheets directory can be referenced here using a relative path. 7 | * 8 | * You're free to add application-wide styles to this file and they'll appear at the bottom of the 9 | * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS 10 | * files in this directory. Styles in this file should be added after the last require_* statement. 11 | * It is generally better to create a new file per style scope. 12 | * 13 | *= require_tree . 14 | *= require_self 15 | */ 16 | 17 | @import 'cfa_styleguide_main'; 18 | @import 'autoclearance'; 19 | -------------------------------------------------------------------------------- /terraform/main/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_az1" { 2 | description = "AWS availability zone 1" 3 | default = "us-gov-west-1a" 4 | } 5 | 6 | variable "aws_az2" { 7 | description = "AWS availability zone 2" 8 | default = "us-gov-west-1b" 9 | } 10 | 11 | variable "rds_username" {} 12 | 13 | variable "environment" { 14 | description = "environment name to append to s3 bucket names (e.g. staging, or prod)" 15 | } 16 | 17 | variable "rails_secret_key_base" { 18 | description = "Secret key base for Rails. Generated by running \"rake secret\"" 19 | } 20 | variable "key_name" { 21 | description = "Desired name of AWS key pair" 22 | } 23 | variable "public_key" { 24 | description = < 12 | # 13 | # To use a different locale, set it with `I18n.locale`: 14 | # 15 | # I18n.locale = :es 16 | # 17 | # This would use the information in config/locales/es.yml. 18 | # 19 | # The following keys must be escaped otherwise they will not be retrieved by 20 | # the default I18n backend: 21 | # 22 | # true, false, on, off, yes, no 23 | # 24 | # Instead, surround them with single quotes. 25 | # 26 | # en: 27 | # 'true': 'foo' 28 | # 29 | # To learn more, please read the Rails Internationalization guide 30 | # available at http://guides.rubyonrails.org/i18n.html. 31 | 32 | en: 33 | hello: "Hello world" 34 | -------------------------------------------------------------------------------- /app/models/count_properties.rb: -------------------------------------------------------------------------------- 1 | class CountProperties < ApplicationRecord 2 | belongs_to :anon_count 3 | has_one :eligibility_estimate, dependent: :destroy 4 | 5 | def self.build(count:, rap_sheet:, event:) 6 | rap_sheet_with_eligibility = RapSheetWithEligibility.new( 7 | rap_sheet: rap_sheet, 8 | county: { courthouses: [] }, 9 | logger: Logger.new('/dev/null') 10 | ) 11 | event_with_eligibility = EventWithEligibility.new( 12 | event: event, 13 | eligibility: rap_sheet_with_eligibility 14 | ) 15 | prop64_classifier = Prop64Classifier.new( 16 | count: count, 17 | event: event_with_eligibility, 18 | eligibility: rap_sheet_with_eligibility 19 | ) 20 | CountProperties.new( 21 | has_prop_64_code: prop64_classifier.prop64_conviction?, 22 | has_two_prop_64_priors: prop64_classifier.has_two_prop_64_priors?, 23 | prop_64_plea_bargain: prop64_classifier.plea_bargain, 24 | eligibility_estimate: EligibilityEstimate.build(prop64_classifier) 25 | ) 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'fileutils' 3 | include FileUtils 4 | 5 | # path to your application root. 6 | APP_ROOT = File.expand_path('..', __dir__) 7 | 8 | def system!(*args) 9 | system(*args) || abort("\n== Command #{args} failed ==") 10 | end 11 | 12 | chdir APP_ROOT do 13 | # This script is a starting point to setup your application. 14 | # Add necessary setup steps to this file. 15 | 16 | puts '== Installing dependencies ==' 17 | system! 'gem install bundler --conservative' 18 | system('bundle check') || system!('bundle install') 19 | 20 | # Install JavaScript dependencies if using Yarn 21 | # system('bin/yarn') 22 | 23 | # puts "\n== Copying sample files ==" 24 | # unless File.exist?('config/database.yml') 25 | # cp 'config/database.yml.sample', 'config/database.yml' 26 | # end 27 | 28 | puts "\n== Preparing database ==" 29 | system! 'bin/rails db:setup' 30 | 31 | puts "\n== Removing old logs and tempfiles ==" 32 | system! 'bin/rails log:clear tmp:clear' 33 | 34 | puts "\n== Restarting application server ==" 35 | system! 'bin/rails restart' 36 | end 37 | -------------------------------------------------------------------------------- /terraform/staging/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_region" { 2 | description = "AWS region to launch servers." 3 | default = "us-gov-west-1" 4 | } 5 | 6 | variable "aws_access_key" {} 7 | variable "aws_secret_key" {} 8 | 9 | variable "aws_az1" { 10 | description = "AWS availability zone 1" 11 | default = "us-gov-west-1a" 12 | } 13 | 14 | variable "aws_az2" { 15 | description = "AWS availability zone 2" 16 | default = "us-gov-west-1b" 17 | } 18 | 19 | variable "rds_username" {} 20 | 21 | variable "analysis_rds_username" {} 22 | 23 | variable "environment" { 24 | description = "environment name to append to s3 bucket names (e.g. staging, or prod)" 25 | } 26 | 27 | variable "rails_secret_key_base" { 28 | description = "Secret key base for Rails. Generated by running \"rake secret\"" 29 | } 30 | 31 | variable "key_name" { 32 | description = "Desired name of AWS key pair" 33 | } 34 | variable "public_key" { 35 | description = < Time.zone.today 34 | if sentence_still_active 35 | 'resentencing' 36 | else 37 | 'redesignation' 38 | end 39 | end 40 | 41 | private 42 | 43 | attr_reader :eligibility 44 | end 45 | -------------------------------------------------------------------------------- /spec/models/event_properties_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe EventProperties do 4 | describe '#build_from_eligibility' do 5 | it 'populates eligibility fields' do 6 | sentence = RapSheetParser::ConvictionSentence.new(probation: 1.year) 7 | disposition = build_disposition(type: 'convicted', severity: 'F', sentence: sentence) 8 | 9 | event = build_court_event( 10 | date: Date.new(2001, 1, 5), 11 | counts: [ 12 | build_count(code: 'PC', section: '456'), 13 | build_count(code: 'PC', section: '789', disposition: disposition) 14 | ] 15 | ) 16 | rap_sheet = build_rap_sheet( 17 | events: [event, build_arrest_event(date: Date.new(2001, 11, 4))] 18 | ) 19 | event_properties = described_class.build(event: event, rap_sheet: rap_sheet) 20 | 21 | expect(event_properties.has_felonies).to eq true 22 | expect(event_properties.has_probation).to eq true 23 | expect(event_properties.has_prison).to eq false 24 | expect(event_properties.has_probation_violations).to eq true 25 | expect(event_properties.dismissed_by_pc1203).to eq false 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /app/domain/rap_sheet_with_eligibility.rb: -------------------------------------------------------------------------------- 1 | class RapSheetWithEligibility < SimpleDelegator 2 | attr_reader :county 3 | 4 | def initialize(rap_sheet:, county:, logger:) 5 | super(rap_sheet) 6 | @county = county 7 | 8 | unless potentially_eligible_counts? 9 | logger.warn('No eligible prop64 convictions found') 10 | end 11 | end 12 | 13 | def eligible_events 14 | convictions 15 | .select { |e| in_courthouses?(e) } 16 | .reject(&:dismissed_by_pc1203?) 17 | .map { |e| EventWithEligibility.new(event: e, eligibility: self) } 18 | end 19 | 20 | def potentially_eligible_counts? 21 | eligible_events.any? do |eligible_event| 22 | eligible_event.potentially_eligible_counts.any? 23 | end 24 | end 25 | 26 | def disqualifiers? 27 | sex_offender_registration? || superstrikes.any? 28 | end 29 | 30 | def has_three_convictions_of_same_type?(code_section) 31 | convictions.select do |e| 32 | e.convicted_counts.any? do |c| 33 | c.subsection_of?([code_section]) 34 | end 35 | end.length >= 3 36 | end 37 | 38 | private 39 | 40 | def in_courthouses?(e) 41 | county[:courthouses].include? e.courthouse 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /config/initializers/content_security_policy.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Define an application-wide content security policy 4 | # For further information see the following documentation 5 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy 6 | 7 | # Rails.application.config.content_security_policy do |policy| 8 | # policy.default_src :self, :https 9 | # policy.font_src :self, :https, :data 10 | # policy.img_src :self, :https, :data 11 | # policy.object_src :none 12 | # policy.script_src :self, :https 13 | # policy.style_src :self, :https 14 | 15 | # # Specify URI for violation reports 16 | # # policy.report_uri "/csp-violation-report-endpoint" 17 | # end 18 | 19 | # If you are using UJS then enable automatic nonce generation 20 | # Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) } 21 | 22 | # Report CSP violations to a specified URI 23 | # For further information see the following documentation: 24 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only 25 | # Rails.application.config.content_security_policy_report_only = true 26 | -------------------------------------------------------------------------------- /scripts/create_postgres_user.sh: -------------------------------------------------------------------------------- 1 | # Not meant to be a re-runnable script,as much as documentation 2 | # on how we set up the RDS instance 3 | 4 | #!/usr/bin/env bash 5 | set -e 6 | 7 | DATABASE="" # database URL 8 | ROOT_USER="" # user with full create privileges 9 | ROOT_PASSWORD="" # password 10 | 11 | ROLE="readonly" 12 | DATABASE_NAME="analysis" 13 | 14 | # name of new user and password 15 | USER=$1 16 | PASSWORD=$2 17 | 18 | PGPASSWORD="$ROOT_PASSWORD" psql -U ${ROOT_USER} -h "$DATABASE" -c "CREATE ROLE $ROLE;" 19 | PGPASSWORD="$ROOT_PASSWORD" psql -U ${ROOT_USER} -h "$DATABASE" -c "GRANT CONNECT ON DATABASE $DATABASE_NAME TO $ROLE;" 20 | PGPASSWORD="$ROOT_PASSWORD" psql -U ${ROOT_USER} -h "$DATABASE" -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ROLE;" 21 | PGPASSWORD="$ROOT_PASSWORD" psql -U ${ROOT_USER} -h "$DATABASE" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ROLE;" 22 | 23 | PGPASSWORD="$ROOT_PASSWORD" psql -U ${ROOT_USER} -h "$DATABASE" -c "CREATE EXTENSION pgcrypto;" 24 | 25 | PGPASSWORD="$ROOT_PASSWORD" psql -U ${ROOT_USER} -h "$DATABASE" -c "CREATE USER $USER WITH PASSWORD '$PASSWORD';" 26 | PGPASSWORD="$ROOT_PASSWORD" psql -U ${ROOT_USER} -h "$DATABASE" -c "GRANT $ROLE TO $USER;" 27 | -------------------------------------------------------------------------------- /config/storage.yml: -------------------------------------------------------------------------------- 1 | test: 2 | service: Disk 3 | root: <%= Rails.root.join("tmp/storage") %> 4 | 5 | local: 6 | service: Disk 7 | root: <%= Rails.root.join("storage") %> 8 | 9 | # Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) 10 | # amazon: 11 | # service: S3 12 | # access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %> 13 | # secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %> 14 | # region: us-east-1 15 | # bucket: your_own_bucket 16 | 17 | # Remember not to checkin your GCS keyfile to a repository 18 | # google: 19 | # service: GCS 20 | # project: your_project 21 | # credentials: <%= Rails.root.join("path/to/gcs.keyfile") %> 22 | # bucket: your_own_bucket 23 | 24 | # Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key) 25 | # microsoft: 26 | # service: AzureStorage 27 | # storage_account_name: your_account_name 28 | # storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %> 29 | # container: your_container_name 30 | 31 | # mirror: 32 | # service: Mirror 33 | # primary: local 34 | # mirrors: [ amazon, google, microsoft ] 35 | -------------------------------------------------------------------------------- /app/models/anon_event.rb: -------------------------------------------------------------------------------- 1 | class AnonEvent < ApplicationRecord 2 | belongs_to :anon_cycle 3 | has_many :anon_counts, dependent: :destroy 4 | has_one :event_properties, dependent: :destroy 5 | 6 | VALID_EVENT_TYPES = %w[court arrest applicant probation custody registration supplemental_arrest deceased mental_health].freeze 7 | 8 | validates :event_type, presence: true 9 | validates :event_type, inclusion: { in: VALID_EVENT_TYPES } 10 | 11 | default_scope { order(date: :asc) } 12 | 13 | def self.build_from_parser(event:, rap_sheet:) 14 | counts = event.counts.map.with_index(1) do |count, i| 15 | AnonCount.build_from_parser( 16 | index: i, 17 | count: count, 18 | event: event, 19 | rap_sheet: rap_sheet 20 | ) 21 | end 22 | 23 | AnonEvent.new( 24 | agency: event.agency, 25 | event_type: event.event_type, 26 | date: event.date, 27 | anon_counts: counts, 28 | event_properties: event_properties(event: event, rap_sheet: rap_sheet) 29 | ) 30 | end 31 | 32 | def self.event_properties(event:, rap_sheet:) 33 | return unless event.is_a?(RapSheetParser::CourtEvent) && event.conviction? 34 | 35 | EventProperties.build(event: event, rap_sheet: rap_sheet) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /spec/models/anon_disposition_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe AnonDisposition do 4 | describe '#build_from_parser' do 5 | it 'creates AnonDispositions from parser output' do 6 | disposition = RapSheetParser::Disposition.new( 7 | type: 'conviction', 8 | sentence: RapSheetParser::ConvictionSentence.new(jail: 10.days), 9 | text: '*DISPO:CONVICTED', 10 | severity: 'M' 11 | ) 12 | anon_disposition = described_class.build_from_parser(disposition) 13 | expect(anon_disposition.disposition_type).to eq 'conviction' 14 | expect(anon_disposition.sentence).to eq '10d jail' 15 | expect(anon_disposition.severity).to eq 'M' 16 | expect(anon_disposition.text).to eq '*DISPO:CONVICTED' 17 | end 18 | 19 | it 'saves dispositions with no sentence' do 20 | disposition = RapSheetParser::Disposition.new( 21 | type: 'dismissed', 22 | sentence: nil, 23 | severity: 'F', 24 | text: 'DISPO:DISMISSED/FURTHERANCE OF JUSTICE' 25 | ) 26 | 27 | anon_disposition = described_class.build_from_parser(disposition) 28 | expect(anon_disposition.sentence).to be nil 29 | expect(anon_disposition.text).to eq 'DISPO:DISMISSED/FURTHERANCE OF JUSTICE' 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /spec/fixtures/summary.csv: -------------------------------------------------------------------------------- 1 | Filename,Name,Date,Case Number,Courthouse,Charge,Severity,Sentence,Superstrikes,2 prior convictions,PC290,Remedy,Eligible,Deceased 2 | chewbacca_rap_sheet.pdf,"BACCA, CHEW E.",1984-09-18,1234567,CASC San Francisco,HS 11360,F,"3y probation, 30d jail, fine, restitution",PC 220,false,true,redesignation,no,false 3 | chewbacca_rap_sheet.pdf,"BACCA, CHEW E.",1984-09-18,1234567,CASC San Francisco,HS 11358,F,"3y probation, 30d jail, fine, restitution",PC 220,true,true,redesignation,no,false 4 | chewbacca_rap_sheet.pdf,"BACCA, CHEW",2010-04-13,34567423423,CASC San Francisco,HS 11358,F,"2y probation, 5d jail",PC 220,true,true,resentencing,no,false 5 | skywalker_rap_sheet.pdf,"SKYWALKER,LUKE JAY",1984-09-18,1234567,CASC San Francisco,HS 11357,F,"3y probation, 30d jail, fine, restitution","",false,false,redesignation,yes,false 6 | skywalker_rap_sheet.pdf,"SKYWALKER,LUKE JAY",1984-09-18,1234567,CASC San Francisco,HS 11358,F,"3y probation, 30d jail, fine, restitution","",true,false,redesignation,no,false 7 | skywalker_rap_sheet.pdf,"SKYWALKER,LUKE JAY",1990-04-13,33857493,CASC San Francisco,HS 11358,F,"2y probation, 30d jail","",true,false,redesignation,no,false 8 | skywalker_rap_sheet.pdf,"SKYWALKER,LUKE JAY",1990-04-13,33857493,CASC San Francisco,PC 32,F,"2y probation, 30d jail","",false,false,redesignation,maybe,false 9 | -------------------------------------------------------------------------------- /app/views/layouts/application.html.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Autoclearance 5 | <%= csrf_meta_tags %> 6 | 7 | <%= stylesheet_link_tag 'application', media: 'all', 'data-turbolinks-track': 'reload' %> 8 | <%= javascript_include_tag 'application', 'data-turbolinks-track': 'reload' %> 9 | 10 | 11 | 12 |
13 |
14 |
15 |
16 |

Clear My Record

17 |
18 |
19 |
20 | <%= yield %> 21 |
22 |
23 |
24 |
25 |

Clear My Record is a service delivered by Code for America on behalf of the people of California.

26 |
27 |
28 | 31 |
32 |
33 |
34 | 35 | 36 | -------------------------------------------------------------------------------- /config/secrets.yml: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | 3 | # Your secret key is used for verifying the integrity of signed cookies. 4 | # If you change this key, all old signed cookies will become invalid! 5 | 6 | # Make sure the secret is at least 30 characters and all random, 7 | # no regular words or you'll be exposed to dictionary attacks. 8 | # You can use `rails secret` to generate a secure secret key. 9 | 10 | # Make sure the secrets in this file are kept private 11 | # if you're sharing your code publicly. 12 | 13 | # Shared secrets are available across all environments. 14 | 15 | # shared: 16 | # api_key: a1B2c3D4e5F6 17 | 18 | # Environmental secrets are only available for that specific environment. 19 | 20 | development: 21 | secret_key_base: 3a657a676941e7f8403905aeea63c925df1d04a8e53f98779a6323ba575cfa429ddb55bc39214d47855dc1201b6eb6381660f5f6592392b06ea995bfecd0c240 22 | 23 | test: 24 | secret_key_base: fd23d16e2e0d13a848c2b48d85ee1221c8840cf569b6157e9f3b32bb056e5b23bfe9ecb7d86bcde3b7b81a79374f720c61c18aab5ff4fb466ca2dfb678a75497 25 | 26 | # Do not keep production secrets in the unencrypted secrets file. 27 | # Instead, either read values from the environment. 28 | # Or, use `bin/rails secrets:setup` to configure encrypted secrets 29 | # and move the `production:` environment over there. 30 | 31 | production: 32 | secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> 33 | -------------------------------------------------------------------------------- /terraform/staging/staging.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "s3" { 3 | key = "terraform_state" 4 | region = "us-gov-west-1" 5 | } 6 | } 7 | 8 | # Specify the provider and access details 9 | provider "aws" { 10 | access_key = "${var.aws_access_key}" 11 | secret_key = "${var.aws_secret_key}" 12 | region = "${var.aws_region}" 13 | } 14 | 15 | module "main" { 16 | source = "../main" 17 | 18 | aws_az1 = "${var.aws_az1}" 19 | aws_az2 = "${var.aws_az2}" 20 | 21 | rds_username = "${var.rds_username}" 22 | 23 | analysis_rds_username = "${var.analysis_rds_username}" 24 | 25 | environment = "${var.environment}" 26 | rails_secret_key_base = "${var.rails_secret_key_base}" 27 | 28 | key_name = "${var.key_name}" 29 | public_key = "${var.public_key}" 30 | } 31 | 32 | module "metabase" { 33 | source = "../metabase" 34 | vpc_id = "${module.main.aws_vpc_default_id}" 35 | public_subnet_id = "${module.main.aws_subnet_public_id}" 36 | private_subnet_id = "${module.main.aws_subnet_private_id}" 37 | db_subnet_group_name = "${module.main.aws_db_subnet_group_default_name}" 38 | beanstalk_role_name = "${module.main.aws_iam_role_beanstalk_role_name}" 39 | metabase_url = "data.staging.clearmyrecord.org" 40 | } 41 | 42 | output "rds_password" { 43 | value = "${module.main.rds_password}" 44 | } 45 | 46 | output "analysis_database_url" { 47 | value = "${module.main.analysis_database_url}" 48 | } 49 | -------------------------------------------------------------------------------- /db/migrate/20180814195919_create_anon_cycles.rb: -------------------------------------------------------------------------------- 1 | class CreateAnonCycles < ActiveRecord::Migration[5.2] 2 | class AnonEvent < ApplicationRecord 3 | end 4 | 5 | class AnonCycle < ApplicationRecord 6 | end 7 | 8 | def up 9 | create_table :anon_cycles, id: :uuid do |t| 10 | t.timestamps 11 | t.references :anon_rap_sheet, type: :uuid, foreign_key: true, null: false 12 | end 13 | 14 | add_reference :anon_events, :anon_cycle, type: :uuid, foreign_key: true 15 | 16 | AnonEvent.reset_column_information 17 | AnonCycle.reset_column_information 18 | AnonEvent.find_each do |event| 19 | cycle = AnonCycle.create!(anon_rap_sheet_id: event.anon_rap_sheet_id) 20 | event.anon_cycle_id = cycle.id 21 | event.save! 22 | end 23 | 24 | remove_reference :anon_events, :anon_rap_sheet 25 | change_column_null :anon_events, :anon_cycle_id, false 26 | end 27 | 28 | def down 29 | add_reference :anon_events, :anon_rap_sheet, type: :uuid, foreign_key: true 30 | 31 | AnonEvent.reset_column_information 32 | AnonCycle.reset_column_information 33 | AnonEvent.find_each do |event| 34 | cycle = AnonCycle.find(event.anon_cycle_id) 35 | event.anon_rap_sheet_id = cycle.anon_rap_sheet_id 36 | event.save! 37 | end 38 | 39 | remove_reference :anon_events, :anon_cycle 40 | change_column_null :anon_events, :anon_rap_sheet_id, false 41 | drop_table :anon_cycles 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /app/models/anon_rap_sheet.rb: -------------------------------------------------------------------------------- 1 | class AnonRapSheet < ApplicationRecord 2 | validates :county, :checksum, presence: true 3 | has_many :anon_cycles, dependent: :destroy 4 | has_one :rap_sheet_properties, dependent: :destroy 5 | 6 | def self.create_or_update(text:, county:, rap_sheet:) 7 | checksum = Digest::SHA2.new.digest text 8 | 9 | existing_rap_sheet = AnonRapSheet.find_by(checksum: checksum) 10 | existing_rap_sheet.destroy! if existing_rap_sheet 11 | 12 | self.create_from_parser(rap_sheet, county: county, checksum: checksum) 13 | end 14 | 15 | def self.create_from_parser(rap_sheet, county:, checksum:) 16 | cycles = rap_sheet.cycles.map do |cycle| 17 | AnonCycle.build_from_parser(cycle: cycle, rap_sheet: rap_sheet) 18 | end 19 | 20 | rap_sheet_properties = RapSheetProperties.build(rap_sheet: rap_sheet) 21 | 22 | AnonRapSheet.create!( 23 | checksum: checksum, 24 | county: county, 25 | year_of_birth: rap_sheet&.personal_info&.date_of_birth&.year, 26 | sex: rap_sheet&.personal_info&.sex, 27 | race: rap_sheet&.personal_info&.race, 28 | anon_cycles: cycles, 29 | person_unique_id: compute_person_unique_id(rap_sheet.personal_info), 30 | rap_sheet_properties: rap_sheet_properties 31 | ) 32 | end 33 | 34 | private 35 | 36 | def self.compute_person_unique_id(personal_info) 37 | return unless personal_info 38 | Digest::SHA2.new.digest "#{personal_info.cii}#{personal_info.names[0]}#{personal_info.date_of_birth}" 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /config/puma.rb: -------------------------------------------------------------------------------- 1 | # Puma can serve each request in a thread from an internal thread pool. 2 | # The `threads` method setting takes two numbers: a minimum and maximum. 3 | # Any libraries that use thread pools should be configured to match 4 | # the maximum value specified for Puma. Default is set to 5 threads for minimum 5 | # and maximum; this matches the default thread size of Active Record. 6 | # 7 | threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 } 8 | threads threads_count, threads_count 9 | 10 | # Specifies the `port` that Puma will listen on to receive requests; default is 3000. 11 | # 12 | port ENV.fetch("PORT") { 3000 } 13 | 14 | # Specifies the `environment` that Puma will run in. 15 | # 16 | environment ENV.fetch("RAILS_ENV") { "development" } 17 | 18 | # Specifies the number of `workers` to boot in clustered mode. 19 | # Workers are forked webserver processes. If using threads and workers together 20 | # the concurrency of the application would be max `threads` * `workers`. 21 | # Workers do not work on JRuby or Windows (both of which do not support 22 | # processes). 23 | # 24 | # workers ENV.fetch("WEB_CONCURRENCY") { 2 } 25 | 26 | # Use the `preload_app!` method when specifying a `workers` number. 27 | # This directive tells Puma to first boot the application and load code 28 | # before forking the application. This takes advantage of Copy On Write 29 | # process behavior so workers use less memory. 30 | # 31 | # preload_app! 32 | 33 | # Allow puma to be restarted by `rails restart` command. 34 | plugin :tmp_restart 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Rails template 3 | *.rbc 4 | capybara-*.html 5 | .rspec 6 | /log 7 | /tmp 8 | /public/system 9 | /coverage/ 10 | /spec/tmp 11 | *.orig 12 | rerun.txt 13 | pickle-email-*.html 14 | 15 | # TODO Comment out this rule if you are OK with secrets being uploaded to the repo 16 | config/initializers/secret_token.rb 17 | config/master.key 18 | 19 | # Only include if you have production secrets in this file, which is no longer a Rails default 20 | # config/secrets.yml 21 | 22 | # dotenv 23 | # TODO Comment out this rule if environment variables can be committed 24 | .env 25 | 26 | ## Environment normalization: 27 | /.bundle 28 | /vendor/bundle 29 | 30 | # these should all be checked in to normalize the environment: 31 | # Gemfile.lock, .ruby-version, .ruby-gemset 32 | 33 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 34 | .rvmrc 35 | 36 | # if using bower-rails ignore default bower_components path bower.json files 37 | /vendor/assets/bower_components 38 | *.bowerrc 39 | bower.json 40 | 41 | # Ignore pow environment settings 42 | .powenv 43 | 44 | # Ignore Byebug command history file. 45 | .byebug_history 46 | 47 | # Ignore node_modules 48 | node_modules/ 49 | 50 | # Terraform 51 | .terraform/ 52 | terraform/**/varfile 53 | terraform/**/backend-config 54 | *.tfstate 55 | *.tfstate.backup 56 | 57 | .idea/ 58 | 59 | # Elastic Beanstalk Files 60 | !.elasticbeanstalk/*.cfg.yml 61 | !.elasticbeanstalk/*.global.yml 62 | 63 | tags 64 | .DS_Store 65 | test_data/ 66 | -------------------------------------------------------------------------------- /app/views/uploads/new.html.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

Upload RAP sheets

6 |
7 |
8 |
9 | 10 |
11 | <%= form_tag(multipart: true) do %> 12 | <%= label_tag 'Select files', nil, class: 'button' do %> 13 | Select Files 14 | <% end %> 15 | <%= file_field_tag 'Select files', { accept: 'application/pdf', class: 'file-upload__input', multiple: true, 'v-on:change': 'fileInputChanged' } %> 16 | <% end %> 17 |
18 |
19 |
20 |
21 |
    22 |
  • 23 |
    {{ file.name }}
    24 |
    • 25 |
26 |
27 |
28 | 29 |
30 |
31 |
32 |
33 | 34 | -------------------------------------------------------------------------------- /app/domain/prop64_classifier.rb: -------------------------------------------------------------------------------- 1 | class Prop64Classifier < SimpleDelegator 2 | attr_reader :event, :eligibility 3 | 4 | def initialize(count:, event:, eligibility:) 5 | super(count) 6 | @event = event 7 | @eligibility = eligibility 8 | end 9 | 10 | def potentially_eligible? 11 | return false if county_filtered 12 | 13 | prop64_conviction? || plea_bargain_classifier.possible_plea_bargain? 14 | end 15 | 16 | def prop64_conviction? 17 | subsection_of?(CodeSections::PROP64_DISMISSIBLE) && !subsection_of?(CodeSections::PROP64_INELIGIBLE) 18 | end 19 | 20 | def eligible? 21 | return 'no' unless potentially_eligible? && !has_disqualifiers? && !has_two_prop_64_priors? 22 | return 'yes' if plea_bargain_classifier.plea_bargain? 23 | return 'maybe' if plea_bargain_classifier.possible_plea_bargain? 24 | 'yes' 25 | end 26 | 27 | def has_two_prop_64_priors? 28 | two_priors_codes = ['HS 11358', 'HS 11359', 'HS 11360'] 29 | two_priors_codes.include?(code_section) && eligibility.has_three_convictions_of_same_type?(code_section) 30 | end 31 | 32 | def plea_bargain 33 | return 'yes' if plea_bargain_classifier.plea_bargain? 34 | return 'maybe' if plea_bargain_classifier.possible_plea_bargain? 35 | 'no' 36 | end 37 | 38 | private 39 | 40 | def county_filtered 41 | eligibility.county[:misdemeanors] == false && disposition.severity == 'M' 42 | end 43 | 44 | def plea_bargain_classifier 45 | PleaBargainClassifier.new(self) 46 | end 47 | 48 | def has_disqualifiers? 49 | eligibility.disqualifiers? 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /app/domain/summary_csv.rb: -------------------------------------------------------------------------------- 1 | class SummaryCSV 2 | def initialize 3 | @rows = [] 4 | end 5 | 6 | def append(filename, eligibility) 7 | eligibility.eligible_events.each do |event| 8 | event.potentially_eligible_counts.each do |count| 9 | rows << [filename] + count_data(count, event, eligibility) 10 | end 11 | end 12 | end 13 | 14 | def text 15 | CSV.generate do |csv| 16 | csv << header 17 | rows.each { |row| csv << row } 18 | end 19 | end 20 | 21 | private 22 | 23 | def header 24 | [ 25 | 'Filename', 26 | 'Name', 27 | 'Date', 28 | 'Case Number', 29 | 'Courthouse', 30 | 'Charge', 31 | 'Severity', 32 | 'Sentence', 33 | 'Superstrikes', 34 | '2 prior convictions', 35 | 'PC290', 36 | 'Remedy', 37 | 'Eligible', 38 | 'Deceased' 39 | ] 40 | end 41 | 42 | def count_data(count, event, eligibility) 43 | [ 44 | name(event, eligibility.personal_info), 45 | event.date, 46 | event.case_number, 47 | event.courthouse, 48 | count.code_section, 49 | count.disposition.severity, 50 | event.sentence, 51 | eligibility.superstrikes.map(&:code_section).join(', '), 52 | eligibility.has_three_convictions_of_same_type?(count.code_section), 53 | eligibility.sex_offender_registration?, 54 | event.remedy, 55 | count.eligible?, 56 | eligibility.deceased_events.any? 57 | ] 58 | end 59 | 60 | def name(event, personal_info) 61 | return unless personal_info 62 | 63 | personal_info.names[event.name_code] 64 | end 65 | 66 | attr_reader :rows 67 | end 68 | -------------------------------------------------------------------------------- /app/helpers/counties.rb: -------------------------------------------------------------------------------- 1 | module Counties 2 | SAN_FRANCISCO = { 3 | name: 'San Francisco', 4 | courthouses: ['CASC San Francisco', 'CASC San Francisco Co', 'CAMC San Francisco'], 5 | misdemeanors: false, 6 | ada: { name: 'Sharon Woo', state_bar_number: '148139' } 7 | }.freeze 8 | LOS_ANGELES = { 9 | name: 'Los Angeles', 10 | courthouses: [ 11 | 'CAMC Beverly Hills', 12 | 'CAMC Compton', 13 | 'CAMC Culver City', 14 | 'CAMC El Monte', 15 | 'CAMC Glendale', 16 | 'CAMC Hollywood', 17 | 'CAMC Long Beach', 18 | 'CAMC Los Angeles Metro', 19 | 'CAMC San Fernando', 20 | 'CAMC Santa Monica', 21 | 'CAMC Van Nuys', 22 | 'CASC Alhambra', 23 | 'CASC Beverly Hills', 24 | 'CASC Glendale', 25 | 'CASC Long Beach', 26 | 'CASC Los Angeles', 27 | 'CASC MC Van Nuys', 28 | 'CASC San Fernando', 29 | 'CASC West Covina', 30 | 'CASC West LA Airport', 31 | 'CASC Los Angeles Central' 32 | ] 33 | }.freeze 34 | SAN_JOAQUIN = { 35 | name: 'San Joaquin', 36 | courthouses: [ 37 | 'CAMC Lodi', 38 | 'CAMC Stockton', 39 | 'CASC Lodi', 40 | 'CASC Manteca', 41 | 'CASC Stockton', 42 | 'CASC Tracy' 43 | ] 44 | }.freeze 45 | CONTRA_COSTA = { 46 | name: 'Contra Costa', 47 | courthouses: [ 48 | 'CAMC Richmond', 49 | 'CAMC Walnut Creek', 50 | 'CASC Richmond', 51 | 'CASC Walnut Creek' 52 | ] 53 | }.freeze 54 | SACRAMENTO = { 55 | name: 'Sacramento', 56 | courthouses: [ 57 | 'CAJV Sacramento', 58 | 'CASC MC Sacramento', 59 | 'CASC Sacramento' 60 | ] 61 | }.freeze 62 | end 63 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | working_directory: ~/autoclearance 5 | 6 | # Primary container image where all commands run 7 | 8 | docker: 9 | - image: circleci/ruby:2.5.1-node-browsers 10 | environment: 11 | RAILS_ENV: test 12 | PGHOST: 127.0.0.1 13 | PGUSER: root 14 | 15 | # Service container image available at `host: localhost` 16 | 17 | - image: circleci/postgres:9.6.10-alpine 18 | environment: 19 | POSTGRES_USER: root 20 | POSTGRES_DB: circle-test_test 21 | 22 | steps: 23 | - checkout 24 | 25 | # Restore bundle cache 26 | - restore_cache: 27 | keys: 28 | - autoclearance-{{ checksum "Gemfile.lock" }} 29 | - autoclearance- 30 | 31 | # Bundle install dependencies 32 | - run: 33 | name: Install dependencies 34 | command: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs 4 --retry 3 35 | 36 | - run: yarn install 37 | 38 | - run: sudo apt install -y postgresql-client pdftk qt5-default libqt5webkit5-dev enscript ghostscript || true 39 | 40 | # Store bundle cache 41 | - save_cache: 42 | key: autoclearance-{{ checksum "Gemfile.lock" }} 43 | paths: 44 | - vendor/bundle 45 | 46 | - run: 47 | name: Database Setup 48 | command: | 49 | bundle exec rake db:create 50 | bundle exec rake db:schema:load 51 | 52 | - run: 53 | name: Run rake 54 | command: bin/rake 55 | 56 | # Save artifacts 57 | - store_test_results: 58 | path: /tmp/test-results 59 | 60 | workflows: 61 | version: 2 62 | build-and-deploy: 63 | jobs: 64 | - build: 65 | filters: 66 | branches: 67 | only: 68 | - master 69 | -------------------------------------------------------------------------------- /spec/models/rap_sheet_properties_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe RapSheetProperties do 4 | describe '#build_from_eligibility' do 5 | let(:subject) { RapSheetProperties.build(rap_sheet: rap_sheet) } 6 | 7 | let(:rap_sheet) do 8 | build_rap_sheet(events: events) 9 | end 10 | 11 | let(:events) do 12 | [ 13 | build_court_event(counts: [build_count(code: 'HS', section: '11358')]), 14 | event 15 | ] 16 | end 17 | 18 | context 'when there are no superstrikes, deceased events, or sex offender registration' do 19 | let(:event) { build_court_event } 20 | 21 | it 'creates RapSheetProperties with superstrikes false' do 22 | expect(subject.has_superstrikes).to eq false 23 | expect(subject.deceased).to eq false 24 | expect(subject.has_sex_offender_registration).to eq false 25 | end 26 | end 27 | 28 | context 'when there are superstrikes' do 29 | let(:event) { build_court_event(counts: [build_count(code: 'PC', section: '187')]) } 30 | 31 | it 'creates RapSheetProperties with superstrikes true' do 32 | expect(subject.has_superstrikes).to eq true 33 | end 34 | end 35 | 36 | context 'when there is a deceased event' do 37 | let(:event) { build_other_event(event_type: 'deceased') } 38 | 39 | it 'creates RapSheetProperties deceased true' do 40 | expect(subject.deceased).to eq true 41 | end 42 | end 43 | 44 | context 'when there is a sex offense registration' do 45 | let(:event) { build_other_event(event_type: 'registration', counts: [build_count(code: 'PC', section: '290')]) } 46 | 47 | it 'creates RapSheetProperties with sex offender registration true' do 48 | expect(subject.has_sex_offender_registration).to eq true 49 | end 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /app/classifiers/plea_bargain_classifier.rb: -------------------------------------------------------------------------------- 1 | class PleaBargainClassifier 2 | def initialize(prop64_classifier) 3 | @prop64_classifier = prop64_classifier 4 | end 5 | 6 | def plea_bargain? 7 | prop64_classifier.subsection_of?(possible_plea_bargain_codes) && code_sections_in_cycle_are_potentially_eligible 8 | end 9 | 10 | def possible_plea_bargain? 11 | prop64_classifier.subsection_of?(possible_plea_bargain_codes) && cycle_contains_prop64_count 12 | end 13 | 14 | private 15 | 16 | attr_reader :prop64_classifier 17 | 18 | def possible_plea_bargain_codes 19 | [ 20 | 'PC 32', 21 | 'HS 11364', 22 | 'HS 11366' 23 | ] 24 | end 25 | 26 | def code_sections_in_cycle_are_potentially_eligible 27 | unrejected_counts_for_event_cycle.all? do |count| 28 | count.subsection_of?(possible_plea_bargain_codes) || prop64_conviction?(count) 29 | end 30 | end 31 | 32 | def unrejected_counts_for_event_cycle 33 | prop64_classifier.event.cycle_events.flat_map do |event| 34 | event.counts.reject do |count| 35 | count_prosecutor_rejected(count) || prosecutor_rejected_update(count) 36 | end 37 | end 38 | end 39 | 40 | def cycle_contains_prop64_count 41 | unrejected_counts_for_event_cycle.any? { |c| prop64_conviction?(c) } 42 | end 43 | 44 | def count_prosecutor_rejected(count) 45 | count.disposition && count.disposition.type == 'prosecutor_rejected' 46 | end 47 | 48 | def prosecutor_rejected_update(count) 49 | count.updates.any? do |update| 50 | update.dispositions.any? do |disposition| 51 | disposition.type == 'prosecutor_rejected' 52 | end 53 | end 54 | end 55 | 56 | def prop64_conviction?(count) 57 | count.subsection_of?(CodeSections::PROP64_DISMISSIBLE) && !count.subsection_of?(CodeSections::PROP64_INELIGIBLE) 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /config/initializers/new_framework_defaults_5_2.rb: -------------------------------------------------------------------------------- 1 | # Be sure to restart your server when you modify this file. 2 | # 3 | # This file contains migration options to ease your Rails 5.2 upgrade. 4 | # 5 | # Once upgraded flip defaults one by one to migrate to the new default. 6 | # 7 | # Read the Guide for Upgrading Ruby on Rails for more info on each option. 8 | 9 | # Make Active Record use stable #cache_key alongside new #cache_version method. 10 | # This is needed for recyclable cache keys. 11 | # Rails.application.config.active_record.cache_versioning = true 12 | 13 | # Use AES-256-GCM authenticated encryption for encrypted cookies. 14 | # Also, embed cookie expiry in signed or encrypted cookies for increased security. 15 | # 16 | # This option is not backwards compatible with earlier Rails versions. 17 | # It's best enabled when your entire app is migrated and stable on 5.2. 18 | # 19 | # Existing cookies will be converted on read then written with the new scheme. 20 | # Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true 21 | 22 | # Use AES-256-GCM authenticated encryption as default cipher for encrypting messages 23 | # instead of AES-256-CBC, when use_authenticated_message_encryption is set to true. 24 | # Rails.application.config.active_support.use_authenticated_message_encryption = true 25 | 26 | # Add default protection from forgery to ActionController::Base instead of in 27 | # ApplicationController. 28 | # Rails.application.config.action_controller.default_protect_from_forgery = true 29 | 30 | # Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and 31 | # 'f' after migrating old data. 32 | Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true 33 | 34 | # Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header. 35 | # Rails.application.config.active_support.use_sha1_digests = true 36 | -------------------------------------------------------------------------------- /public/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | We're sorry, but something went wrong (500) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

We're sorry, but something went wrong.

62 |
63 |

If you are the application owner check the logs for more information.

64 |
65 | 66 | 67 | -------------------------------------------------------------------------------- /public/422.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The change you wanted was rejected (422) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The change you wanted was rejected.

62 |

Maybe you tried to change something you didn't have access to.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /public/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The page you were looking for doesn't exist (404) 5 | 6 | 55 | 56 | 57 | 58 | 59 |
60 |
61 |

The page you were looking for doesn't exist.

62 |

You may have mistyped the address or the page may have moved.

63 |
64 |

If you are the application owner check the logs for more information.

65 |
66 | 67 | 68 | -------------------------------------------------------------------------------- /scripts/adduser.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | USERNAME_1='lkogler' 4 | KEY_1="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaSVSLKZ4dnw2KrwGExsTf1ZHd5d3VhhxFAWZi9kK/gi02U2bebsxHlBUcIYhU/q49A6zod/3hve8kUH5GGfbvLn/Gnjp4ivDG6vQPQdbTHYFYjNb3sIs+1wKdCDyXKhTdhkEjGyAa20sxbKezjRlzNo+RT7NbBk3l4ASX6uU3IDdmwXP3pRVsi14ao3KsXo1zaeC8LtWxNAoCMH0uWNhVdCtXssyEmMHE0s4yJFLGhKVyxaAsZkw6ADM7zRcRY5CXIeXVvXwD2HQ3m4FBiqQuioeYp0JO8CwldPoONuahMehTv7dMaP5xybXVY2kyAp/UU7sZRWANwGmpB6t7yrlL" 5 | 6 | USERNAME_2='zak' 7 | KEY_2="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDlgG+Yepuof2fQwAQHK8Nc/fIddk3oTJTONXvGx0O0f3hKid31rsAPqM7+nIKYXL4fb9QVdWOyi0a/AquOms4yPFbDFFTKH/uCPaEAxu338lGtbuCEvvUMF48p5cZvXc0ELVOtag7VJ+RCTcyv2WOW8c7tZuyUM/IZjnq4KWVv25TtBOwmOxk0xiQop9Y1lMyS+VQSmSTUCoIc/lF6YRQ3t9yb5pMgQ5UKyPJupWxu1PTkOiO08+7DgOlRNpvv/LdyGnP6q5m5XQPXHKmpSQUKZ7hHmDYzUXrkhYBIm2sUiFITBaILUvJCCeNbAHJKU9X5mqw/BMchfVi7E0EHNNAD zak@zaksoup.com" 8 | 9 | USERNAME_3='symonnesingleton' 10 | KEY_3="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDDgK94ygoI90a9poDk7kVO7w2nvulPNGMbJGcK68vx6BJC6ekjHchYiITI7PdjqAz4rZ/sEHzE75DlHeetMOis9hZewjgdNegdA0v0ms7cuDScKICPOzct4ylskotMyfEcLCmqGZwKzKCG7YDr5LpQ6SUVK6o+4zNY4G/nUzW+Lhk+Uu6xg16IH21Nzzk7bhhqG3hqrQa9wgMji0pQdsTyuJb0RYJnhh/2vM8VV4xwI9YtTesunLMaKrJuGnL3CyXWx8AZ0vV2QRkKp/TzW52P16r0MmGQ2TSoNtw4iMMC3fmRg73VzT8zWI8NSP2qen+TKx63xWDAsiNX6BTD8nlWl0+fnFOsormILwzuW+yaOPpV7M8SL1L25Zrmb1TbEPfGHNNUdWmq1Yzvtq5d4awNPFmGMGQtayX3iZDSqluaG1F5cRBq3Pox21FFHYkPG3V7spSZe9h5I0qfEKr8dV3OFzakpONQGdAW74Egab9xiFPSr4IUQb3EiwAjNFhC0c4V1V25XCtmcUKeTgvBK2j/Jn60hj6jJVdWypjMYaJMN2o1lK/whIoCqN1KSaJo+GjMFO0oDrG3wn2cnt8r+4+21+UrIoWfvRZwg6MKtxqEvrP13L3wk5GYvKg7s1DxXQwdZmUv+1IHtHUFO447NAXuytn00s1sgPcgBOGE1ldilw== symonne@codeforamerica.org" 11 | 12 | function set_up_user() { 13 | USERNAME=$1 14 | KEY=$2 15 | 16 | if ! id -u ${USERNAME} 17 | then 18 | echo "setting up $USERNAME" 19 | sudo adduser ${USERNAME} 20 | sudo su - ${USERNAME} <> .ssh/authorized_keys 26 | COMMAND 27 | fi 28 | } 29 | 30 | set_up_user ${USERNAME_1} "${KEY_1}" 31 | set_up_user ${USERNAME_2} "${KEY_2}" 32 | set_up_user ${USERNAME_3} "${KEY_3}" 33 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | git_source(:github) do |repo_name| 4 | repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?('/') 5 | "https://github.com/#{repo_name}.git" 6 | end 7 | 8 | gem 'pg' 9 | gem 'rails', '~> 5.2.2' 10 | # Use Puma as the app server 11 | gem 'puma', '~> 3.7' 12 | # Use SCSS for stylesheets 13 | gem 'sass-rails', '~> 5.0' 14 | # Use Uglifier as compressor for JavaScript assets 15 | gem 'uglifier', '>= 1.3.0' 16 | # See https://github.com/rails/execjs#readme for more supported runtimes 17 | # gem 'therubyracer', platforms: :ruby 18 | gem 'bootsnap' 19 | # Use CoffeeScript for .coffee assets and views 20 | gem 'coffee-rails', '~> 4.2' 21 | # Turbolinks makes navigating your web application faster. Read more: https://github.com/turbolinks/turbolinks 22 | gem 'turbolinks', '~> 5' 23 | # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder 24 | gem 'jbuilder', '~> 2.5' 25 | # Use Redis adapter to run Action Cable in production 26 | # gem 'redis', '~> 4.0' 27 | # Use ActiveModel has_secure_password 28 | # gem 'bcrypt', '~> 3.1.7' 29 | 30 | # Use Capistrano for deployment 31 | # gem 'capistrano-rails', group: :development 32 | gem 'fog-aws' 33 | gem 'pdf-reader' 34 | gem 'rap_sheet_parser', git: 'https://github.com/codeforamerica/rap_sheet_parser' 35 | # gem 'rap_sheet_parser', :path => '../rap_sheet_parser' 36 | 37 | gem 'cliver' 38 | gem 'pdf-forms' 39 | 40 | gem 'cfa-styleguide', git: 'https://github.com/codeforamerica/cfa-styleguide-gem' 41 | 42 | group :development, :test do 43 | gem 'bundler-audit' 44 | gem 'capybara', '~> 2.13' 45 | gem 'capybara-selenium' 46 | gem 'fog-local' 47 | gem 'pgsync' 48 | gem 'pry-byebug' 49 | gem 'rspec-rails' 50 | gem 'rubocop', '~> 0.58.2' 51 | gem 'selenium-webdriver' 52 | end 53 | 54 | group :development do 55 | gem 'listen', '>= 3.0.5', '< 3.2' 56 | # Access an IRB console on exception pages or by using <%= console %> anywhere in the code. 57 | gem 'web-console', '>= 3.3.0' 58 | # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring 59 | gem 'spring' 60 | gem 'spring-watcher-listen', '~> 2.0.0' 61 | end 62 | 63 | # Windows does not include zoneinfo files, so bundle the tzinfo-data gem 64 | gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] 65 | -------------------------------------------------------------------------------- /config/environments/test.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # The test environment is used exclusively to run your application's 5 | # test suite. You never need to work with it otherwise. Remember that 6 | # your test database is "scratch space" for the test suite and is wiped 7 | # and recreated between test runs. Don't rely on the data there! 8 | config.cache_classes = true 9 | 10 | # Do not eager load code on boot. This avoids loading your whole application 11 | # just for the purpose of running a single test. If you are using a tool that 12 | # preloads Rails for running tests, you may have to set it to true. 13 | config.eager_load = false 14 | 15 | # Configure public file server for tests with Cache-Control for performance. 16 | config.public_file_server.enabled = true 17 | config.public_file_server.headers = { 18 | 'Cache-Control' => "public, max-age=#{1.hour.to_i}" 19 | } 20 | 21 | # Show full error reports and disable caching. 22 | config.consider_all_requests_local = true 23 | config.action_controller.perform_caching = false 24 | 25 | # Raise exceptions instead of rendering exception templates. 26 | config.action_dispatch.show_exceptions = false 27 | 28 | # Disable request forgery protection in test environment. 29 | config.action_controller.allow_forgery_protection = false 30 | 31 | # Store uploaded files on the local file system in a temporary directory 32 | config.active_storage.service = :test 33 | 34 | config.action_mailer.perform_caching = false 35 | 36 | # Tell Action Mailer not to deliver emails to the real world. 37 | # The :test delivery method accumulates sent emails in the 38 | # ActionMailer::Base.deliveries array. 39 | config.action_mailer.delivery_method = :test 40 | 41 | # Print deprecation notices to the stderr. 42 | config.active_support.deprecation = :stderr 43 | 44 | # Raises error for missing translations 45 | # config.action_view.raise_on_missing_translations = true 46 | 47 | config.fog_params = { 48 | provider: 'Local', 49 | local_root: '/tmp' 50 | } 51 | 52 | config.input_bucket = 'autoclearance-rap-sheet-inputs' 53 | config.output_bucket = 'autoclearance-outputs' 54 | 55 | config.logged_exceptions = RapSheetParser::RapSheetParserException 56 | end 57 | -------------------------------------------------------------------------------- /spec/fixtures/skywalker_rap_sheet.txt: -------------------------------------------------------------------------------- 1 | RE: QHY.CA0190043.99000015.ORGANA- DATE:20060508 TIME:06:32:53 2 | RESTRICTED-DO NOT USE FOR EMPLOYMENT,LICENSING OR CERTIFICATION PURPOSES 3 | ATTN:ORGANA-061020343 4 | 5 | * * * * * * * * * * * * * * * * * * * * * 6 | ** PALM PRINT ON FILE AT DOJ FOR ADDITIONAL INFORMATION PLEASE E- 7 | MAIL PALM.PRINT@DOJ.CA.GOV 8 | III CALIFORNIA ONLY SOURCE RECORD 9 | 10 | **LIVE SCAN 11 | CII/A99000099 12 | DOB/19660119 SEX/M RAC/WHITE 13 | HGT/502 WGT/317 EYE/GRN HAIR/PNK POB/CA 14 | NAM/01 SKYWALKER,LUKE JAY 15 | 02 SKYWALKER,LUKE 16 | 17 | * * * * 18 | ARR/DET/CITE: NAM:02 DOB:19550505 19 | 19840725 CASO LOS ANGELES 20 | 21 | CNT:01 #1111111 22 | 11357 HS-POSSESS MARIJUANA 23 | 24 | CNT:02 25 | 496 PC-RECEIVE/ETC KNOWN STOLEN PROPERTY 26 | COM: WARRANT NBR A-400000 BOTH CNTS 27 | 28 | CNT:03 29 | 11358 HS-CULTIVATION MARIJUANA 30 | 31 | - - - - 32 | 33 | COURT: NAM:01 34 | 19840918 CASC SAN FRANCISCO 35 | 36 | CNT:01 #1234567 37 | 11357 HS-POSSESS MARIJUANA 38 | *DISPO:CONVICTED 39 | CONV STATUS:FELONY 40 | 41 | CNT:02 42 | 496 PC-RECEIVE/ETC KNOWN STOLEN PROPERTY 43 | *DISPO:CONVICTED 44 | CONV STATUS:MISDEMEANOR 45 | SEN: 3 YEARS PROBATION, 30 DAYS JAIL, 46 | FINE, RESTN 47 | 48 | CNT:03 49 | 11358 HS-CULTIVATION MARIJUANA 50 | *DISPO:CONVICTED 51 | CONV STATUS:FELONY 52 | 53 | CNT:04 54 | 11358 HS-CULTIVATION MARIJUANA 55 | *DISPO:CONVICTED 56 | CONV STATUS:MISDEMEANOR 57 | 58 | * * * * 59 | ARR/DET/CITE: NAM:02 DOB:19550505 60 | 20040319 CASO LOS ANGELES 61 | 62 | CNT:01 #2222222 63 | 11357 HS-POSSESS MARIJUANA 64 | COM: ADR-19940410 (300,NEWT AV, , ,OAKLAND,CA,94612) 65 | 66 | - - - - 67 | 68 | COURT: NAM:01 69 | 20040413 CASC LOS ANGELES 70 | 71 | CNT:01 #3456789 72 | 11358 HS-CULTIVATION MARIJUANA 73 | *DISPO:CONVICTED 74 | CONV STATUS:FELONY 75 | SEN: 2 YEARS PROBATION, 5 DAYS JAIL 76 | 77 | * * * * 78 | 79 | COURT: NAM:01 80 | 19900413 CASC SAN FRANCISCO 81 | CNT:01 #33857493 82 | 11359(D) HS-EMPLOY MINOR WITH INTENT TO SELL MARIJUANA 83 | *DISPO:CONVICTED 84 | CONV STATUS:FELONY 85 | SEN: 2 YEARS PROBATION, 30 DAYS JAIL 86 | 87 | CNT:02 88 | 11358 HS-CULTIVATION MARIJUANA 89 | *DISPO:CONVICTED 90 | CONV STATUS:FELONY 91 | 92 | CNT:03 93 | 32 PC-ACCESSORY AFTER THE FACT 94 | *DISPO:CONVICTED 95 | CONV STATUS:FELONY 96 | 97 | 98 | * * ** 99 | 100 | * * * END OF MESSAGE * * * 101 | -------------------------------------------------------------------------------- /spec/models/anon_event_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe AnonEvent do 4 | describe '#build_from_parser' do 5 | it 'creates court event and event properties from parser output' do 6 | event = build_court_event( 7 | date: Date.new(1991, 1, 5), 8 | courthouse: 'Some courthouse', 9 | counts: [ 10 | build_count(code: 'PC', section: '456'), 11 | build_count(code: 'PC', section: '789') 12 | ], 13 | name_code: nil 14 | ) 15 | rap_sheet = build_rap_sheet(events: [event]) 16 | 17 | anon_event = described_class.build_from_parser(event: event, rap_sheet: rap_sheet) 18 | expect(anon_event.agency).to eq 'Some courthouse' 19 | expect(anon_event.event_type).to eq 'court' 20 | expect(anon_event.date).to eq Date.new(1991, 1, 5) 21 | 22 | event_properties = anon_event.event_properties 23 | expect(event_properties.has_felonies).to eq false 24 | 25 | counts = anon_event.anon_counts 26 | expect(counts[0].code).to eq 'PC' 27 | expect(counts[0].section).to eq '456' 28 | expect(counts[0].count_number).to eq 1 29 | expect(counts[1].code).to eq 'PC' 30 | expect(counts[1].section).to eq '789' 31 | expect(counts[1].count_number).to eq 2 32 | end 33 | 34 | it 'only creates event properties if court event has convictions' do 35 | event = build_court_event( 36 | counts: [ 37 | build_count(disposition: build_disposition(type: 'dismissed')) 38 | ] 39 | ) 40 | rap_sheet = build_rap_sheet(events: [event]) 41 | anon_event = described_class.build_from_parser(event: event, rap_sheet: rap_sheet) 42 | expect(anon_event.event_properties).to be_nil 43 | end 44 | 45 | it 'creates arrest event from parser output' do 46 | count = build_count( 47 | code: 'PC', 48 | section: '456' 49 | ) 50 | 51 | event = build_arrest_event( 52 | date: Date.new(1991, 1, 5), 53 | agency: 'Some arrest agency', 54 | counts: [count] 55 | ) 56 | rap_sheet = build_rap_sheet(events: [event]) 57 | 58 | anon_event = described_class.build_from_parser(event: event, rap_sheet: rap_sheet) 59 | expect(anon_event.agency).to eq 'Some arrest agency' 60 | expect(anon_event.event_type).to eq 'arrest' 61 | expect(anon_event.date).to eq Date.new(1991, 1, 5) 62 | 63 | expect(anon_event.event_properties).to be_nil 64 | 65 | count = anon_event.anon_counts[0] 66 | expect(count.code).to eq 'PC' 67 | expect(count.section).to eq '456' 68 | expect(count.count_number).to eq 1 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /config/environments/development.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # In the development environment your application's code is reloaded on 5 | # every request. This slows down response time but is perfect for development 6 | # since you don't have to restart the web server when you make code changes. 7 | config.cache_classes = false 8 | 9 | # Do not eager load code on boot. 10 | config.eager_load = false 11 | 12 | # Show full error reports. 13 | config.consider_all_requests_local = true 14 | 15 | # Enable/disable caching. By default caching is disabled. 16 | # Run rails dev:cache to toggle caching. 17 | if Rails.root.join('tmp', 'caching-dev.txt').exist? 18 | config.action_controller.perform_caching = true 19 | 20 | config.cache_store = :memory_store 21 | config.public_file_server.headers = { 22 | 'Cache-Control' => "public, max-age=#{2.days.to_i}" 23 | } 24 | else 25 | config.action_controller.perform_caching = false 26 | 27 | config.cache_store = :null_store 28 | end 29 | 30 | # Store uploaded files on the local file system (see config/storage.yml for options) 31 | config.active_storage.service = :local 32 | 33 | # Don't care if the mailer can't send. 34 | config.action_mailer.raise_delivery_errors = false 35 | 36 | config.action_mailer.perform_caching = false 37 | 38 | # Print deprecation notices to the Rails logger. 39 | config.active_support.deprecation = :log 40 | 41 | # Raise an error on page load if there are pending migrations. 42 | config.active_record.migration_error = :page_load 43 | 44 | # Highlight code that triggered database queries in logs. 45 | config.active_record.verbose_query_logs = true 46 | 47 | # Debug mode disables concatenation and preprocessing of assets. 48 | # This option may cause significant delays in view rendering with a large 49 | # number of complex assets. 50 | config.assets.debug = true 51 | 52 | # Suppress logger output for asset requests. 53 | config.assets.quiet = true 54 | 55 | # Raises error for missing translations 56 | # config.action_view.raise_on_missing_translations = true 57 | 58 | # Use an evented file watcher to asynchronously detect changes in source code, 59 | # routes, locales, etc. This feature depends on the listen gem. 60 | config.file_watcher = ActiveSupport::EventedFileUpdateChecker 61 | 62 | config.fog_params = { 63 | provider: 'Local', 64 | local_root: './test_data' 65 | } 66 | 67 | config.input_bucket = 'autoclearance-rap-sheet-inputs' 68 | config.output_bucket = 'autoclearance-outputs' 69 | 70 | config.logged_exceptions = StandardError 71 | end 72 | -------------------------------------------------------------------------------- /spec/fixtures/chewbacca_rap_sheet.txt: -------------------------------------------------------------------------------- 1 | RE: QHY.CA0190043.99000015.ORGANA- DATE:20060508 TIME:06:32:53 2 | RESTRICTED-DO NOT USE FOR EMPLOYMENT,LICENSING OR CERTIFICATION PURPOSES 3 | ATTN:ORGANA-061020343 4 | 5 | * * * * * * * * * * * * * * * * * * * * * 6 | ** PALM PRINT ON FILE AT DOJ FOR ADDITIONAL INFORMATION PLEASE E- 7 | MAIL PALM.PRINT@DOJ.CA.GOV 8 | III CALIFORNIA ONLY SOURCE RECORD 9 | 10 | **LIVE SCAN 11 | CII/A99000099 12 | DOB/19660119 SEX/M RAC/WHITE 13 | HGT/502 WGT/317 EYE/GRN HAIR/PNK POB/CA 14 | NAM/01 BACCA, CHEW 15 | 02 BACCA, CHEW E. 16 | 03 WOOKIE, CHEWBACCA 17 | 18 | * * * * 19 | ARR/DET/CITE: NAM:02 DOB:19550505 20 | 19840725 CASO LOS ANGELES 21 | 22 | CNT:01 #1111111 23 | 11360 HS-TRANSPORTATION FOR SALE 24 | 25 | CNT:02 26 | 496 PC-RECEIVE/ETC KNOWN STOLEN PROPERTY 27 | COM: WARRANT NBR A-400000 BOTH CNTS 28 | 29 | CNT:03 30 | 11358 HS-CULTIVATION MARIJUANA 31 | 32 | - - - - 33 | 34 | COURT: NAM:02 35 | 19840918 CASC SAN FRANCISCO 36 | 37 | CNT:01 #1234567 38 | 11360 HS-TRANSPORTATION FOR SALE 39 | *DISPO:CONVICTED 40 | CONV STATUS:FELONY 41 | 42 | CNT:02 43 | 496 PC-RECEIVE/ETC KNOWN STOLEN PROPERTY 44 | *DISPO:CONVICTED 45 | CONV STATUS:FELONY 46 | SEN: 3 YEARS PROBATION, 30 DAYS JAIL, 47 | FINE, RESTN 48 | 49 | CNT:03 50 | 11358 HS-CULTIVATION MARIJUANA 51 | *DISPO:CONVICTED 52 | CONV STATUS:FELONY 53 | 54 | * * * * 55 | ARR/DET/CITE: NAM:02 DOB:19550505 56 | 20040319 CASO LOS ANGELES 57 | 58 | CNT:01 #2222222 59 | 11357 HS-POSSESS MARIJUANA 60 | 61 | - - - - 62 | 63 | COURT: NAM:01 64 | 20040413 CASC LOS ANGELES 65 | 66 | CNT:01 #3456789 67 | 11358 HS-CULTIVATION MARIJUANA 68 | *DISPO:CONVICTED 69 | CONV STATUS:FELONY 70 | 71 | CNT:02 72 | 220 PC-ASSAULT 73 | *DISPO:CONVICTED 74 | CONV STATUS:FELONY 75 | SEN: 2 YEARS PROBATION, 5 DAYS JAIL 76 | 77 | * * * * 78 | 79 | COURT: NAM:01 80 | 20100413 CASC SAN FRANCISCO 81 | 82 | CNT:01 #34567423423 83 | 11358 HS-CULTIVATION MARIJUANA 84 | *DISPO:CONVICTED 85 | CONV STATUS:FELONY 86 | SEN: 2 YEARS PROBATION, 5 DAYS JAIL 87 | 88 | * * * * 89 | 90 | COURT: NAM:03 91 | 20150505 CASC SAN FRANCISCO 92 | 93 | CNT:01 #abcde 94 | 11358 HS-CULTIVATION MARIJUANA 95 | *DISPO:CONVICTED 96 | CONV STATUS:FELONY 97 | SEN: 2 YEARS PROBATION, 5 DAYS JAIL 98 | 99 | 20010101 100 | DISPO:CONV SET ASIDE & DISM PER 1203.4 PC 101 | 102 | * * * * 103 | 104 | REGISTRATION: NAM:01 105 | 20171216 CASO SAN DIEGO 106 | 107 | CNT:01 108 | 290 PC-REGISTRATION OF SEX OFFENDER 109 | 110 | * * * END OF MESSAGE * * * 111 | -------------------------------------------------------------------------------- /spec/models/count_properties_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe CountProperties do 4 | describe '#build' do 5 | it 'populates eligibility fields for prop 64 code sections' do 6 | count = build_count(code: 'HS', section: '11357') 7 | 8 | event = build_court_event( 9 | date: Date.new(2001, 1, 5), 10 | counts: [count] 11 | ) 12 | 13 | count_properties = described_class.build( 14 | count: count, 15 | rap_sheet: build_rap_sheet(events: [event]), 16 | event: event 17 | ) 18 | 19 | expect(count_properties.has_prop_64_code).to eq true 20 | expect(count_properties.has_two_prop_64_priors).to eq false 21 | expect(count_properties.prop_64_plea_bargain).to eq 'no' 22 | expect(count_properties.eligibility_estimate.prop_64_eligible).to eq 'yes' 23 | end 24 | 25 | it 'populates eligibility fields for plea bargained counts' do 26 | arrest_count = build_count(code: 'HS', section: '11358') 27 | bargain_count = build_count(code: 'PC', section: '32') 28 | 29 | arrest_event = build_arrest_event(date: Date.new(2001, 11, 4), counts: [arrest_count]) 30 | bargain_event = build_court_event( 31 | date: Date.new(2001, 1, 5), 32 | counts: [bargain_count] 33 | ) 34 | 35 | count_properties = described_class.build( 36 | count: bargain_count, 37 | rap_sheet: build_rap_sheet(events: [arrest_event, bargain_event]), 38 | event: bargain_event 39 | ) 40 | 41 | expect(count_properties.has_prop_64_code).to eq false 42 | expect(count_properties.has_two_prop_64_priors).to eq false 43 | expect(count_properties.prop_64_plea_bargain).to eq 'yes' 44 | end 45 | 46 | it 'populates eligibility fields for prop 64 code sections with two priors' do 47 | count_1 = build_count(code: 'HS', section: '11358') 48 | count_2 = build_count(code: 'HS', section: '11358') 49 | count_3 = build_count(code: 'HS', section: '11358') 50 | 51 | event_1 = build_court_event( 52 | date: Date.new(2001, 1, 5), 53 | counts: [count_1] 54 | ) 55 | event_2 = build_court_event( 56 | date: Date.new(2002, 2, 5), 57 | counts: [count_2] 58 | ) 59 | event_3 = build_court_event( 60 | date: Date.new(2003, 3, 5), 61 | counts: [count_3] 62 | ) 63 | 64 | count_properties = described_class.build( 65 | count: count_3, 66 | rap_sheet: build_rap_sheet(events: [event_1, event_2, event_3]), 67 | event: event_3 68 | ) 69 | 70 | expect(count_properties.has_prop_64_code).to eq true 71 | expect(count_properties.has_two_prop_64_priors).to eq true 72 | expect(count_properties.prop_64_plea_bargain).to eq 'no' 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /spec/lib/fill_misdemeanor_petitions_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'csv' 3 | require 'pdf-forms' 4 | require_relative '../../app/lib/fill_misdemeanor_petitions' 5 | 6 | describe FillMisdemeanorPetitions do 7 | it 'populates motions per row' do 8 | t = Tempfile.new("test_csv") 9 | t << "some_headers\n" 10 | t << "123, 010101, SMITH/AGENT ,1/1/99,,,,,, #N/A , #N/A ,11357©, #N/A, , , , #N/A, , \n" 11 | t << "456, 293848, CANDY/AGENT ,1/1/99,,,,,, 11357(a) , #N/A , #N/A, #N/A , 11359(a), #N/A,11360(a),, \n" 12 | t.close 13 | 14 | output_dir = Dir.mktmpdir 15 | 16 | travel_to Date.new(1999, 10, 1) do 17 | described_class.new(path: t, output_dir: output_dir).fill 18 | end 19 | 20 | expect(Dir["#{output_dir}/**/*.pdf"].length).to eq 2 21 | 22 | fields = get_fields_from_pdf(File.new("#{output_dir}/with_sf_number/petition_SMITH_AGENT_123.pdf")) 23 | 24 | expect(fields).to eq( 25 | "defendant" => "SMITH/AGENT (DOB 1/1/99)", 26 | "sf_number" => "010101", 27 | "court_number" => "123", 28 | "11357a" => "No", 29 | "11357b" => "No", 30 | "11357c" => "Yes", 31 | "11360b" => "No", 32 | "others" => "No", 33 | "other_charges_description" => "", 34 | "date" => "1999-10-01", 35 | "completed_sentence" => "Yes", 36 | "court_dismisses" => "Yes", 37 | "court_granted" => "Yes", 38 | "court_relieved" => "Yes", 39 | "court_sealed" => "Yes", 40 | "record_sealed" => "Yes", 41 | "dismissal" => "Yes" 42 | ) 43 | 44 | fields = get_fields_from_pdf(File.new("#{output_dir}/with_sf_number/petition_CANDY_AGENT_456.pdf")) 45 | expect(fields).to eq( 46 | "defendant" => "CANDY/AGENT (DOB 1/1/99)", 47 | "sf_number" => "293848", 48 | "court_number" => "456", 49 | "11357a" => "Yes", 50 | "11357b" => "No", 51 | "11357c" => "No", 52 | "11360b" => "No", 53 | "others" => "Yes", 54 | "other_charges_description" => "11359(a), 11360(a)", 55 | "date" => "1999-10-01", 56 | "completed_sentence" => "Yes", 57 | "court_dismisses" => "Yes", 58 | "court_granted" => "Yes", 59 | "court_relieved" => "Yes", 60 | "court_sealed" => "Yes", 61 | "record_sealed" => "Yes", 62 | "dismissal" => "Yes" 63 | ) 64 | end 65 | 66 | it 'does not fill in sf number if it is zero' do 67 | t = Tempfile.new("test_csv") 68 | t << "some_headers\n" 69 | t << "123, 0 , SMITH/AGENT ,1/1/99,,,,,, #N/A , #N/A ,11357(c), #N/A, , , , #N/A, , \n" 70 | t.close 71 | 72 | output_dir = Dir.mktmpdir 73 | described_class.new(path: t, output_dir: output_dir).fill 74 | expect(Dir["#{output_dir}/**/*.pdf"].length).to eq 1 75 | 76 | fields = get_fields_from_pdf(File.new("#{output_dir}/no_sf_number/petition_SMITH_AGENT_123.pdf")) 77 | 78 | expect(fields["sf_number"]).to eq '' 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /spec/models/anon_count_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe AnonCount do 4 | describe '#build_from_parser' do 5 | let(:event) { build_court_event(counts: [count]) } 6 | let(:rap_sheet) { build_rap_sheet(events: [event]) } 7 | let(:subject) do 8 | described_class.build_from_parser( 9 | index: 5, 10 | count: count, 11 | event: event, 12 | rap_sheet: rap_sheet 13 | ) 14 | end 15 | 16 | context 'for a count with disposition convicted' do 17 | let(:count) do 18 | build_count( 19 | code_section_description: 'ARMED ROBBERY', 20 | code: 'HS', 21 | section: '11358', 22 | disposition: build_disposition(type: 'convicted', sentence: nil, text: 'DISPO:CONVICTED') 23 | ) 24 | end 25 | 26 | it 'creates an anon count with associated count properties' do 27 | expect(subject.count_number).to eq 5 28 | expect(subject.description).to eq 'ARMED ROBBERY' 29 | expect(subject.code).to eq 'HS' 30 | expect(subject.section).to eq '11358' 31 | expect(subject.anon_disposition.disposition_type).to eq 'convicted' 32 | expect(subject.anon_disposition.sentence).to be_nil 33 | expect(subject.anon_disposition.text).to eq 'DISPO:CONVICTED' 34 | expect(subject.count_properties).to_not be_nil 35 | expect(subject.count_properties.has_prop_64_code).to eq true 36 | end 37 | end 38 | 39 | context 'for a count with disposition other than convicted' do 40 | let(:count) do 41 | build_count( 42 | code_section_description: 'ARMED ROBBERY', 43 | code: 'PC', 44 | section: '1505', 45 | disposition: build_disposition(type: 'dismissed', sentence: nil, text: 'DISPO:DISMISSED') 46 | ) 47 | end 48 | it 'creates an AnonCount with no count_properties' do 49 | expect(subject.count_number).to eq 5 50 | expect(subject.description).to eq 'ARMED ROBBERY' 51 | expect(subject.code).to eq 'PC' 52 | expect(subject.section).to eq '1505' 53 | expect(subject.anon_disposition.disposition_type).to eq 'dismissed' 54 | expect(subject.anon_disposition.sentence).to be_nil 55 | expect(subject.anon_disposition.text).to eq 'DISPO:DISMISSED' 56 | expect(subject.count_properties).to be_nil 57 | end 58 | end 59 | 60 | context 'for a count with no disposition' do 61 | let(:count) do 62 | build_count( 63 | code_section_description: 'ARMED ROBBERY', 64 | code: 'PC', 65 | section: '1505', 66 | disposition: nil 67 | ) 68 | end 69 | it 'creates an AnonCount with no disposition and no count_properties' do 70 | expect(subject.anon_disposition).to be_nil 71 | expect(subject.count_properties).to be_nil 72 | end 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /app/lib/fill_prop64_motion.rb: -------------------------------------------------------------------------------- 1 | class FillProp64Motion 2 | def initialize(ada, counts, event, personal_info) 3 | @ada = ada 4 | @counts = counts 5 | @event = event 6 | @personal_info = personal_info 7 | end 8 | 9 | def filled_motion 10 | tempfile = Tempfile.new 11 | 12 | pdftk = PdfForms.new(Cliver.detect('pdftk')) 13 | 14 | pdftk.fill_form 'app/assets/motions/prop64.pdf', tempfile.path, 15 | 'Assistant District Attorney' => @ada[:name], 16 | 'CA SB' => @ada[:state_bar_number], 17 | 'Defendant' => name, 18 | 'SCN' => event.case_number, 19 | 'FOR RESENTENCING OR DISMISSAL HS 113618b' => on_or_off(event.remedy == 'resentencing'), 20 | 'FOR REDESIGNATION OR DISMISSALSEALING HS 113618f' => on_or_off(event.remedy == 'redesignation'), 21 | '11357 Possession of Marijuana' => on_or_off(number_of_counts_for_code('HS 11357').positive?), 22 | '11358 Cultivation of Marijuana' => on_or_off(number_of_counts_for_code('HS 11358').positive?), 23 | '11359 Possession of Marijuana for Sale' => on_or_off(number_of_counts_for_code('HS 11359').positive?), 24 | '11360 Transportation Distribution or' => on_or_off(number_of_counts_for_code('HS 11360').positive?), 25 | '11357 Count' => count_multiplier('HS 11357'), 26 | '11358 Count' => count_multiplier('HS 11358'), 27 | '11359 Count' => count_multiplier('HS 11359'), 28 | '11360 Count' => count_multiplier('HS 11360'), 29 | 'Other Health and Safety Code Section' => on_or_off(other_code_sections.any?), 30 | 'Text2' => other_code_sections.join(', '), 31 | 'The District Attorney finds that defendant is eligible for relief and now requests the court to recall and resentence' => 'On' 32 | 33 | tempfile 34 | end 35 | 36 | private 37 | 38 | def other_code_sections 39 | code_sections = counts.reject do |count| 40 | count.subsection_of?(['HS 11357', 'HS 11358', 'HS 11359', 'HS 11360']) 41 | end.map(&:code_section) 42 | 43 | code_sections.uniq.map do |code| 44 | count = code_sections.count(code) 45 | 46 | count == 1 ? code : "#{code} x#{count}" 47 | end 48 | end 49 | 50 | def name 51 | return unless personal_info 52 | 53 | first_name_key = personal_info.names.keys.min_by(&:to_i) 54 | if first_name_key != event.name_code 55 | "#{personal_info.names[first_name_key]} AKA #{personal_info.names[event.name_code]}" 56 | else 57 | personal_info.names[first_name_key] 58 | end 59 | end 60 | 61 | def number_of_counts_for_code(code) 62 | counts.select { |count| count.subsection_of?([code]) }.length 63 | end 64 | 65 | def count_multiplier(code) 66 | counts = number_of_counts_for_code(code) 67 | counts > 1 ? "x#{counts}" : '' 68 | end 69 | 70 | def on_or_off(bool) 71 | bool ? 'On' : 'Off' 72 | end 73 | 74 | attr_reader :counts, :event, :personal_info 75 | end 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Autoclearance 2 | 3 | ## Development 4 | Clone this repository and navigate to it. 5 | 6 | Homebrew is used to install local dependencies. [Install it here.](https://brew.sh/) 7 | Then. run `brew bundle` from the `autoclearance` directory. 8 | 9 | `ruby-install` and `chruby` are used to manage Ruby versions. 10 | [Follow instructions here for autoswitching with chruby](https://github.com/postmodern/chruby#auto-switching) 11 | 12 | Run `gem install bundler` and then `bundle` to install Ruby dependencies. 13 | 14 | Install PDFtk for PDF filling functionality 15 | 16 | [Install PDFTK from here](https://www.pdflabs.com/tools/pdftk-the-pdf-toolkit/pdftk_server-2.02-mac_osx-10.11-setup.pkg) 17 | 18 | ### To set up the Beanstalk CLI 19 | To add a Elastic Beanstalk profile for the environment, add the following to `~/.aws/config` 20 | ``` 21 | [profile autoclearance] 22 | aws_access_key_id= 23 | aws_secret_access_key= 24 | region=us-gov-west-1 25 | ``` 26 | 27 | Initialize Elastic Beanstalk 28 | `eb init --profile autoclearance --region us-gov-west-1` 29 | 30 | ## Deploying 31 | Create two files: `backend-config` and `varfile` to supply Amazon credentials. Examples are located at `backend-config.example` and `varfile.example` 32 | To deploy using Terraform, cd to the terraform directory and run `terraform init -backend-config config_file` 33 | 34 | Create a new keypair through the AWS console for SSH access to the bastion. Safely store the keyfile. 35 | Generate a publickey for your `varfile` by running `ssh-keygen -y -f /path/to/private_key.pem`. 36 | 37 | When the bastion is initially created, you will need to use these credentials to run the bastion setup script 38 | with: `./bastion_setup.sh `, 39 | which creates individual user accounts and sets up logging to CloudWatch from the bastion. 40 | 41 | To apply Terraform settings, run: `terraform apply -var-file varfile` 42 | 43 | To push a new revision to Beanstalk: 44 | ``` 45 | eb init -r us-gov-west-1 --profile autoclearance 46 | ``` 47 | 48 | Set your Elastic Beanstalk environment by running: 49 | `eb use --profile autoclearance` 50 | 51 | Deploy code to environment by running from the repository root: 52 | `eb deploy` 53 | 54 | 55 | ## To SSH to the EC2 instance via the bastion host 56 | Add your credentials to your local SSH agent by running: `ssh-add ` 57 | SSH to the instance by proxying through the Bastion by running: 58 | `ssh -o ProxyCommand='ssh -W %h:%p @' @` 59 | 60 | ## Notes 61 | In order for the config rule "s3-bucket-ssl-requests-only" to be in compliance, you will need to deny HTTP access for the bucket created by Elastic Beanstalk to store application artifacts. To do this, add this policy excerpt to the created bucket. 62 | ``` 63 | { 64 | "Sid": "DenyUnSecureCommunications", 65 | "Effect": "Deny", 66 | "Principal": { 67 | "AWS": "*" 68 | }, 69 | "Action": "s3:*", 70 | "Resource": "arn:aws-us-gov:s3:::{bucket_name}/*", 71 | "Condition": { 72 | "Bool": { 73 | "aws:SecureTransport": false 74 | } 75 | } 76 | } 77 | ``` 78 | -------------------------------------------------------------------------------- /app/lib/fill_misdemeanor_petitions.rb: -------------------------------------------------------------------------------- 1 | require 'csv' 2 | 3 | class FillMisdemeanorPetitions 4 | def initialize(path:, output_dir:) 5 | @path = path 6 | @output_dir = output_dir 7 | end 8 | 9 | def fill 10 | # Row 11 | # 0 Courtno 12 | # 1 SFNO 13 | # 2 True Name 14 | # 3 DOB 15 | # 4 Conviction Date 16 | # 5 Sentence Date 17 | # 6 Case Type 18 | # 7 Dispo Code 19 | # 8 Disposition 20 | # 9 11357(a) 21 | # 10 11357(b) 22 | # 11 11357(c) 23 | # 12 11359 24 | # 13 11359(a) 25 | # 14 11359(b) 26 | # 15 11360(a) 27 | # 16 11360(b) 28 | # 17 Other 29 | # 18 Petition Created 30 | # 19 Date Petition Drafted 31 | # 20 Date Petition Submitted 32 | # 21 Petition Granted 33 | # 22 Date Sent to PDR 34 | # 23 Notes 35 | # 24 Copy Requested 36 | # 25 Email Address 37 | # 26 Mailing Address 38 | # 27 Willing to Speak to Media 39 | # 28 Phone Number 40 | 41 | Dir.mkdir("#{output_dir}/with_sf_number") unless File.exists?("#{output_dir}/with_sf_number") 42 | Dir.mkdir("#{output_dir}/no_sf_number") unless File.exists?("#{output_dir}/no_sf_number") 43 | 44 | CSV.foreach(path, headers: true, encoding: 'ISO-8859-1') do |row| 45 | court_number = row[0].strip 46 | name = row[2].strip 47 | date_of_birth = row[3].strip 48 | count_11357a = row[9].strip 49 | count_11357b = row[10].strip 50 | count_11357c = row[11].strip 51 | 52 | other_charges = others(row) 53 | 54 | fields = { 55 | "defendant": "#{name} (DOB #{date_of_birth})", 56 | "sf_number": sf_number(row), 57 | "court_number": court_number, 58 | "11357a": checkbox_val(count_11357a), 59 | "11357b": checkbox_val(count_11357b), 60 | "11357c": checkbox_val(count_11357c), 61 | "11360b": "No", # ignored right now since column always empty 62 | "others": yes(other_charges.present?), 63 | "other_charges_description": other_charges.join(', '), 64 | "date": Time.zone.today 65 | } 66 | 67 | fill_petition("#{name.tr('/', '_')}_#{court_number}", sf_number(row), fields) 68 | end 69 | end 70 | 71 | private 72 | 73 | attr_reader :path, :output_dir 74 | 75 | def sf_number(row) 76 | value = row[1].strip 77 | 78 | value unless value == "0" 79 | end 80 | 81 | def others(row) 82 | count_11359 = row[12]&.strip 83 | count_11359a = row[13]&.strip 84 | count_11359b = row[14]&.strip 85 | count_11360a = row[15]&.strip 86 | 87 | [count_11359, count_11359a, count_11359b, count_11360a].compact.reject do |data| 88 | data.empty? || data.include?('#N/A') 89 | end 90 | end 91 | 92 | def yes(bool) 93 | bool ? "Yes" : "No" 94 | end 95 | 96 | def checkbox_val(data) 97 | data.include?('#N/A') ? "No" : "Yes" 98 | end 99 | 100 | def fill_petition(name, sf_number, fields) 101 | sub_directory = sf_number ? "with_sf_number" : "no_sf_number" 102 | file_name = "#{output_dir}/#{sub_directory}/petition_#{name}.pdf" 103 | pdftk = PdfForms.new(Cliver.detect('pdftk')) 104 | pdftk.fill_form 'app/assets/petitions/prop64_misdemeanors.pdf', file_name, fields 105 | end 106 | end 107 | -------------------------------------------------------------------------------- /spec/rails_helper.rb: -------------------------------------------------------------------------------- 1 | # This file is copied to spec/ when you run 'rails generate rspec:install' 2 | require 'spec_helper' 3 | ENV['RAILS_ENV'] ||= 'test' 4 | require File.expand_path('../../config/environment', __FILE__) 5 | # Prevent database truncation if the environment is production 6 | abort("The Rails environment is running in production mode!") if Rails.env.production? 7 | require 'rspec/rails' 8 | # Add additional requires below this line. Rails is not loaded until this point! 9 | 10 | # Requires supporting ruby files with custom matchers and macros, etc, in 11 | # spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are 12 | # run as spec files by default. This means that files in spec/support that end 13 | # in _spec.rb will both be required and run as specs, causing the specs to be 14 | # run twice. It is recommended that you do not name files matching this glob to 15 | # end with _spec.rb. You can configure this pattern with the --pattern 16 | # option on the command line or in ~/.rspec, .rspec or `.rspec-local`. 17 | # 18 | # The following line is provided for convenience purposes. It has the downside 19 | # of increasing the boot-up time by auto-requiring all files in the support 20 | # directory. Alternatively, in the individual `*_spec.rb` files, manually 21 | # require only the support files necessary. 22 | # 23 | # Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f } 24 | 25 | # Checks for pending migrations and applies them before tests are run. 26 | # If you are not using ActiveRecord, you can remove this line. 27 | ActiveRecord::Migration.maintain_test_schema! 28 | 29 | Capybara.javascript_driver = :selenium_chrome_headless 30 | 31 | RSpec.configure do |config| 32 | # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures 33 | config.fixture_path = "#{::Rails.root}/spec/fixtures" 34 | 35 | # If you're not using ActiveRecord, or you'd prefer not to run each of your 36 | # examples within a transaction, remove the following line or assign false 37 | # instead of true. 38 | config.use_transactional_fixtures = true 39 | 40 | # RSpec Rails can automatically mix in different behaviours to your tests 41 | # based on their file location, for example enabling you to call `get` and 42 | # `post` in specs under `spec/controllers`. 43 | # 44 | # You can disable this behaviour by removing the line below, and instead 45 | # explicitly tag your specs with their type, e.g.: 46 | # 47 | # RSpec.describe UsersController, :type => :controller do 48 | # # ... 49 | # end 50 | # 51 | # The different available types are documented in the features, such as in 52 | # https://relishapp.com/rspec/rspec-rails/docs 53 | config.infer_spec_type_from_file_location! 54 | 55 | # Filter lines from Rails gems in backtraces. 56 | config.filter_rails_from_backtrace! 57 | # arbitrary gems may also be filtered via: 58 | # config.filter_gems_from_backtrace("gem name") 59 | end 60 | 61 | def build_rap_sheet_with_eligibility(rap_sheet:, county: build_county, logger: Logger.new('/dev/null')) 62 | RapSheetWithEligibility.new( 63 | rap_sheet: rap_sheet, 64 | county: county, 65 | logger: logger 66 | ) 67 | end 68 | 69 | def build_county(courthouses: ['CASC GOTHAM CITY', 'CAMC METROPOLIS'], name: 'Gotham City', misdemeanors: true) 70 | { 71 | courthouses: courthouses, 72 | name: name, 73 | misdemeanors: misdemeanors 74 | } 75 | end 76 | -------------------------------------------------------------------------------- /terraform/policies/mfa_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "AllowAllUsersToListAccounts", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "iam:ListAccountAliases", 9 | "iam:ListUsers", 10 | "iam:ListVirtualMFADevices", 11 | "iam:GetAccountPasswordPolicy", 12 | "iam:GetAccountSummary" 13 | ], 14 | "Resource": "*" 15 | }, 16 | { 17 | "Sid": "AllowIndividualUserToSeeAndManageOnlyTheirOwnAccountInformation", 18 | "Effect": "Allow", 19 | "Action": [ 20 | "iam:ChangePassword", 21 | "iam:CreateAccessKey", 22 | "iam:CreateLoginProfile", 23 | "iam:DeleteAccessKey", 24 | "iam:DeleteLoginProfile", 25 | "iam:GetLoginProfile", 26 | "iam:ListAccessKeys", 27 | "iam:UpdateAccessKey", 28 | "iam:UpdateLoginProfile", 29 | "iam:ListSigningCertificates", 30 | "iam:DeleteSigningCertificate", 31 | "iam:UpdateSigningCertificate", 32 | "iam:UploadSigningCertificate", 33 | "iam:ListSSHPublicKeys", 34 | "iam:GetSSHPublicKey", 35 | "iam:DeleteSSHPublicKey", 36 | "iam:UpdateSSHPublicKey", 37 | "iam:UploadSSHPublicKey" 38 | ], 39 | "Resource": "arn:aws-us-gov:iam::*:user/${aws:username}" 40 | }, 41 | { 42 | "Sid": "AllowIndividualUserToListOnlyTheirOwnMFA", 43 | "Effect": "Allow", 44 | "Action": [ 45 | "iam:ListMFADevices" 46 | ], 47 | "Resource": [ 48 | "arn:aws-us-gov:iam::*:mfa/*", 49 | "arn:aws-us-gov:iam::*:user/${aws:username}" 50 | ] 51 | }, 52 | { 53 | "Sid": "AllowIndividualUserToManageTheirOwnMFA", 54 | "Effect": "Allow", 55 | "Action": [ 56 | "iam:CreateVirtualMFADevice", 57 | "iam:DeleteVirtualMFADevice", 58 | "iam:EnableMFADevice", 59 | "iam:ResyncMFADevice" 60 | ], 61 | "Resource": [ 62 | "arn:aws-us-gov:iam::*:mfa/${aws:username}", 63 | "arn:aws-us-gov:iam::*:user/${aws:username}" 64 | ] 65 | }, 66 | { 67 | "Sid": "AllowIndividualUserToDeactivateOnlyTheirOwnMFAOnlyWhenUsingMFA", 68 | "Effect": "Allow", 69 | "Action": [ 70 | "iam:DeactivateMFADevice" 71 | ], 72 | "Resource": [ 73 | "arn:aws-us-gov:iam::*:mfa/${aws:username}", 74 | "arn:aws-us-gov:iam::*:user/${aws:username}" 75 | ], 76 | "Condition": { 77 | "Bool": { 78 | "aws:MultiFactorAuthPresent": "true" 79 | } 80 | } 81 | }, 82 | { 83 | "Sid": "BlockMostAccessUnlessSignedInWithMFA", 84 | "Effect": "Deny", 85 | "NotAction": [ 86 | "iam:CreateVirtualMFADevice", 87 | "iam:DeleteVirtualMFADevice", 88 | "iam:ListVirtualMFADevices", 89 | "iam:EnableMFADevice", 90 | "iam:ResyncMFADevice", 91 | "iam:ListAccountAliases", 92 | "iam:ListUsers", 93 | "iam:ListSSHPublicKeys", 94 | "iam:ListAccessKeys", 95 | "iam:ListServiceSpecificCredentials", 96 | "iam:ListMFADevices", 97 | "iam:GetAccountSummary", 98 | "sts:GetSessionToken" 99 | ], 100 | "Resource": "*", 101 | "Condition": { 102 | "BoolIfExists": { 103 | "aws:MultiFactorAuthPresent": "false" 104 | } 105 | } 106 | } 107 | ] 108 | } 109 | -------------------------------------------------------------------------------- /spec/helpers/pdf_reader_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'pdf-reader' 3 | require_relative '../../app/helpers/pdf_reader' 4 | 5 | describe PDFReader do 6 | describe '#text' do 7 | it 'extracts the RAP sheet text from the PDF' do 8 | expected_text = File.read('spec/fixtures/skywalker_rap_sheet.txt') 9 | actual_text = described_class.new(IO.binread('spec/fixtures/skywalker_rap_sheet.pdf')).text 10 | 11 | expect(strip_whitespace(actual_text)).to eq(strip_whitespace(expected_text)) 12 | end 13 | end 14 | 15 | describe '.strip_page_header_and_footer' do 16 | let :expected_text do 17 | expected_text = <<~EXPECTED_TEXT 18 | document begin here 19 | another line 20 | EXPECTED_TEXT 21 | end 22 | 23 | it 'removes blank lines and whitespace at the beginning of page' do 24 | input_text = <<~INPUT_TEXT 25 | 26 | 27 | \t 28 | 29 | document begin here 30 | another line 31 | INPUT_TEXT 32 | 33 | expect(described_class.strip_page_header_and_footer(input_text)).to eq expected_text 34 | end 35 | 36 | it 'removes blank lines and whitespace at the end of page' do 37 | input_text = <<~INPUT_TEXT 38 | document begin here 39 | another line 40 | 41 | 42 | \t 43 | 44 | INPUT_TEXT 45 | 46 | expect(described_class.strip_page_header_and_footer(input_text)).to eq expected_text 47 | end 48 | 49 | it 'removes lines beginning with "Page" from beginning of page' do 50 | input_text = <<~INPUT_TEXT 51 | 52 | Page 1 of 5 53 | 54 | document begin here 55 | another line 56 | INPUT_TEXT 57 | 58 | expect(described_class.strip_page_header_and_footer(input_text)).to eq expected_text 59 | end 60 | 61 | it 'removes lines beginning with "Route" from beginning of page' do 62 | input_text = <<~INPUT_TEXT 63 | 64 | 65 | Route Print 66 | 67 | document begin here 68 | another line 69 | INPUT_TEXT 70 | 71 | expect(described_class.strip_page_header_and_footer(input_text)).to eq expected_text 72 | end 73 | 74 | it 'removes lines beginning with "http://" from end of page' do 75 | input_text = <<~INPUT_TEXT 76 | document begin here 77 | another line 78 | http://example.com 79 | 80 | INPUT_TEXT 81 | 82 | expect(described_class.strip_page_header_and_footer(input_text)).to eq expected_text 83 | end 84 | 85 | it 'removes lines beginning with "https://" from end of page' do 86 | input_text = <<~INPUT_TEXT 87 | document begin here 88 | another line 89 | 90 | 91 | https://example.com 5/16/1997 92 | INPUT_TEXT 93 | 94 | expect(described_class.strip_page_header_and_footer(input_text)).to eq expected_text 95 | end 96 | 97 | it 'removes lines beginning with "file://" or "file://" from end of page' do 98 | input_text = <<~INPUT_TEXT 99 | document begin here 100 | another line 101 | file:///example.com 5/16/1997 102 | 103 | file:///path/to/file 104 | 105 | INPUT_TEXT 106 | 107 | expect(described_class.strip_page_header_and_footer(input_text)).to eq expected_text 108 | end 109 | 110 | it 'returns a newline when passed a blank input' do 111 | expect(described_class.strip_page_header_and_footer("\n\n\n\n")).to eq "\n" 112 | end 113 | end 114 | 115 | def strip_whitespace(str) 116 | str.gsub(/[\s\n]/, '') 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /spec/lib/fill_prop64_motion_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | RSpec.describe FillProp64Motion do 4 | let(:personal_info) do 5 | build_personal_info(names: { 6 | '01' => 'SAYS, SIMON', 7 | '03' => 'DAVE' 8 | }) 9 | end 10 | 11 | let(:counts) do 12 | [ 13 | build_count(code: 'HS', section: '11357(a)'), 14 | build_count(code: 'HS', section: '11357(a)'), 15 | build_count(code: 'HS', section: '11359') 16 | ] 17 | end 18 | 19 | let(:event) do 20 | build_court_event( 21 | name_code: name_code, 22 | case_number: 'MYCASE01', 23 | date: Date.new(1994, 1, 2) 24 | ) 25 | end 26 | 27 | let(:ada) { { name: 'Some Attorney', state_bar_number: '8675309' } } 28 | 29 | subject do 30 | eligibility = build_rap_sheet_with_eligibility(rap_sheet: build_rap_sheet) 31 | event_with_eligibility = EventWithEligibility.new(event: event, eligibility: eligibility) 32 | 33 | file = described_class.new(ada, counts, event_with_eligibility, personal_info).filled_motion 34 | 35 | get_fields_from_pdf(file) 36 | end 37 | 38 | context 'rap sheet with name code different than the first' do 39 | let(:name_code) { '03' } 40 | 41 | it 'fills out fields' do 42 | fields = subject 43 | 44 | expect(fields['Assistant District Attorney']).to eq 'Some Attorney' 45 | expect(fields['CA SB']).to eq '8675309' 46 | expect(fields['Defendant']).to eq 'SAYS, SIMON AKA DAVE' 47 | expect(fields['SCN']).to eq 'MYCASE01' 48 | expect(fields['FOR RESENTENCING OR DISMISSAL HS 113618b']).to eq 'Off' 49 | expect(fields['FOR REDESIGNATION OR DISMISSALSEALING HS 113618f']).to eq 'On' 50 | expect(fields['11357 Possession of Marijuana']).to eq('On') 51 | expect(fields['11357 Count']).to eq('x2') 52 | expect(fields['11358 Cultivation of Marijuana']).to eq('Off') 53 | expect(fields['11358 Count']).to eq('') 54 | expect(fields['11359 Possession of Marijuana for Sale']).to eq('On') 55 | expect(fields['11359 Count']).to eq('') 56 | expect(fields['11360 Transportation Distribution or']).to eq('Off') 57 | expect(fields['11360 Count']).to eq('') 58 | expect(fields['Other Health and Safety Code Section']).to eq 'Off' 59 | expect(fields['Text2']).to eq '' 60 | expect(fields['The District Attorney finds that defendant is eligible for relief and now requests the court to recall and resentence']).to eq 'On' 61 | end 62 | end 63 | 64 | context 'rap sheet with code sections not found on the checkboxes' do 65 | let(:name_code) { '03' } 66 | let(:counts) do 67 | [ 68 | build_count(code: 'PC', section: '32'), 69 | build_count(code: 'PC', section: '32'), 70 | build_count(code: 'PC', section: '123') 71 | ] 72 | end 73 | 74 | it 'fills out the Other field with code_section' do 75 | fields = subject 76 | 77 | expect(fields['Other Health and Safety Code Section']).to eq 'On' 78 | expect(fields['Text2']).to eq 'PC 32 x2, PC 123' 79 | end 80 | end 81 | 82 | context 'name code is the same as the first' do 83 | let(:name_code) { '01' } 84 | 85 | it 'only includes the primary name if the event name is the same as the primary name' do 86 | fields = subject 87 | 88 | expect(fields['Defendant']).to eq 'SAYS, SIMON' 89 | end 90 | end 91 | 92 | context 'a redacted rap sheet' do 93 | let(:personal_info) { nil } 94 | let(:name_code) { '01' } 95 | it 'fills out fields' do 96 | fields = subject 97 | 98 | expect(fields['Defendant']).to eq '' 99 | end 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /spec/domain/summary_csv_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | require 'csv' 3 | 4 | describe SummaryCSV do 5 | let(:eligibility1) do 6 | court_count1 = build_count( 7 | code: 'HS', 8 | section: '11360', 9 | disposition: build_disposition( 10 | type: 'convicted', 11 | sentence: RapSheetParser::ConvictionSentence.new(jail: 2.days), 12 | severity: 'M' 13 | ) 14 | ) 15 | eligible_event1 = build_court_event( 16 | date: Date.new(1994, 1, 2), 17 | case_number: 'DEF', 18 | courthouse: 'CAMC METROPOLIS', 19 | counts: [court_count1], 20 | name_code: '001' 21 | ) 22 | rap_sheet1 = build_rap_sheet( 23 | events: [eligible_event1], 24 | personal_info: personal_info 25 | ) 26 | 27 | build_rap_sheet_with_eligibility(rap_sheet: rap_sheet1) 28 | end 29 | 30 | let(:eligibility2) do 31 | court_count2 = build_count( 32 | code: 'HS', 33 | section: '11357', 34 | disposition: build_disposition( 35 | type: 'convicted', 36 | sentence: RapSheetParser::ConvictionSentence.new(jail: 5.days), 37 | severity: 'M' 38 | ) 39 | ) 40 | eligible_event2 = build_court_event( 41 | date: Date.new(1994, 1, 2), 42 | case_number: 'ABC', 43 | courthouse: 'CASC GOTHAM CITY', 44 | counts: [court_count2], 45 | name_code: '002' 46 | ) 47 | rap_sheet2 = build_rap_sheet( 48 | events: [eligible_event2], 49 | personal_info: build_personal_info(names: { '002' => 'n a m e' }) 50 | ) 51 | 52 | build_rap_sheet_with_eligibility(rap_sheet: rap_sheet2) 53 | end 54 | 55 | let(:personal_info) { build_personal_info(names: { '001' => 'defendant' }) } 56 | 57 | it 'generates text for a CSV containing eligibility results' do 58 | summary_csv = described_class.new 59 | summary_csv.append('filename1', eligibility1) 60 | summary_csv.append('filename2', eligibility2) 61 | 62 | subject = summary_csv.text 63 | 64 | expect(subject).to include 'filename1,defendant,1994-01-02,DEF,CAMC METROPOLIS,HS 11360,M,2d jail,"",false,false,redesignation,yes,false' 65 | expect(subject).to include 'filename2,n a m e,1994-01-02,ABC,CASC GOTHAM CITY,HS 11357,M,5d jail,"",false,false,redesignation,yes,false' 66 | end 67 | 68 | context 'missing personal info' do 69 | let(:personal_info) {} 70 | 71 | it 'skips names if missing personal info' do 72 | summary_csv = described_class.new 73 | summary_csv.append('filename1', eligibility1) 74 | 75 | subject = summary_csv.text 76 | 77 | expect(subject).to include 'filename1,,1994-01-02,DEF,CAMC METROPOLIS,HS 11360,M,2d jail,"",false,false,redesignation,yes,false' 78 | end 79 | end 80 | 81 | it 'adds deceased events to single csv' do 82 | deceased_event = build_other_event(event_type: 'deceased') 83 | court_count = build_count( 84 | code: 'HS', 85 | section: '11360', 86 | disposition: build_disposition( 87 | type: 'convicted', 88 | sentence: RapSheetParser::ConvictionSentence.new(jail: 2.days), 89 | severity: 'M' 90 | ) 91 | ) 92 | eligible_event = build_court_event( 93 | date: Date.new(1994, 1, 2), 94 | case_number: 'DEF', 95 | counts: [court_count], 96 | name_code: '001' 97 | ) 98 | rap_sheet = build_rap_sheet(events: [eligible_event, deceased_event], personal_info: personal_info) 99 | eligibility = build_rap_sheet_with_eligibility(rap_sheet: rap_sheet) 100 | 101 | summary_csv = described_class.new 102 | summary_csv.append('filename1', eligibility) 103 | 104 | subject = summary_csv.text 105 | 106 | expect(subject).to include 'filename1,defendant,1994-01-02,DEF,CASC GOTHAM CITY,HS 11360,M,2d jail,"",false,false,redesignation,yes,true' 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /app/helpers/rap_sheet_deidentifier.rb: -------------------------------------------------------------------------------- 1 | class RapSheetDeidentifier 2 | def initialize 3 | @connection = Fog::Storage.new(Rails.configuration.fog_params) 4 | @input_directory = @connection.directories.new(key: Rails.configuration.input_bucket) 5 | @output_directory = @connection.directories.new(key: Rails.configuration.output_bucket) 6 | end 7 | 8 | def run 9 | @input_directory.files.to_a.sort_by(&:key).each do |input_file| 10 | text = PDFReader.new(input_file.body).text 11 | next unless first_cycle_delimiter_index(text) 12 | 13 | id = Digest::MD5.hexdigest(text) 14 | deidentified_text = deidentify(text) 15 | @output_directory.files.create( 16 | key: "redacted_rap_sheet_#{id}.txt", 17 | body: deidentified_text 18 | ) 19 | 20 | @output_directory.files.create( 21 | key: "redacted_rap_sheet_#{id}.pdf", 22 | body: pdf_contents(deidentified_text) 23 | ) 24 | end 25 | end 26 | 27 | private 28 | 29 | def pdf_contents(text) 30 | tmp_text = Tempfile.new 31 | tmp_text.write(text) 32 | tmp_text.close 33 | 34 | tmp_pdf = Tempfile.new 35 | `enscript -B #{tmp_text.path} -q -o - | ps2pdf -q - #{tmp_pdf.path}` 36 | 37 | tmp_pdf.read 38 | end 39 | 40 | def deidentify(text) 41 | t = remove_personal_info_section(text) 42 | t = randomize_dob(t) 43 | t = strip_addresses(t) 44 | randomize_case_numbers(t) 45 | end 46 | 47 | def randomize_dob(text) 48 | dob_regex = %r{DOB[:\/](\d{8})} 49 | matches = text.scan(dob_regex) 50 | matches.uniq.each do |m| 51 | date_string = m[0] 52 | text = text.gsub(date_string, random_date) 53 | end 54 | text 55 | end 56 | 57 | def strip_addresses(text) 58 | address_regex = /COM:\s*ADR-[\d]{6,8}\s*\([^)]*\)/m 59 | text.gsub(address_regex, 'COM: [ADDRESS REDACTED]') 60 | end 61 | 62 | def random_date 63 | Time.zone.at(rand * Time.zone.now.to_i).strftime('%Y%m%d') 64 | end 65 | 66 | def randomize_case_numbers(text) 67 | case_number_regex = /#(.+)$/ 68 | matches = text.scan(case_number_regex) 69 | matches.uniq.each do |m| 70 | case_number_string = m[0] 71 | chars = case_number_string.chars 72 | chars.each_with_index do |c, i| 73 | next unless /\d/.match?(c) 74 | 75 | digit = c.to_i 76 | digit = (digit + rand(10)) % 10 77 | chars[i] = digit 78 | end 79 | new_case_number = chars.join 80 | text = text.gsub(case_number_string, new_case_number) 81 | end 82 | text 83 | end 84 | 85 | def remove_personal_info_section(text) 86 | text.slice!(0..(first_cycle_delimiter_index(text) - 1)) 87 | text.prepend(fake_personal_info) 88 | end 89 | 90 | def first_cycle_delimiter_index(text) 91 | text.index(/^(\s| )*(\*(\s| )?){4}$/) 92 | end 93 | end 94 | 95 | def fake_personal_info 96 | <<~TEXT 97 | FROM: CLET ISN: 00003 DATE: 02/05/01 TIME: 09:36:27 RESP MSG 98 | TO: CT032474 OSN: 00003 DATE: 05/15/12 TIME: 09:36:30 99 | USERID: OBI_WAN 100 | 101 | IH 102 | 103 | RE: QHY.CA0313113A.10243565.PM,PM,P DATE:20010205 TIME:09:36:26 104 | RESTRICTED-DO NOT USE FOR EMPLOYMENT,LICENSING OR CERTIFICATION PURPOSES 105 | ATTN:PM,PM,P,O,1954249, 106 | 107 | ************************************************************************ 108 | FOR CALIFORNIA AGENCIES ONLY - HAS PREVIOUS QUALIFYING OFFENSE. COLLECT DNA IF INCARCERATED, CONFINED, OR ON PROBATION OR PAROLE FOLLOWING ANY MISDEMEANOR OR FELONY CONVICTION. REQUEST KITS AND INFO AT (510) 310- 3000 OR PC296.PC296@DOJ.CA.GOV. 109 | ************************************************************************ 110 | 111 | ** III MULTIPLE SOURCE RECORD 112 | 113 | CII/A01234557 114 | DOB/19681122 SEX/M RAC/WHITE 115 | HGT/511 WGT/265 EYE/BLU HAI/BRO POB/CA 116 | NAM/01 LAST, FIRST 117 | 02 NAME, BOB 118 | FBI/7778889LA2 119 | CDL/C45667234 C14546456 120 | SOC/549377146 121 | OCC/CONCRET 122 | TEXT 123 | end 124 | -------------------------------------------------------------------------------- /spec/domain/event_with_eligibility_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | require 'rap_sheet_parser' 3 | require_relative '../../app/domain/event_with_eligibility' 4 | describe EventWithEligibility do 5 | let(:rap_sheet_with_eligibility) { build_rap_sheet_with_eligibility(rap_sheet: build_rap_sheet) } 6 | subject { described_class.new(event: event, eligibility: rap_sheet_with_eligibility) } 7 | 8 | describe 'remedy' do 9 | let(:event) do 10 | build_court_event( 11 | date: Time.zone.today - 7.days, 12 | courthouse: 'CASC San Francisco', 13 | counts: [build_count(disposition: build_disposition(sentence: sentence))] 14 | ) 15 | end 16 | 17 | context 'when the sentence is still active' do 18 | let(:sentence) do 19 | RapSheetParser::ConvictionSentence.new(probation: 3.years) 20 | end 21 | 22 | it 'returns resentencing' do 23 | expect(subject.remedy).to eq 'resentencing' 24 | end 25 | end 26 | 27 | context 'when the sentence is not active' do 28 | let(:sentence) do 29 | RapSheetParser::ConvictionSentence.new(probation: 2.days) 30 | end 31 | 32 | it 'returns redesignation' do 33 | expect(subject.remedy).to eq 'redesignation' 34 | end 35 | end 36 | 37 | context 'when there is no sentence' do 38 | let(:sentence) { nil } 39 | 40 | it 'returns redisgnation' do 41 | expect(subject.remedy).to eq 'redesignation' 42 | end 43 | end 44 | end 45 | 46 | describe 'potentially_eligible_counts' do 47 | context 'when event has prop 64 eligible and ineligible counts' do 48 | let(:event) do 49 | prop64_eligible_count = build_count(code: 'HS', section: '11357') 50 | prop64_ineligible_count = build_count(code: 'HS', section: '12345') 51 | build_court_event(counts: [prop64_eligible_count, prop64_ineligible_count]) 52 | end 53 | 54 | it 'filters out counts that are not prop 64 convictions' do 55 | expect(subject.potentially_eligible_counts.length).to eq 1 56 | expect(subject.potentially_eligible_counts[0].code_section).to eq 'HS 11357' 57 | end 58 | end 59 | 60 | context 'there are dismissed counts' do 61 | let(:event) do 62 | prop64_eligible_count = build_count(code: 'HS', section: '11357', disposition: build_disposition(type: 'convicted')) 63 | prop64_ineligible_count = build_count(code: 'HS', section: '11360', disposition: build_disposition(type: 'dismissed')) 64 | build_court_event(counts: [prop64_eligible_count, prop64_ineligible_count]) 65 | end 66 | 67 | it 'filters out counts without convictions' do 68 | expect(subject.potentially_eligible_counts.length).to eq 1 69 | expect(subject.potentially_eligible_counts[0].code_section).to eq 'HS 11357' 70 | end 71 | end 72 | end 73 | 74 | describe 'eligible_counts' do 75 | let(:eligible_counts) { subject.eligible_counts } 76 | 77 | context 'when event has prop 64 eligible and ineligible counts' do 78 | let(:event) do 79 | prop64_eligible_count = build_count(code: 'HS', section: '11357') 80 | prop64_ineligible_count = build_count(code: 'HS', section: '12345') 81 | build_court_event(counts: [prop64_eligible_count, prop64_ineligible_count]) 82 | end 83 | 84 | it 'filters out counts that are not prop 64 convictions' do 85 | expect(eligible_counts.length).to eq 1 86 | expect(eligible_counts[0].code_section).to eq 'HS 11357' 87 | end 88 | end 89 | 90 | context 'there are dismissed counts' do 91 | let(:event) do 92 | prop64_eligible_count = build_count(code: 'HS', section: '11357', disposition: build_disposition(type: 'convicted')) 93 | prop64_ineligible_count = build_count(code: 'HS', section: '11360', disposition: build_disposition(type: 'dismissed')) 94 | build_court_event(counts: [prop64_eligible_count, prop64_ineligible_count]) 95 | end 96 | 97 | it 'filters out counts without convictions' do 98 | expect(eligible_counts.length).to eq 1 99 | expect(eligible_counts[0].code_section).to eq 'HS 11357' 100 | end 101 | end 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /config/environments/production.rb: -------------------------------------------------------------------------------- 1 | Rails.application.configure do 2 | # Settings specified here will take precedence over those in config/application.rb. 3 | 4 | # Code is not reloaded between requests. 5 | config.cache_classes = true 6 | 7 | # Eager load code on boot. This eager loads most of Rails and 8 | # your application in memory, allowing both threaded web servers 9 | # and those relying on copy on write to perform better. 10 | # Rake tasks automatically ignore this option for performance. 11 | config.eager_load = true 12 | 13 | # Full error reports are disabled and caching is turned on. 14 | config.consider_all_requests_local = false 15 | config.action_controller.perform_caching = true 16 | 17 | # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] 18 | # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). 19 | # config.require_master_key = true 20 | 21 | # Disable serving static files from the `/public` folder by default since 22 | # Apache or NGINX already handles this. 23 | config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? 24 | 25 | # Compress JavaScripts and CSS. 26 | config.assets.js_compressor = :uglifier 27 | # config.assets.css_compressor = :sass 28 | 29 | # Do not fallback to assets pipeline if a precompiled asset is missed. 30 | config.assets.compile = false 31 | 32 | # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb 33 | 34 | # Enable serving of images, stylesheets, and JavaScripts from an asset server. 35 | # config.action_controller.asset_host = 'http://assets.example.com' 36 | 37 | # Specifies the header that your server uses for sending files. 38 | # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache 39 | # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX 40 | 41 | # Store uploaded files on the local file system (see config/storage.yml for options) 42 | config.active_storage.service = :local 43 | 44 | # Mount Action Cable outside main process or domain 45 | # config.action_cable.mount_path = nil 46 | # config.action_cable.url = 'wss://example.com/cable' 47 | # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] 48 | 49 | # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. 50 | # config.force_ssl = true 51 | 52 | # Use the lowest log level to ensure availability of diagnostic information 53 | # when problems arise. 54 | config.log_level = :info 55 | 56 | # Prepend all log lines with the following tags. 57 | config.log_tags = [:request_id] 58 | 59 | # Use a different cache store in production. 60 | # config.cache_store = :mem_cache_store 61 | 62 | # Use a real queuing backend for Active Job (and separate queues per environment) 63 | # config.active_job.queue_adapter = :resque 64 | # config.active_job.queue_name_prefix = "autoclearance_#{Rails.env}" 65 | 66 | config.action_mailer.perform_caching = false 67 | 68 | # Ignore bad email addresses and do not raise email delivery errors. 69 | # Set this to true and configure the email server for immediate delivery to raise delivery errors. 70 | # config.action_mailer.raise_delivery_errors = false 71 | 72 | # Enable locale fallbacks for I18n (makes lookups for any locale fall back to 73 | # the I18n.default_locale when a translation cannot be found). 74 | config.i18n.fallbacks = true 75 | 76 | # Send deprecation notices to registered listeners. 77 | config.active_support.deprecation = :notify 78 | 79 | # Use default logging formatter so that PID and timestamp are not suppressed. 80 | config.log_formatter = ::Logger::Formatter.new 81 | 82 | # Use a different logger for distributed setups. 83 | # require 'syslog/logger' 84 | # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') 85 | 86 | if ENV["RAILS_LOG_TO_STDOUT"].present? 87 | logger = ActiveSupport::Logger.new(STDOUT) 88 | logger.formatter = config.log_formatter 89 | config.logger = ActiveSupport::TaggedLogging.new(logger) 90 | end 91 | 92 | # Do not dump schema after migrations. 93 | config.active_record.dump_schema_after_migration = false 94 | 95 | config.fog_params = { 96 | provider: 'aws', 97 | use_iam_profile: true 98 | } 99 | 100 | config.input_bucket = 'autoclearance-rap-sheet-inputs-prod' 101 | config.output_bucket = 'autoclearance-outputs-prod' 102 | 103 | config.logged_exceptions = StandardError 104 | end 105 | -------------------------------------------------------------------------------- /app/helpers/rap_sheet_processor.rb: -------------------------------------------------------------------------------- 1 | require 'csv' 2 | require 'rap_sheet_parser' 3 | 4 | class RapSheetProcessor 5 | def initialize(county) 6 | @county = county 7 | @summary_csv = SummaryCSV.new 8 | @summary_errors = '' 9 | @warning_log = StringIO.new 10 | @num_motions = 0 11 | @rap_sheets_processed = 0 12 | end 13 | 14 | def run 15 | start_time = Time.zone.now 16 | initial_rap_sheet_count = AnonRapSheet.count 17 | 18 | input_directory.files.to_a.sort_by(&:key).each do |input_file| 19 | process_one_rap_sheet(input_file) 20 | end 21 | 22 | timestamp = start_time.strftime('%Y%m%d-%H%M%S') 23 | save_summary_csv(timestamp) 24 | save_summary_errors(timestamp) 25 | save_summary_warnings(timestamp) 26 | 27 | time = (Time.zone.now - start_time) 28 | 29 | # rubocop:disable Rails/Output 30 | puts "#{@rap_sheets_processed} RAP #{'sheet'.pluralize(@rap_sheets_processed)} " \ 31 | "produced #{@num_motions} #{'motion'.pluralize(@num_motions)} " \ 32 | "in #{time} seconds" 33 | 34 | new_rap_sheets = AnonRapSheet.count - initial_rap_sheet_count 35 | existing_rap_sheets = @rap_sheets_processed - new_rap_sheets 36 | puts "Added #{new_rap_sheets} RAP #{'sheet'.pluralize(new_rap_sheets)} to analysis db. " \ 37 | "Overwrote #{existing_rap_sheets} existing RAP #{'sheet'.pluralize(existing_rap_sheets)}." 38 | # rubocop:enable Rails/Output 39 | end 40 | 41 | private 42 | 43 | attr_accessor :summary_csv, :summary_errors 44 | 45 | def process_one_rap_sheet(input_file) 46 | warning_logger = Logger.new( 47 | @warning_log, 48 | formatter: proc { |_severity, _datetime, _progname, msg| "[#{input_file.key}] #{msg}\n" } 49 | ) 50 | 51 | eligibility = rap_sheet_with_eligibility(input_file, warning_logger) 52 | 53 | unless eligibility.potentially_eligible_counts? 54 | input_file.destroy 55 | return 56 | end 57 | 58 | summary_csv.append(input_file.key, eligibility) 59 | 60 | # Create a bunch of PDFs 61 | eligibility 62 | .eligible_events 63 | .select { |event| event.eligible_counts.any? } 64 | .each_with_index do |event, index| 65 | eligible_counts = event.eligible_counts.select { |c| c.plea_bargain == 'no' } 66 | next if eligible_counts.empty? 67 | file_name = "#{input_file.key.gsub('.pdf', '')}_motion_#{index}.pdf" 68 | 69 | @num_motions += 1 70 | output_directory.files.create( 71 | key: file_name, 72 | body: FillProp64Motion.new(@county[:ada], eligible_counts, event, eligibility.personal_info).filled_motion, 73 | content_type: 'application/pdf' 74 | ) 75 | end 76 | input_file.destroy 77 | rescue Rails.configuration.logged_exceptions => e 78 | error = ([e.message] + e.backtrace).join("\n") 79 | output_directory.files.create( 80 | key: input_file.key.gsub('.pdf', '.error'), 81 | body: error 82 | ) 83 | summary_errors << "#{input_file.key}:\n#{error}\n\n" 84 | end 85 | 86 | def save_summary_errors(timestamp) 87 | return if summary_errors.blank? 88 | 89 | output_directory.files.create( 90 | key: "summary_#{timestamp}.error", 91 | body: summary_errors 92 | ) 93 | end 94 | 95 | def save_summary_warnings(timestamp) 96 | return if @warning_log.string.blank? 97 | 98 | output_directory.files.create( 99 | key: "summary_#{timestamp}.warning", 100 | body: @warning_log.string 101 | ) 102 | end 103 | 104 | def save_summary_csv(timestamp) 105 | output_directory.files.create( 106 | key: "summary_#{timestamp}.csv", 107 | body: summary_csv.text, 108 | content_type: 'text/csv' 109 | ) 110 | end 111 | 112 | def input_directory 113 | @input_directory ||= connection.directories.new(key: Rails.configuration.input_bucket) 114 | end 115 | 116 | def output_directory 117 | @output_directory ||= connection.directories.new(key: Rails.configuration.output_bucket) 118 | end 119 | 120 | def connection 121 | @connection ||= Fog::Storage.new(Rails.configuration.fog_params) 122 | end 123 | 124 | def rap_sheet_with_eligibility(input_file, logger) 125 | text = PDFReader.new(input_file.body).text 126 | 127 | rap_sheet = RapSheetParser::Parser.new.parse(text, logger: logger) 128 | 129 | @rap_sheets_processed += 1 130 | 131 | AnonRapSheet.create_or_update( 132 | text: text, 133 | county: @county[:name], 134 | rap_sheet: rap_sheet 135 | ) 136 | 137 | RapSheetWithEligibility.new( 138 | rap_sheet: rap_sheet, 139 | county: @county, 140 | logger: logger 141 | ) 142 | end 143 | end 144 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rap_sheet_factory' 2 | require 'active_support/testing/time_helpers' 3 | Dir[File.join(__dir__, 'support/**/*.rb')].each { |f| require f } 4 | 5 | # This file was generated by the `rails generate rspec:install` command. Conventionally, all 6 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 7 | # The generated `.rspec` file contains `--require spec_helper` which will cause 8 | # this file to always be loaded, without a need to explicitly require it in any 9 | # files. 10 | # 11 | # Given that it is always loaded, you are encouraged to keep this file as 12 | # light-weight as possible. Requiring heavyweight dependencies from this file 13 | # will add to the boot time of your test suite on EVERY test run, even for an 14 | # individual file that may not need all of that loaded. Instead, consider making 15 | # a separate helper file that requires the additional dependencies and performs 16 | # the additional setup, and require it from the spec files that actually need 17 | # it. 18 | # 19 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 20 | RSpec.configure do |config| 21 | config.include RapSheetParser::RapSheetFactory 22 | config.include PdfHelpers 23 | config.include ActiveSupport::Testing::TimeHelpers 24 | 25 | # rspec-expectations config goes here. You can use an alternate 26 | # assertion/expectation library such as wrong or the stdlib/minitest 27 | # assertions if you prefer. 28 | config.expect_with :rspec do |expectations| 29 | # This option will default to `true` in RSpec 4. It makes the `description` 30 | # and `failure_message` of custom matchers include text for helper methods 31 | # defined using `chain`, e.g.: 32 | # be_bigger_than(2).and_smaller_than(4).description 33 | # # => "be bigger than 2 and smaller than 4" 34 | # ...rather than: 35 | # # => "be bigger than 2" 36 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 37 | end 38 | 39 | # rspec-mocks config goes here. You can use an alternate test double 40 | # library (such as bogus or mocha) by changing the `mock_with` option here. 41 | config.mock_with :rspec do |mocks| 42 | # Prevents you from mocking or stubbing a method that does not exist on 43 | # a real object. This is generally recommended, and will default to 44 | # `true` in RSpec 4. 45 | mocks.verify_partial_doubles = true 46 | end 47 | 48 | # This option will default to `:apply_to_host_groups` in RSpec 4 (and will 49 | # have no way to turn it off -- the option exists only for backwards 50 | # compatibility in RSpec 3). It causes shared context metadata to be 51 | # inherited by the metadata hash of host groups and examples, rather than 52 | # triggering implicit auto-inclusion in groups with matching metadata. 53 | config.shared_context_metadata_behavior = :apply_to_host_groups 54 | 55 | # Allows RSpec to persist some state between runs in order to support 56 | # the `--only-failures` and `--next-failure` CLI options. We recommend 57 | # you configure your source control system to ignore this file. 58 | config.example_status_persistence_file_path = 'tmp/examples.txt' 59 | 60 | # The settings below are suggested to provide a good initial experience 61 | # with RSpec, but feel free to customize to your heart's content. 62 | =begin 63 | # This allows you to limit a spec run to individual examples or groups 64 | # you care about by tagging them with `:focus` metadata. When nothing 65 | # is tagged with `:focus`, all examples get run. RSpec also provides 66 | # aliases for `it`, `describe`, and `context` that include `:focus` 67 | # metadata: `fit`, `fdescribe` and `fcontext`, respectively. 68 | config.filter_run_when_matching :focus 69 | 70 | # Limits the available syntax to the non-monkey patched syntax that is 71 | # recommended. For more details, see: 72 | # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ 73 | # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 74 | # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode 75 | config.disable_monkey_patching! 76 | 77 | # Many RSpec users commonly either run the entire suite or an individual 78 | # file, and it's useful to allow more verbose output when running an 79 | # individual spec file. 80 | if config.files_to_run.one? 81 | # Use the documentation formatter for detailed output, 82 | # unless a formatter has already been configured 83 | # (e.g. via a command-line flag). 84 | config.default_formatter = "doc" 85 | end 86 | 87 | # Print the 10 slowest examples and example groups at the 88 | # end of the spec run, to help surface which specs are running 89 | # particularly slow. 90 | config.profile_examples = 10 91 | 92 | # Run specs in random order to surface order dependencies. If you find an 93 | # order dependency and want to debug it, you can fix the order by providing 94 | # the seed, which is printed after each run. 95 | # --seed 1234 96 | config.order = :random 97 | 98 | # Seed global randomization in this process using the `--seed` CLI option. 99 | # Setting this allows you to use `--seed` to deterministically reproduce 100 | # test failures related to randomization by passing the same `--seed` value 101 | # as the one that triggered the failure. 102 | Kernel.srand config.seed 103 | =end 104 | end 105 | -------------------------------------------------------------------------------- /db/schema.rb: -------------------------------------------------------------------------------- 1 | # This file is auto-generated from the current state of the database. Instead 2 | # of editing this file, please use the migrations feature of Active Record to 3 | # incrementally modify your database, and then regenerate this schema definition. 4 | # 5 | # Note that this schema.rb definition is the authoritative source for your 6 | # database schema. If you need to create the application database on another 7 | # system, you should be using db:schema:load, not running all the migrations 8 | # from scratch. The latter is a flawed and unsustainable approach (the more migrations 9 | # you'll amass, the slower it'll run and the greater likelihood for issues). 10 | # 11 | # It's strongly recommended that you check this file into your version control system. 12 | 13 | ActiveRecord::Schema.define(version: 2018_09_18_213324) do 14 | 15 | # These are extensions that must be enabled in order to support this database 16 | enable_extension "pgcrypto" 17 | enable_extension "plpgsql" 18 | 19 | create_table "anon_counts", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 20 | t.string "code" 21 | t.string "section" 22 | t.string "description" 23 | t.datetime "created_at", null: false 24 | t.datetime "updated_at", null: false 25 | t.uuid "anon_event_id", null: false 26 | t.integer "count_number", null: false 27 | t.index ["anon_event_id"], name: "index_anon_counts_on_anon_event_id" 28 | end 29 | 30 | create_table "anon_cycles", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 31 | t.datetime "created_at", null: false 32 | t.datetime "updated_at", null: false 33 | t.uuid "anon_rap_sheet_id", null: false 34 | t.index ["anon_rap_sheet_id"], name: "index_anon_cycles_on_anon_rap_sheet_id" 35 | end 36 | 37 | create_table "anon_dispositions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 38 | t.datetime "created_at", null: false 39 | t.datetime "updated_at", null: false 40 | t.uuid "anon_count_id", null: false 41 | t.string "disposition_type", null: false 42 | t.string "sentence" 43 | t.text "text" 44 | t.string "severity" 45 | t.index ["anon_count_id"], name: "index_anon_dispositions_on_anon_count_id" 46 | end 47 | 48 | create_table "anon_events", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 49 | t.string "agency" 50 | t.string "event_type", null: false 51 | t.date "date" 52 | t.datetime "created_at", null: false 53 | t.datetime "updated_at", null: false 54 | t.uuid "anon_cycle_id", null: false 55 | t.index ["anon_cycle_id"], name: "index_anon_events_on_anon_cycle_id" 56 | end 57 | 58 | create_table "anon_rap_sheets", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 59 | t.binary "checksum", null: false 60 | t.datetime "created_at", null: false 61 | t.datetime "updated_at", null: false 62 | t.string "county", null: false 63 | t.integer "year_of_birth" 64 | t.string "sex" 65 | t.string "race" 66 | t.binary "person_unique_id" 67 | t.index ["checksum"], name: "index_anon_rap_sheets_on_checksum", unique: true 68 | end 69 | 70 | create_table "count_properties", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 71 | t.datetime "created_at", null: false 72 | t.datetime "updated_at", null: false 73 | t.uuid "anon_count_id", null: false 74 | t.boolean "has_prop_64_code", null: false 75 | t.boolean "has_two_prop_64_priors", null: false 76 | t.string "prop_64_plea_bargain", null: false 77 | t.index ["anon_count_id"], name: "index_count_properties_on_anon_count_id" 78 | end 79 | 80 | create_table "eligibility_estimates", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 81 | t.datetime "created_at", null: false 82 | t.datetime "updated_at", null: false 83 | t.uuid "count_properties_id", null: false 84 | t.string "prop_64_eligible", null: false 85 | t.index ["count_properties_id"], name: "index_eligibility_estimates_on_count_properties_id" 86 | end 87 | 88 | create_table "event_properties", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 89 | t.datetime "created_at", null: false 90 | t.datetime "updated_at", null: false 91 | t.uuid "anon_event_id", null: false 92 | t.boolean "has_felonies", null: false 93 | t.boolean "has_probation", null: false 94 | t.boolean "has_probation_violations", null: false 95 | t.boolean "has_prison", null: false 96 | t.boolean "dismissed_by_pc1203", null: false 97 | t.index ["anon_event_id"], name: "index_event_properties_on_anon_event_id" 98 | end 99 | 100 | create_table "rap_sheet_properties", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| 101 | t.datetime "created_at", null: false 102 | t.datetime "updated_at", null: false 103 | t.uuid "anon_rap_sheet_id", null: false 104 | t.boolean "has_superstrikes", null: false 105 | t.boolean "has_sex_offender_registration", null: false 106 | t.boolean "deceased", null: false 107 | t.index ["anon_rap_sheet_id"], name: "index_rap_sheet_properties_on_anon_rap_sheet_id" 108 | end 109 | 110 | add_foreign_key "anon_counts", "anon_events" 111 | add_foreign_key "anon_cycles", "anon_rap_sheets" 112 | add_foreign_key "anon_dispositions", "anon_counts" 113 | add_foreign_key "anon_events", "anon_cycles" 114 | add_foreign_key "count_properties", "anon_counts" 115 | add_foreign_key "eligibility_estimates", "count_properties", column: "count_properties_id" 116 | add_foreign_key "event_properties", "anon_events" 117 | add_foreign_key "rap_sheet_properties", "anon_rap_sheets" 118 | end 119 | -------------------------------------------------------------------------------- /spec/helpers/rap_sheet_processor_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe RapSheetProcessor do 4 | describe '#run' do 5 | subject do 6 | travel_to(Time.zone.local(2011, 1, 2, 1, 4, 44)) do 7 | described_class.new(Counties::SAN_FRANCISCO).run 8 | end 9 | end 10 | 11 | before :each do 12 | Dir.mkdir '/tmp/autoclearance-rap-sheet-inputs' unless Dir.exist? '/tmp/autoclearance-rap-sheet-inputs' 13 | Dir.mkdir '/tmp/autoclearance-outputs' unless Dir.exist? '/tmp/autoclearance-outputs' 14 | end 15 | 16 | after :each do 17 | FileUtils.rm Dir.glob('/tmp/autoclearance-rap-sheet-inputs/*') 18 | FileUtils.rm Dir.glob('/tmp/autoclearance-outputs/*') 19 | end 20 | 21 | it 'reads rap sheets from input bucket and saves conviction data to output bucket' do 22 | FileUtils.cp('spec/fixtures/skywalker_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 23 | FileUtils.cp('spec/fixtures/chewbacca_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 24 | 25 | expect { subject }.to output(Regexp.new('2 RAP sheets produced 1 motion in ([0-9]|\.)+ seconds\n'\ 26 | 'Added 2 RAP sheets to analysis db. Overwrote 0 existing RAP sheets.')).to_stdout 27 | 28 | expect(Dir['/tmp/autoclearance-rap-sheet-inputs/*']).to be_empty 29 | expect(Dir['/tmp/autoclearance-outputs/*']) 30 | .to contain_exactly( 31 | '/tmp/autoclearance-outputs/summary_20110102-010444.csv', 32 | '/tmp/autoclearance-outputs/skywalker_rap_sheet_motion_0.pdf' 33 | ) 34 | 35 | skywalker_motion_fields = get_fields_from_pdf(File.open('/tmp/autoclearance-outputs/skywalker_rap_sheet_motion_0.pdf')) 36 | expect(skywalker_motion_fields['Defendant']).to eq 'SKYWALKER,LUKE JAY' 37 | 38 | summary_expected_text = File.read('spec/fixtures/summary.csv') 39 | summary_actual_text = File.read('/tmp/autoclearance-outputs/summary_20110102-010444.csv') 40 | expect(summary_actual_text).to eq(summary_expected_text) 41 | end 42 | 43 | it 'catches exceptions in the parser and logs the error to an output file' do 44 | FileUtils.cp('spec/fixtures/not_a_valid_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 45 | FileUtils.cp('spec/fixtures/skywalker_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 46 | 47 | fake_exception = RapSheetParser::RapSheetParserException.new( 48 | double('parser', failure_reason: 'Error: Not a valid RAP sheet'), 49 | '' 50 | ) 51 | 52 | allow_any_instance_of(RapSheetParser::Parser).to receive(:parse).and_wrap_original do |m, *args| 53 | raise fake_exception if args[0].include? 'not_a_valid_rap_sheet' 54 | 55 | m.call(*args) 56 | end 57 | 58 | expect { subject }.to output(Regexp.new('1 RAP sheet produced 1 motion in ([0-9]|\.)+ seconds\n'\ 59 | 'Added 1 RAP sheet to analysis db. Overwrote 0 existing RAP sheets.')).to_stdout 60 | 61 | expect(Dir['/tmp/autoclearance-rap-sheet-inputs/*']) 62 | .to contain_exactly('/tmp/autoclearance-rap-sheet-inputs/not_a_valid_rap_sheet.pdf') 63 | expect(Dir['/tmp/autoclearance-outputs/*']) 64 | .to contain_exactly( 65 | '/tmp/autoclearance-outputs/not_a_valid_rap_sheet.error', 66 | '/tmp/autoclearance-outputs/summary_20110102-010444.error', 67 | '/tmp/autoclearance-outputs/summary_20110102-010444.csv', 68 | '/tmp/autoclearance-outputs/skywalker_rap_sheet_motion_0.pdf' 69 | ) 70 | 71 | actual_error = File.read('/tmp/autoclearance-outputs/not_a_valid_rap_sheet.error') 72 | actual_summary_error = File.read('/tmp/autoclearance-outputs/summary_20110102-010444.error') 73 | 74 | expect(actual_error).to include('Error: Not a valid RAP sheet') 75 | expect(actual_summary_error).to include('not_a_valid_rap_sheet.pdf:') 76 | expect(actual_summary_error).to include('Error: Not a valid RAP sheet') 77 | end 78 | 79 | it 'sends warnings from the parser to a warnings file' do 80 | FileUtils.cp('spec/fixtures/not_a_valid_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 81 | 82 | expect { subject }.to output(Regexp.new('1 RAP sheet produced 0 motions in ([0-9]|\.)+ seconds\n'\ 83 | 'Added 1 RAP sheet to analysis db. Overwrote 0 existing RAP sheets.')).to_stdout 84 | 85 | actual_warning = File.read('/tmp/autoclearance-outputs/summary_20110102-010444.warning') 86 | 87 | expect(actual_warning).to include("[not_a_valid_rap_sheet.pdf] Unrecognized event:\n[not_a_valid_rap_sheet.pdf] HI") 88 | expect(actual_warning).to include("[not_a_valid_rap_sheet.pdf] Unrecognized event:\n[not_a_valid_rap_sheet.pdf] BYE") 89 | end 90 | 91 | it 'saves a database entry for each rap sheet' do 92 | FileUtils.cp('spec/fixtures/skywalker_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 93 | 94 | expect { described_class.new(Counties::SAN_FRANCISCO).run } 95 | .to output(/1 RAP sheet produced 1 motion in ([0-9]|\.)+ seconds\nAdded 1 RAP sheet to analysis db. Overwrote 0 existing RAP sheets./).to_stdout 96 | 97 | FileUtils.cp('spec/fixtures/skywalker_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 98 | FileUtils.cp('spec/fixtures/chewbacca_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 99 | 100 | expect { described_class.new(Counties::SAN_FRANCISCO).run } 101 | .to output(/2 RAP sheets produced 1 motion in ([0-9]|\.)+ seconds\nAdded 1 RAP sheet to analysis db. Overwrote 1 existing RAP sheet./).to_stdout 102 | 103 | expect(AnonRapSheet.count).to eq 2 104 | end 105 | end 106 | end 107 | -------------------------------------------------------------------------------- /spec/domain/rap_sheet_with_eligibility_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | require 'rap_sheet_parser' 3 | require_relative '../../app/domain/event_with_eligibility' 4 | require_relative '../../app/domain/rap_sheet_with_eligibility' 5 | 6 | describe RapSheetWithEligibility do 7 | describe '#eligible_events' do 8 | it 'filters out events not in given county' do 9 | eligible_event1 = build_court_event( 10 | courthouse: 'Some courthouse', 11 | case_number: 'abc' 12 | ) 13 | eligible_event2 = build_court_event( 14 | courthouse: 'Another courthouse', 15 | case_number: 'def' 16 | ) 17 | ineligible_event = build_court_event( 18 | courthouse: 'ineligible courthouse' 19 | ) 20 | rap_sheet = build_rap_sheet(events: [eligible_event1, eligible_event2, ineligible_event]) 21 | 22 | subject = build_rap_sheet_with_eligibility( 23 | rap_sheet: rap_sheet, county: build_county(courthouses: ['Some courthouse', 'Another courthouse']) 24 | ).eligible_events 25 | expect(subject.length).to eq 2 26 | expect(subject[0].case_number).to eq 'abc' 27 | expect(subject[1].case_number).to eq 'def' 28 | end 29 | 30 | it 'filters out events dismissed by PC1203' do 31 | eligible_event = build_court_event 32 | ineligible_event = build_court_event( 33 | counts: [ 34 | build_count( 35 | updates: [ 36 | RapSheetParser::Update.new(dispositions: [build_disposition(type: 'pc1203_dismissed')]) 37 | ] 38 | ) 39 | ] 40 | ) 41 | rap_sheet = build_rap_sheet(events: [eligible_event, ineligible_event]) 42 | 43 | expect(build_rap_sheet_with_eligibility(rap_sheet: rap_sheet).eligible_events).to eq [eligible_event] 44 | end 45 | end 46 | 47 | describe '#has_three_convictions_of_same_type?' do 48 | it 'returns true if rap sheet contains three convictions with the same code section' do 49 | event1 = build_court_event(counts: [build_count(code: 'PC', section: '123')]) 50 | event2 = build_court_event(counts: [build_count(code: 'PC', section: '123')]) 51 | event3 = build_court_event(counts: [build_count(code: 'PC', section: '123')]) 52 | 53 | rap_sheet = build_rap_sheet(events: [event1, event2, event3]) 54 | 55 | expect(build_rap_sheet_with_eligibility(rap_sheet: rap_sheet).has_three_convictions_of_same_type?('PC 123')).to eq true 56 | end 57 | 58 | it 'returns false if rap sheet does not contain two convictions and one dismissed count for the same code section' do 59 | event1 = build_court_event(counts: [build_count(code: 'PC', section: '123', disposition: build_disposition(type: 'convicted'))]) 60 | event2 = build_court_event(counts: [build_count(code: 'PC', section: '123', disposition: build_disposition(type: 'convicted'))]) 61 | event3 = build_court_event(counts: [build_count(code: 'PC', section: '123', disposition: build_disposition(type: 'dismissed'))]) 62 | 63 | rap_sheet = build_rap_sheet(events: [event1, event2, event3]) 64 | 65 | expect(build_rap_sheet_with_eligibility(rap_sheet: rap_sheet).has_three_convictions_of_same_type?('PC 123')).to eq false 66 | end 67 | 68 | it 'returns false if rap sheet contains three convictions for the same code section in two events' do 69 | event1 = build_court_event( 70 | counts: [ 71 | build_count(code: 'PC', section: '123'), 72 | build_count(code: 'PC', section: '123') 73 | ] 74 | ) 75 | event2 = build_court_event(counts: [build_count(code: 'PC', section: '123')]) 76 | 77 | rap_sheet = build_rap_sheet(events: [event1, event2]) 78 | 79 | expect(build_rap_sheet_with_eligibility(rap_sheet: rap_sheet).has_three_convictions_of_same_type?('PC 123')).to eq false 80 | end 81 | end 82 | 83 | context 'warnings for rap sheets with nothing potentially eligible' do 84 | it 'saves warnings for rap sheets that did not seem to have any potentially prop64 eligible counts' do 85 | count = build_count(code: 'HS', section: '1234') 86 | events = [build_court_event(counts: [count], courthouse: 'CASC San Francisco')] 87 | rap_sheet = build_rap_sheet(events: events) 88 | 89 | warning_file = StringIO.new 90 | build_rap_sheet_with_eligibility(rap_sheet: rap_sheet, logger: Logger.new(warning_file)) 91 | 92 | expect(warning_file.string).to include('No eligible prop64 convictions found') 93 | end 94 | end 95 | 96 | describe '#disqualifiers?' do 97 | let(:code_section) {} 98 | 99 | subject { build_rap_sheet_with_eligibility(rap_sheet: build_rap_sheet(events: events)) } 100 | 101 | context 'sex offender registration' do 102 | let(:events) do 103 | [ 104 | build_other_event( 105 | event_type: 'registration', 106 | counts: [build_count(code: 'PC', section: '290')] 107 | ) 108 | ] 109 | end 110 | 111 | it 'returns true if registered sex offender' do 112 | expect(subject.disqualifiers?).to eq true 113 | end 114 | end 115 | 116 | context 'superstrike on rap sheet' do 117 | let(:events) do 118 | superstrike = build_count(code: 'PC', section: '187') 119 | [build_court_event(counts: [superstrike])] 120 | end 121 | 122 | it 'returns true if superstrike' do 123 | expect(subject.disqualifiers?).to eq true 124 | end 125 | end 126 | 127 | context 'none of the above' do 128 | let(:events) do 129 | event1_count = build_count(code: 'HS', section: '11357') 130 | [build_court_event(case_number: '111', counts: [event1_count])] 131 | end 132 | 133 | it 'returns false' do 134 | expect(subject.disqualifiers?).to eq false 135 | end 136 | end 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /.rubocop_todo.yml: -------------------------------------------------------------------------------- 1 | # This configuration was generated by 2 | # `rubocop --auto-gen-config` 3 | # on 2018-08-29 10:29:16 -0700 using RuboCop version 0.58.2. 4 | # The point is for the user to remove these configuration records 5 | # one by one as the offenses are removed from the code base. 6 | # Note that changes in the inspected code, or installation of new 7 | # versions of RuboCop, may require this file to be generated again. 8 | 9 | # Offense count: 6 10 | # Cop supports --auto-correct. 11 | # Configuration parameters: IndentationWidth. 12 | # SupportedStyles: special_inside_parentheses, consistent, align_braces 13 | Layout/IndentHash: 14 | EnforcedStyle: consistent 15 | 16 | # Offense count: 2 17 | # Cop supports --auto-correct. 18 | Lint/DeprecatedClassMethods: 19 | Exclude: 20 | - 'app/lib/fill_misdemeanor_petitions.rb' 21 | 22 | # Offense count: 1 23 | Lint/HandleExceptions: 24 | Exclude: 25 | - 'Rakefile' 26 | 27 | # Offense count: 2 28 | Lint/IneffectiveAccessModifier: 29 | Exclude: 30 | - 'app/models/anon_count.rb' 31 | - 'app/models/anon_rap_sheet.rb' 32 | 33 | 34 | # Offense count: 2 35 | # Configuration parameters: ContextCreatingMethods, MethodCreatingMethods. 36 | Lint/UselessAccessModifier: 37 | Exclude: 38 | - 'app/models/anon_count.rb' 39 | - 'app/models/anon_rap_sheet.rb' 40 | 41 | # Offense count: 1 42 | Lint/UselessAssignment: 43 | Exclude: 44 | - 'spec/helpers/pdf_reader_spec.rb' 45 | 46 | # Offense count: 9 47 | Metrics/AbcSize: 48 | Max: 47 49 | 50 | # Offense count: 42 51 | # Configuration parameters: CountComments, ExcludedMethods. 52 | # ExcludedMethods: refine 53 | Metrics/BlockLength: 54 | Max: 383 55 | 56 | # Offense count: 1 57 | # Configuration parameters: CountComments. 58 | Metrics/ClassLength: 59 | Max: 113 60 | 61 | # Offense count: 1 62 | Metrics/CyclomaticComplexity: 63 | Max: 7 64 | 65 | # Offense count: 13 66 | # Configuration parameters: CountComments. 67 | Metrics/MethodLength: 68 | Max: 35 69 | 70 | # Offense count: 3 71 | # Configuration parameters: NamePrefix, NamePrefixBlacklist, NameWhitelist, MethodDefinitionMacros. 72 | # NamePrefix: is_, has_, have_ 73 | # NamePrefixBlacklist: is_, has_, have_ 74 | # NameWhitelist: is_a? 75 | # MethodDefinitionMacros: define_method, define_singleton_method 76 | Naming/PredicateName: 77 | Exclude: 78 | - 'spec/**/*' 79 | - 'app/domain/rap_sheet_with_eligibility.rb' 80 | 81 | # Offense count: 1 82 | # Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames. 83 | # AllowedNames: io, id, to, by, on, in, at, ip 84 | Naming/UncommunicativeMethodParamName: 85 | Exclude: 86 | - 'app/domain/rap_sheet_with_eligibility.rb' 87 | 88 | # Offense count: 38 89 | # Configuration parameters: EnforcedStyle. 90 | # SupportedStyles: snake_case, normalcase, non_integer 91 | Naming/VariableNumber: 92 | Exclude: 93 | - 'app/lib/fill_misdemeanor_petitions.rb' 94 | - 'spec/models/anon_cycle_spec.rb' 95 | 96 | # Offense count: 1 97 | # Cop supports --auto-correct. 98 | Style/BlockComments: 99 | Exclude: 100 | - 'spec/spec_helper.rb' 101 | 102 | # Offense count: 37 103 | Style/Documentation: 104 | Enabled: false 105 | 106 | # Offense count: 1 107 | # Cop supports --auto-correct. 108 | Style/ExpandPathArguments: 109 | Exclude: 110 | - 'spec/rails_helper.rb' 111 | 112 | # Offense count: 92 113 | # Cop supports --auto-correct. 114 | # Configuration parameters: EnforcedStyle. 115 | # SupportedStyles: when_needed, always, never 116 | Style/FrozenStringLiteralComment: 117 | Enabled: false 118 | 119 | # Offense count: 4 120 | # Configuration parameters: MinBodyLength. 121 | Style/GuardClause: 122 | Exclude: 123 | - 'app/domain/rap_sheet_with_eligibility.rb' 124 | 125 | # Offense count: 6 126 | # Cop supports --auto-correct. 127 | # Configuration parameters: EnforcedStyle, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols. 128 | # SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys 129 | Style/HashSyntax: 130 | Exclude: 131 | - 'app/controllers/application_controller.rb' 132 | - 'db/migrate/20180814010846_add_index_to_rap_sheet_checksum.rb' 133 | 134 | # Offense count: 1 135 | # Cop supports --auto-correct. 136 | Style/IfUnlessModifier: 137 | Exclude: 138 | - 'app/domain/rap_sheet_with_eligibility.rb' 139 | 140 | # Offense count: 4 141 | # Cop supports --auto-correct. 142 | # Configuration parameters: AutoCorrect, EnforcedStyle. 143 | # SupportedStyles: predicate, comparison 144 | Style/NumericPredicate: 145 | Exclude: 146 | - 'spec/**/*' 147 | 148 | # Offense count: 1 149 | # Cop supports --auto-correct. 150 | Style/RedundantSelf: 151 | Exclude: 152 | - 'app/models/anon_rap_sheet.rb' 153 | 154 | # Offense count: 1 155 | # Cop supports --auto-correct. 156 | # Configuration parameters: ConvertCodeThatCanStartToReturnNil, Whitelist. 157 | # Whitelist: present?, blank?, presence, try, try! 158 | Style/SafeNavigation: 159 | Exclude: 160 | - 'app/models/anon_rap_sheet.rb' 161 | 162 | # Offense count: 216 163 | # Cop supports --auto-correct. 164 | # Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline. 165 | # SupportedStyles: single_quotes, double_quotes 166 | Style/StringLiterals: 167 | Exclude: 168 | - 'Brewfile' 169 | - 'app/lib/fill_misdemeanor_petitions.rb' 170 | - 'config/environments/production.rb' 171 | - 'config/puma.rb' 172 | - 'db/schema.rb' 173 | - 'lib/tasks/rap_sheet.rake' 174 | - 'spec/lib/fill_misdemeanor_petitions_spec.rb' 175 | - 'spec/rails_helper.rb' 176 | 177 | # Offense count: 3 178 | # Cop supports --auto-correct. 179 | # Configuration parameters: MinSize. 180 | # SupportedStyles: percent, brackets 181 | Style/SymbolArray: 182 | EnforcedStyle: brackets 183 | 184 | # Offense count: 228 185 | # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. 186 | # URISchemes: http, https 187 | Metrics/LineLength: 188 | Max: 168 189 | -------------------------------------------------------------------------------- /spec/domain/prop64_classifier_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe Prop64Classifier do 4 | let(:count) { build_count(code: code, section: section) } 5 | let(:event) { build_court_event(counts: [count]) } 6 | let(:event_with_eligibility) { EventWithEligibility.new(event: event, eligibility: eligibility) } 7 | let(:code) {} 8 | let(:section) {} 9 | let(:eligibility) { build_rap_sheet_with_eligibility(rap_sheet: build_rap_sheet(events: [event])) } 10 | 11 | subject { described_class.new(count: count, event: event_with_eligibility, eligibility: eligibility) } 12 | 13 | describe '#potentially_eligible?' do 14 | context 'when the count is an eligible code' do 15 | let(:code) { 'HS' } 16 | let(:section) { '11359' } 17 | it 'returns true' do 18 | expect(subject).to be_potentially_eligible 19 | end 20 | end 21 | 22 | context 'when the count is a subsection of an eligible code' do 23 | let(:code) { 'HS' } 24 | let(:section) { '11359(a)' } 25 | it 'returns true' do 26 | expect(subject).to be_potentially_eligible 27 | end 28 | end 29 | 30 | context 'when the count is an ineligible code' do 31 | let(:code) { 'HS' } 32 | let(:section) { '12345' } 33 | it 'returns false' do 34 | expect(subject).not_to be_potentially_eligible 35 | end 36 | end 37 | 38 | context 'when the count is an ineligible subsection' do 39 | let(:code) { 'HS' } 40 | let(:section) { '11359(d)' } 41 | it 'returns false' do 42 | expect(subject).not_to be_potentially_eligible 43 | end 44 | end 45 | 46 | context 'when excluding misdemeanors' do 47 | let(:county) { build_county(misdemeanors: false) } 48 | let(:eligibility) { build_rap_sheet_with_eligibility(rap_sheet: build_rap_sheet(events: [event]), county: county) } 49 | let(:count) { build_count(code: 'HS', section: '11359', disposition: build_disposition(severity: 'M')) } 50 | 51 | it 'returns false' do 52 | expect(subject).not_to be_potentially_eligible 53 | end 54 | end 55 | 56 | context 'plea bargains' do 57 | context 'when the count is a possible plea bargain' do 58 | before do 59 | plea_bargain_classifier = instance_double(PleaBargainClassifier, possible_plea_bargain?: true) 60 | allow(PleaBargainClassifier).to receive(:new) 61 | .with(subject) 62 | .and_return(plea_bargain_classifier) 63 | end 64 | 65 | it 'returns true' do 66 | expect(subject.potentially_eligible?).to eq true 67 | end 68 | end 69 | 70 | context 'when the count is not a possible plea bargain' do 71 | before do 72 | plea_bargain_classifier = instance_double(PleaBargainClassifier, possible_plea_bargain?: false) 73 | allow(PleaBargainClassifier).to receive(:new) 74 | .with(subject) 75 | .and_return(plea_bargain_classifier) 76 | end 77 | 78 | it 'returns false' do 79 | expect(subject.potentially_eligible?).to eq false 80 | end 81 | end 82 | end 83 | end 84 | 85 | describe '#eligible?' do 86 | let(:has_three_convictions_of_same_type?) { false } 87 | let(:disqualifiers?) { false } 88 | let(:code) { 'HS' } 89 | let(:section) { '11358' } 90 | let(:eligibility) { double(disqualifiers?: disqualifiers?, has_three_convictions_of_same_type?: has_three_convictions_of_same_type?, county: build_county) } 91 | 92 | it 'returns true if prop64 conviction and no disqualifiers' do 93 | expect(subject.eligible?).to eq 'yes' 94 | end 95 | 96 | context 'rap sheet disqualifiers' do 97 | let(:disqualifiers?) { true } 98 | it 'returns false if rap sheet level disqualifiers' do 99 | expect(subject.eligible?).to eq 'no' 100 | end 101 | end 102 | 103 | context '2 prop 64 priors' do 104 | let(:has_three_convictions_of_same_type?) { true } 105 | it 'returns false if rap sheet level disqualifiers' do 106 | expect(subject.eligible?).to eq 'no' 107 | end 108 | end 109 | 110 | context 'plea bargain detection' do 111 | it 'returns true if plea bargain' do 112 | plea_bargain_classifier = instance_double(PleaBargainClassifier, plea_bargain?: true, possible_plea_bargain?: true) 113 | allow(PleaBargainClassifier).to receive(:new) 114 | .with(subject) 115 | .and_return(plea_bargain_classifier) 116 | 117 | expect(subject.eligible?).to eq('yes') 118 | end 119 | 120 | it 'returns "maybe" if possible plea bargain only' do 121 | plea_bargain_classifier = instance_double(PleaBargainClassifier, plea_bargain?: false, possible_plea_bargain?: true) 122 | allow(PleaBargainClassifier).to receive(:new) 123 | .with(subject) 124 | .and_return(plea_bargain_classifier) 125 | 126 | expect(subject.eligible?).to eq('maybe') 127 | end 128 | end 129 | end 130 | 131 | describe '#has_two_prop_64_priors?' do 132 | let(:code) { 'HS' } 133 | let(:section) { '11358' } 134 | let(:eligibility) { double(has_three_convictions_of_same_type?: has_three_convictions_of_same_type?) } 135 | let(:result) { subject.has_two_prop_64_priors? } 136 | 137 | context 'if there are fewer than 3 of a prop-64 two-priors relevant code' do 138 | let(:has_three_convictions_of_same_type?) { false } 139 | 140 | it 'returns false' do 141 | expect(result).to be false 142 | end 143 | end 144 | 145 | context 'if there 3 or more of a prop-64 two-priors relevant code' do 146 | let(:has_three_convictions_of_same_type?) { true } 147 | 148 | it 'returns true' do 149 | expect(result).to be true 150 | end 151 | end 152 | 153 | context 'if there 3 or more of a non-relevant code' do 154 | let(:section) { '11357' } 155 | let(:has_three_convictions_of_same_type?) {} 156 | 157 | it 'returns false' do 158 | expect(result).to be false 159 | end 160 | end 161 | end 162 | end 163 | -------------------------------------------------------------------------------- /spec/helpers/rap_sheet_deidentifier_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_helper' 2 | 3 | describe RapSheetDeidentifier do 4 | describe '#run' do 5 | before :each do 6 | Dir.mkdir '/tmp/autoclearance-rap-sheet-inputs' unless Dir.exist? '/tmp/autoclearance-rap-sheet-inputs' 7 | Dir.mkdir '/tmp/autoclearance-outputs' unless Dir.exist? '/tmp/autoclearance-outputs' 8 | end 9 | 10 | after :each do 11 | FileUtils.rm Dir.glob('/tmp/autoclearance-rap-sheet-inputs/*') 12 | FileUtils.rm Dir.glob('/tmp/autoclearance-outputs/*') 13 | end 14 | 15 | it 'reads rap sheets from input bucket and saves anonymized rap sheets to output bucket' do 16 | FileUtils.cp('spec/fixtures/skywalker_rap_sheet.pdf', '/tmp/autoclearance-rap-sheet-inputs/') 17 | 18 | described_class.new.run 19 | 20 | expect(Dir['/tmp/autoclearance-rap-sheet-inputs/*'].length).to eq 1 21 | redacted_text = Dir['/tmp/autoclearance-outputs/redacted_rap_sheet_*.txt'] 22 | expect(redacted_text.length).to eq 1 23 | 24 | skywalker_actual_text = File.read(redacted_text[0]) 25 | expect(skywalker_actual_text).to include('496 PC-RECEIVE/ETC KNOWN STOLEN PROPERTY') 26 | expect(skywalker_actual_text).not_to include('SKYWALKER') 27 | expect(skywalker_actual_text).not_to include('LUKE') 28 | expect(skywalker_actual_text).not_to include('JAY') 29 | expect(skywalker_actual_text).not_to include('A99000099') 30 | expect(skywalker_actual_text).not_to include('19660119') 31 | expect(skywalker_actual_text).not_to include('19550505') 32 | expect(skywalker_actual_text).not_to include('#1111111') 33 | expect(skywalker_actual_text).not_to include('#1234567') 34 | expect(skywalker_actual_text).not_to include('#2222222') 35 | expect(skywalker_actual_text).not_to include('#3456789') 36 | expect(skywalker_actual_text).not_to include('#33857493') 37 | expect(skywalker_actual_text).not_to include('300,NEWT AV, , ,OAKLAND,CA,94612') 38 | 39 | redacted_pdf = Dir['/tmp/autoclearance-outputs/redacted_rap_sheet_*.pdf'] 40 | expect(redacted_pdf.length).to eq 1 41 | 42 | pdf_text = PDFReader.new(File.read(redacted_pdf[0])).text 43 | expect(pdf_text).to include('CII/A01234557') 44 | expect(pdf_text).to include('496 PC-RECEIVE/ETC KNOWN STOLEN PROPERTY') 45 | expect(pdf_text).not_to include('SKYWALKER') 46 | expect(pdf_text).not_to include('LUKE') 47 | expect(pdf_text).not_to include('JAY') 48 | expect(pdf_text).not_to include('A99000099') 49 | expect(pdf_text).not_to include('19660119') 50 | expect(pdf_text).not_to include('19550505') 51 | expect(pdf_text).not_to include('#1111111') 52 | expect(pdf_text).not_to include('#1234567') 53 | expect(pdf_text).not_to include('#2222222') 54 | expect(pdf_text).not_to include('#3456789') 55 | expect(pdf_text).not_to include('#33857493') 56 | expect(pdf_text).not_to include('300,NEWT AV, , ,OAKLAND,CA,94612') 57 | end 58 | 59 | it 'skip rap sheets without any cycles' do 60 | File.open('/tmp/autoclearance-rap-sheet-inputs/bad_rap_sheet.pdf', 'w') 61 | 62 | expect_any_instance_of(PDFReader).to receive(:text).and_return('Personal info') 63 | 64 | described_class.new.run 65 | 66 | expect(Dir['/tmp/autoclearance-outputs/*'].length).to eq 0 67 | end 68 | 69 | it 'allows for nbsp in cycle delimiters' do 70 | File.open('/tmp/autoclearance-rap-sheet-inputs/example_rap_sheet.pdf', 'w') 71 | 72 | # cycle delimiter below contains invisible nbsp characters 73 | rap_sheet_text = <<~TEXT 74 | personal info 75 | * * * * 76 | cycle text 77 | * * * END OF MESSAGE * * * 78 | TEXT 79 | 80 | expect_any_instance_of(PDFReader).to receive(:text).and_return(rap_sheet_text) 81 | 82 | described_class.new.run 83 | 84 | expect(Dir['/tmp/autoclearance-outputs/*.txt'].length).to eq 1 85 | end 86 | 87 | it 'allows for invalid dates of birth' do 88 | File.open('/tmp/autoclearance-rap-sheet-inputs/example_rap_sheet.pdf', 'w') 89 | 90 | rap_sheet_text = <<~TEXT 91 | personal info 92 | * * * * 93 | DOB:19910000 94 | * * * END OF MESSAGE * * * 95 | TEXT 96 | 97 | expect_any_instance_of(PDFReader).to receive(:text).and_return(rap_sheet_text) 98 | 99 | described_class.new.run 100 | output_text_files = Dir['/tmp/autoclearance-outputs/*.txt'] 101 | expect(output_text_files.length).to eq 1 102 | actual_text = File.read(output_text_files[0]) 103 | expect(actual_text).not_to include('19910000') 104 | end 105 | 106 | it 'removes addresses with arbitrary whitspace' do 107 | File.open('/tmp/autoclearance-rap-sheet-inputs/example_rap_sheet.pdf', 'w') 108 | 109 | rap_sheet_text = <<~TEXT 110 | personal info 111 | * * * * 112 | COM: ADR-050404 (123, MAIN ST, , , OAKLAND, CA, , ) 113 | * * * END OF MESSAGE * * * 114 | TEXT 115 | 116 | expect_any_instance_of(PDFReader).to receive(:text).and_return(rap_sheet_text) 117 | 118 | described_class.new.run 119 | actual_text = File.read(Dir['/tmp/autoclearance-outputs/*.txt'][0]) 120 | expect(actual_text).not_to include('123, MAIN ST') 121 | end 122 | 123 | it 'does not consume extra lines when stripping addresses' do 124 | File.open('/tmp/autoclearance-rap-sheet-inputs/example_rap_sheet.pdf', 'w') 125 | 126 | rap_sheet_text = <<~TEXT 127 | personal info 128 | * * * * 129 | COM: ADR-050404 (123, MAIN ST, , , OAKLAND, CA, , ) 130 | CNT:01 #12345 131 | 148(A)(1) PC-OBSTRUCT/ETC PUBLIC OFFICER/ETC 132 | * * * END OF MESSAGE * * * 133 | TEXT 134 | 135 | expect_any_instance_of(PDFReader).to receive(:text).and_return(rap_sheet_text) 136 | 137 | described_class.new.run 138 | actual_text = File.read(Dir['/tmp/autoclearance-outputs/*.txt'][0]) 139 | expect(actual_text).to include('148(A)(1) PC-OBSTRUCT/ETC PUBLIC OFFICER/ETC') 140 | end 141 | end 142 | end 143 | -------------------------------------------------------------------------------- /terraform/metabase/metabase.tf: -------------------------------------------------------------------------------- 1 | variable "vpc_id" {} 2 | variable "public_subnet_id" {} 3 | variable "private_subnet_id" {} 4 | variable "db_subnet_group_name" {} 5 | variable "beanstalk_role_name" {} 6 | variable "metabase_url" {} 7 | 8 | data "aws_vpc" "host" { 9 | id = "${var.vpc_id}" 10 | } 11 | 12 | data "aws_subnet" "public_subnet" { 13 | id = "${var.public_subnet_id}" 14 | } 15 | 16 | data "aws_subnet" "private_subnet" { 17 | id = "${var.private_subnet_id}" 18 | } 19 | 20 | resource "aws_elastic_beanstalk_application" "beanstalk_application" { 21 | name = "Metabase" 22 | } 23 | 24 | resource "aws_kms_key" "k" { 25 | description = "metabase" 26 | } 27 | 28 | resource "random_string" "rds_password" { 29 | length = 32 30 | special = false 31 | } 32 | 33 | resource "aws_iam_role" "instance_role" { 34 | name = "metabase_instance_role" 35 | 36 | assume_role_policy = <