├── .standard.yml ├── .rspec ├── assets └── syntax_search.gif ├── lib ├── syntax_suggest.rb └── syntax_suggest │ ├── version.rb │ ├── unvisited_lines.rb │ ├── mini_stringio.rb │ ├── ripper_errors.rb │ ├── priority_engulf_queue.rb │ ├── pathname_from_message.rb │ ├── parse_blocks_from_indent_line.rb │ ├── lex_value.rb │ ├── capture │ ├── falling_indent_lines.rb │ └── before_after_keyword_ends.rb │ ├── lex_all.rb │ ├── display_code_with_line_numbers.rb │ ├── display_invalid_blocks.rb │ ├── priority_queue.rb │ ├── code_block.rb │ ├── core_ext.rb │ ├── explain_syntax.rb │ ├── scan_history.rb │ ├── cli.rb │ ├── code_search.rb │ ├── left_right_lex_count.rb │ ├── block_expand.rb │ ├── code_frontier.rb │ ├── api.rb │ ├── code_line.rb │ ├── capture_code_context.rb │ ├── around_block_scan.rb │ └── clean_document.rb ├── exe └── syntax_suggest ├── bin ├── setup ├── bundle ├── rake ├── rspec └── console ├── .github ├── dependabot.yml └── workflows │ ├── check_changelog.yml │ ├── sync-ruby.yml │ └── ci.yml ├── Rakefile ├── .gitignore ├── Gemfile ├── spec ├── unit │ ├── mini_stringio_spec.rb │ ├── lex_all_spec.rb │ ├── capture │ │ ├── falling_indent_lines_spec.rb │ │ └── before_after_keyword_ends_spec.rb │ ├── core_ext_spec.rb │ ├── code_block_spec.rb │ ├── pathname_from_message_spec.rb │ ├── priority_queue_spec.rb │ ├── api_spec.rb │ ├── scan_history_spec.rb │ ├── code_frontier_spec.rb │ ├── display_invalid_blocks_spec.rb │ ├── code_line_spec.rb │ ├── around_block_scan_spec.rb │ ├── block_expand_spec.rb │ ├── capture_code_context_spec.rb │ ├── cli_spec.rb │ ├── explain_syntax_spec.rb │ └── clean_document_spec.rb ├── integration │ ├── exe_cli_spec.rb │ ├── syntax_suggest_spec.rb │ └── ruby_command_line_spec.rb ├── fixtures │ ├── webmock.rb.txt │ ├── this_project_extra_def.rb.txt │ ├── derailed_require_tree.rb.txt │ └── routes.rb.txt └── spec_helper.rb ├── LICENSE.txt ├── syntax_suggest.gemspec ├── CODE_OF_CONDUCT.md └── README.md /.standard.yml: -------------------------------------------------------------------------------- 1 | ruby_version: 3.0.0 2 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --require spec_helper 3 | -------------------------------------------------------------------------------- /assets/syntax_search.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruby/syntax_suggest/HEAD/assets/syntax_search.gif -------------------------------------------------------------------------------- /lib/syntax_suggest.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "syntax_suggest/core_ext" 4 | -------------------------------------------------------------------------------- /lib/syntax_suggest/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | VERSION = "2.0.2" 5 | end 6 | -------------------------------------------------------------------------------- /exe/syntax_suggest: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "syntax_suggest/api" 4 | 5 | SyntaxSuggest::Cli.new( 6 | argv: ARGV 7 | ).call 8 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: 'github-actions' 4 | directory: '/' 5 | schedule: 6 | interval: 'weekly' 7 | labels: 8 | - 'skip changelog' 9 | -------------------------------------------------------------------------------- /bin/bundle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Make sure syntax_suggest default gem is not loaded first 5 | RUBYOPT="${RUBYOPT-} --disable=syntax_suggest" 6 | RUBYOPT="$RUBYOPT" bundle "$@" 7 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | require "rspec/core/rake_task" 5 | 6 | RSpec::Core::RakeTask.new(:spec) 7 | 8 | task default: :spec 9 | task test: :spec 10 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Make sure syntax_suggest default gem is not loaded first 5 | RUBYOPT="${RUBYOPT-} --disable=syntax_suggest" 6 | RUBYOPT="$RUBYOPT" bundle exec rake "$@" 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | scratch.rb 10 | /*.lock 11 | 12 | # rspec failure tracking 13 | .rspec_status 14 | .DS_Store 15 | -------------------------------------------------------------------------------- /bin/rspec: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # Make sure syntax_suggest default gem is not loaded first 5 | RUBYOPT="${RUBYOPT-} --disable=syntax_suggest" 6 | RUBYOPT="$RUBYOPT" bundle exec rspec "$@" 7 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | # Specify your gem's dependencies in dead_end.gemspec 6 | gemspec 7 | 8 | gem "rake", "~> 12.0" 9 | gem "rspec", "~> 3.0" 10 | gem "stackprof" 11 | gem "standard" 12 | gem "ruby-prof" 13 | 14 | gem "benchmark-ips" 15 | gem "prism" 16 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "syntax_suggest" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start(__FILE__) 15 | -------------------------------------------------------------------------------- /spec/unit/mini_stringio_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe "MiniStringIO" do 7 | it "#puts with no inputs" do 8 | io = MiniStringIO.new 9 | io.puts 10 | expect(io.string).to eq($/) 11 | end 12 | 13 | it "#puts with an input" do 14 | io = MiniStringIO.new 15 | io.puts "Hello" 16 | expect(io.string).to eq(["Hello", $/].join) 17 | end 18 | 19 | it "#puts with an input with a newline" do 20 | io = MiniStringIO.new 21 | io.puts "Hello\n" 22 | expect(io.string).to eq(["Hello\n", $/].join) 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/integration/exe_cli_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe "exe" do 7 | def exe_path 8 | if ruby_core? 9 | root_dir.join("../libexec").join("syntax_suggest") 10 | else 11 | root_dir.join("exe").join("syntax_suggest") 12 | end 13 | end 14 | 15 | def exe(cmd) 16 | ruby = ENV.fetch("RUBY", "ruby") 17 | out = run!("#{ruby} #{exe_path} #{cmd}", raise_on_nonzero_exit: false) 18 | puts out if ENV["SYNTAX_SUGGEST_DEBUG"] 19 | out 20 | end 21 | 22 | it "prints the version" do 23 | out = exe("-v") 24 | expect(out.strip).to include(SyntaxSuggest::VERSION) 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /spec/unit/lex_all_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe "EndBlockParse" do 7 | it "finds blocks based on `end` keyword" do 8 | source = <<~EOM 9 | describe "cat" # 1 10 | Cat.call do # 2 11 | end # 3 12 | end # 4 13 | # 5 14 | it "dog" do # 6 15 | Dog.call do # 7 16 | end # 8 17 | end # 9 18 | EOM 19 | 20 | lex = LexAll.new(source: source) 21 | expect(lex.map(&:token).to_s).to include("dog") 22 | expect(lex.first.line).to eq(1) 23 | expect(lex.last.line).to eq(9) 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /.github/workflows/check_changelog.yml: -------------------------------------------------------------------------------- 1 | name: Check Changelog 2 | 3 | on: 4 | pull_request: 5 | types: [opened, reopened, edited, labeled, unlabeled, synchronize] 6 | 7 | jobs: 8 | check-changelog: 9 | runs-on: ubuntu-latest 10 | if: | 11 | !contains(github.event.pull_request.body, '[skip changelog]') && 12 | !contains(github.event.pull_request.body, '[changelog skip]') && 13 | !contains(github.event.pull_request.body, '[skip ci]') && 14 | !contains(github.event.pull_request.labels.*.name, 'skip changelog') 15 | steps: 16 | - uses: actions/checkout@v6 17 | - name: Check that CHANGELOG is touched 18 | run: | 19 | git fetch origin ${{ github.base_ref }} --depth 1 && \ 20 | git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md 21 | -------------------------------------------------------------------------------- /spec/fixtures/webmock.rb.txt: -------------------------------------------------------------------------------- 1 | describe "webmock tests" do 2 | before(:each) do 3 | WebMock.enable! 4 | end 5 | 6 | after(:each) do 7 | WebMock.disable! 8 | end 9 | 10 | it "port" do 11 | port = rand(1000...9999) 12 | stub_request(:any, "localhost:#{port}") 13 | 14 | query = Cutlass::FunctionQuery.new( 15 | port: port 16 | ).call 17 | 18 | expect(WebMock).to have_requested(:post, "localhost:#{port}"). 19 | with(body: "{}") 20 | end 21 | 22 | it "body" do 23 | body = { lol: "hi" } 24 | port = 8080 25 | stub_request(:any, "localhost:#{port}") 26 | 27 | query = Cutlass::FunctionQuery.new( 28 | port: port 29 | body: body 30 | ).call 31 | 32 | expect(WebMock).to have_requested(:post, "localhost:#{port}"). 33 | with(body: body.to_json) 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/syntax_suggest/unvisited_lines.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Tracks which lines various code blocks have expanded to 5 | # and which are still unexplored 6 | class UnvisitedLines 7 | def initialize(code_lines:) 8 | @unvisited = code_lines.sort_by(&:indent_index) 9 | @visited_lines = {} 10 | @visited_lines.compare_by_identity 11 | end 12 | 13 | def empty? 14 | @unvisited.empty? 15 | end 16 | 17 | def peek 18 | @unvisited.last 19 | end 20 | 21 | def pop 22 | @unvisited.pop 23 | end 24 | 25 | def visit_block(block) 26 | block.lines.each do |line| 27 | next if @visited_lines[line] 28 | @visited_lines[line] = true 29 | end 30 | 31 | while @visited_lines[@unvisited.last] 32 | @unvisited.pop 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/syntax_suggest/mini_stringio.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Mini String IO [Private] 5 | # 6 | # Acts like a StringIO with reduced API, but without having to require that 7 | # class. 8 | # 9 | # The original codebase emitted directly to $stderr, but now SyntaxError#detailed_message 10 | # needs a string output. To accomplish that we kept the original print infrastructure in place and 11 | # added this class to accumulate the print output into a string. 12 | class MiniStringIO 13 | EMPTY_ARG = Object.new 14 | 15 | def initialize(isatty: $stderr.isatty) 16 | @string = +"" 17 | @isatty = isatty 18 | end 19 | 20 | attr_reader :isatty 21 | def puts(value = EMPTY_ARG, **) 22 | if !value.equal?(EMPTY_ARG) 23 | @string << value 24 | end 25 | @string << $/ 26 | end 27 | 28 | attr_reader :string 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /spec/unit/capture/falling_indent_lines_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe Capture::FallingIndentLines do 7 | it "on_falling_indent" do 8 | source = <<~EOM 9 | class OH 10 | def lol 11 | print 'lol 12 | end 13 | 14 | def hello 15 | it "foo" do 16 | end 17 | 18 | def yolo 19 | print 'haha' 20 | end 21 | end 22 | EOM 23 | 24 | code_lines = CleanDocument.new(source: source).call.lines 25 | block = CodeBlock.new(lines: code_lines[6]) 26 | 27 | lines = [] 28 | Capture::FallingIndentLines.new( 29 | block: block, 30 | code_lines: code_lines 31 | ).call do |line| 32 | lines << line 33 | end 34 | lines.sort! 35 | 36 | expect(lines.join).to eq(<<~EOM) 37 | class OH 38 | def hello 39 | end 40 | end 41 | EOM 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/unit/capture/before_after_keyword_ends_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe Capture::BeforeAfterKeywordEnds do 7 | it "before after keyword ends" do 8 | source = <<~EOM 9 | def nope 10 | print 'not me' 11 | end 12 | 13 | def lol 14 | print 'lol' 15 | end 16 | 17 | def hello # 8 18 | 19 | def yolo 20 | print 'haha' 21 | end 22 | 23 | def nada 24 | print 'nope' 25 | end 26 | EOM 27 | 28 | code_lines = CleanDocument.new(source: source).call.lines 29 | block = CodeBlock.new(lines: code_lines[8]) 30 | 31 | expect(block.to_s).to include("def hello") 32 | 33 | lines = Capture::BeforeAfterKeywordEnds.new( 34 | block: block, 35 | code_lines: code_lines 36 | ).call 37 | lines.sort! 38 | 39 | expect(lines.join).to include(<<~EOM) 40 | def lol 41 | end 42 | def yolo 43 | end 44 | EOM 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /.github/workflows/sync-ruby.yml: -------------------------------------------------------------------------------- 1 | name: Sync ruby 2 | on: 3 | push: 4 | branches: [main] 5 | jobs: 6 | sync: 7 | name: Sync ruby 8 | runs-on: ubuntu-latest 9 | if: ${{ github.repository_owner == 'ruby' }} 10 | steps: 11 | - uses: actions/checkout@v6 12 | 13 | - name: Create GitHub App token 14 | id: app-token 15 | uses: actions/create-github-app-token@v2 16 | with: 17 | app-id: 2060836 18 | private-key: ${{ secrets.RUBY_SYNC_DEFAULT_GEMS_PRIVATE_KEY }} 19 | owner: ruby 20 | repositories: ruby 21 | 22 | - name: Sync to ruby/ruby 23 | uses: convictional/trigger-workflow-and-wait@v1.6.5 24 | with: 25 | owner: ruby 26 | repo: ruby 27 | workflow_file_name: sync_default_gems.yml 28 | github_token: ${{ steps.app-token.outputs.token }} 29 | ref: master 30 | client_payload: | 31 | {"gem":"${{ github.event.repository.name }}","before":"${{ github.event.before }}","after":"${{ github.event.after }}"} 32 | propagate_failure: true 33 | wait_interval: 10 34 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 schneems 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/syntax_suggest/ripper_errors.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Capture parse errors from Ripper 5 | # 6 | # Prism returns the errors with their messages, but Ripper 7 | # does not. To get them we must make a custom subclass. 8 | # 9 | # Example: 10 | # 11 | # puts RipperErrors.new(" def foo").call.errors 12 | # # => ["syntax error, unexpected end-of-input, expecting ';' or '\\n'"] 13 | class RipperErrors < Ripper 14 | attr_reader :errors 15 | 16 | # Comes from ripper, called 17 | # on every parse error, msg 18 | # is a string 19 | def on_parse_error(msg) 20 | @errors ||= [] 21 | @errors << msg 22 | end 23 | 24 | alias_method :on_alias_error, :on_parse_error 25 | alias_method :on_assign_error, :on_parse_error 26 | alias_method :on_class_name_error, :on_parse_error 27 | alias_method :on_param_error, :on_parse_error 28 | alias_method :compile_error, :on_parse_error 29 | 30 | def call 31 | @run_once ||= begin 32 | @errors = [] 33 | parse 34 | true 35 | end 36 | self 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /spec/unit/core_ext_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative "../spec_helper" 2 | 3 | module SyntaxSuggest 4 | RSpec.describe "Core extension" do 5 | it "SyntaxError monkepatch ensures there is a newline to the end of the file" do 6 | skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") 7 | 8 | Dir.mktmpdir do |dir| 9 | tmpdir = Pathname(dir) 10 | file = tmpdir.join("file.rb") 11 | file.write(<<~EOM.strip) 12 | print 'no newline 13 | EOM 14 | 15 | core_ext_file = lib_dir.join("syntax_suggest").join("core_ext") 16 | require_relative core_ext_file 17 | 18 | original_message = "blerg" 19 | error = SyntaxError.new(original_message) 20 | def error.set_tmp_path_for_testing=(path) 21 | @tmp_path_for_testing = path 22 | end 23 | error.set_tmp_path_for_testing = file 24 | def error.path 25 | @tmp_path_for_testing 26 | end 27 | 28 | detailed = error.detailed_message(highlight: false, syntax_suggest: true) 29 | expect(detailed).to include("'no newline\n#{original_message}") 30 | expect(detailed).to_not include("print 'no newline#{original_message}") 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /syntax_suggest.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | begin 4 | require_relative "lib/syntax_suggest/version" 5 | rescue LoadError # Fallback to load version file in ruby core repository 6 | require_relative "version" 7 | end 8 | 9 | Gem::Specification.new do |spec| 10 | spec.name = "syntax_suggest" 11 | spec.version = SyntaxSuggest::VERSION 12 | spec.authors = ["schneems"] 13 | spec.email = ["richard.schneeman+foo@gmail.com"] 14 | 15 | spec.summary = "Find syntax errors in your source in a snap" 16 | spec.description = 'When you get an "unexpected end" in your syntax this gem helps you find it' 17 | spec.homepage = "https://github.com/ruby/syntax_suggest.git" 18 | spec.license = "MIT" 19 | spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0") 20 | 21 | spec.metadata["homepage_uri"] = spec.homepage 22 | spec.metadata["source_code_uri"] = "https://github.com/ruby/syntax_suggest.git" 23 | 24 | # Specify which files should be added to the gem when it is released. 25 | # The `git ls-files -z` loads the files in the RubyGem that have been added into git. 26 | spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do 27 | `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|assets)/}) } 28 | end 29 | spec.bindir = "exe" 30 | spec.executables = ["syntax_suggest"] 31 | spec.require_paths = ["lib"] 32 | end 33 | -------------------------------------------------------------------------------- /lib/syntax_suggest/priority_engulf_queue.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Keeps track of what elements are in the queue in 5 | # priority and also ensures that when one element 6 | # engulfs/covers/eats another that the larger element 7 | # evicts the smaller element 8 | class PriorityEngulfQueue 9 | def initialize 10 | @queue = PriorityQueue.new 11 | end 12 | 13 | def to_a 14 | @queue.to_a 15 | end 16 | 17 | def empty? 18 | @queue.empty? 19 | end 20 | 21 | def length 22 | @queue.length 23 | end 24 | 25 | def peek 26 | @queue.peek 27 | end 28 | 29 | def pop 30 | @queue.pop 31 | end 32 | 33 | def push(block) 34 | prune_engulf(block) 35 | @queue << block 36 | flush_deleted 37 | 38 | self 39 | end 40 | 41 | private def flush_deleted 42 | while @queue&.peek&.deleted? 43 | @queue.pop 44 | end 45 | end 46 | 47 | private def prune_engulf(block) 48 | # If we're about to pop off the same block, we can skip deleting 49 | # things from the frontier this iteration since we'll get it 50 | # on the next iteration 51 | return if @queue.peek && (block <=> @queue.peek) == 1 52 | 53 | if block.starts_at != block.ends_at # A block of size 1 cannot engulf another 54 | @queue.to_a.each { |b| 55 | if b.starts_at >= block.starts_at && b.ends_at <= block.ends_at 56 | b.delete 57 | true 58 | end 59 | } 60 | end 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/syntax_suggest/pathname_from_message.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Converts a SyntaxError message to a path 5 | # 6 | # Handles the case where the filename has a colon in it 7 | # such as on a windows file system: https://github.com/ruby/syntax_suggest/issues/111 8 | # 9 | # Example: 10 | # 11 | # message = "/tmp/scratch:2:in `require_relative': /private/tmp/bad.rb:1: syntax error, unexpected `end' (SyntaxError)" 12 | # puts PathnameFromMessage.new(message).call.name 13 | # # => "/tmp/scratch.rb" 14 | # 15 | class PathnameFromMessage 16 | EVAL_RE = /^\(eval.*\):\d+/ 17 | STREAMING_RE = /^-:\d+/ 18 | attr_reader :name 19 | 20 | def initialize(message, io: $stderr) 21 | @line = message.lines.first 22 | @parts = @line.split(":") 23 | @guess = [] 24 | @name = nil 25 | @io = io 26 | end 27 | 28 | def call 29 | if skip_missing_file_name? 30 | if ENV["SYNTAX_SUGGEST_DEBUG"] 31 | @io.puts "SyntaxSuggest: Could not find filename from #{@line.inspect}" 32 | end 33 | else 34 | until stop? 35 | @guess << @parts.shift 36 | @name = Pathname(@guess.join(":")) 37 | end 38 | 39 | if @parts.empty? 40 | @io.puts "SyntaxSuggest: Could not find filename from #{@line.inspect}" 41 | @name = nil 42 | end 43 | end 44 | 45 | self 46 | end 47 | 48 | def stop? 49 | return true if @parts.empty? 50 | return false if @guess.empty? 51 | 52 | @name&.exist? 53 | end 54 | 55 | def skip_missing_file_name? 56 | @line.match?(EVAL_RE) || @line.match?(STREAMING_RE) 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/syntax_suggest/parse_blocks_from_indent_line.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # This class is responsible for generating initial code blocks 5 | # that will then later be expanded. 6 | # 7 | # The biggest concern when guessing code blocks, is accidentally 8 | # grabbing one that contains only an "end". In this example: 9 | # 10 | # def dog 11 | # begonn # misspelled `begin` 12 | # puts "bark" 13 | # end 14 | # end 15 | # 16 | # The following lines would be matched (from bottom to top): 17 | # 18 | # 1) end 19 | # 20 | # 2) puts "bark" 21 | # end 22 | # 23 | # 3) begonn 24 | # puts "bark" 25 | # end 26 | # 27 | # At this point it has no where else to expand, and it will yield this inner 28 | # code as a block 29 | class ParseBlocksFromIndentLine 30 | attr_reader :code_lines 31 | 32 | def initialize(code_lines:) 33 | @code_lines = code_lines 34 | end 35 | 36 | # Builds blocks from bottom up 37 | def each_neighbor_block(target_line) 38 | scan = AroundBlockScan.new(code_lines: code_lines, block: CodeBlock.new(lines: target_line)) 39 | .force_add_empty 40 | .force_add_hidden 41 | .scan_while { |line| line.indent >= target_line.indent } 42 | 43 | neighbors = scan.code_block.lines 44 | 45 | block = CodeBlock.new(lines: neighbors) 46 | if neighbors.length <= 2 || block.valid? 47 | yield block 48 | else 49 | until neighbors.empty? 50 | lines = [neighbors.pop] 51 | while (block = CodeBlock.new(lines: lines)) && block.invalid? && neighbors.any? 52 | lines.prepend neighbors.pop 53 | end 54 | 55 | yield block if block 56 | end 57 | end 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | lint: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout code 12 | uses: actions/checkout@v6 13 | - name: Set up Ruby 14 | uses: ruby/setup-ruby@v1 15 | with: 16 | ruby-version: ruby 17 | bundler-cache: true 18 | - name: Linting 19 | run: bundle exec standardrb 20 | env: 21 | RUBYOPT: --disable=syntax_suggest 22 | 23 | ruby-versions: 24 | uses: ruby/actions/.github/workflows/ruby_versions.yml@master 25 | with: 26 | engine: cruby 27 | 28 | test: 29 | needs: ruby-versions 30 | runs-on: ubuntu-latest 31 | strategy: 32 | fail-fast: false 33 | matrix: 34 | ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }} 35 | steps: 36 | - name: Checkout code 37 | uses: actions/checkout@v6 38 | - name: Set up Ruby 39 | uses: ruby/setup-ruby@v1 40 | with: 41 | ruby-version: ${{ matrix.ruby }} 42 | bundler-cache: true 43 | - name: test 44 | run: bin/rake test 45 | continue-on-error: ${{ matrix.ruby == 'head' }} 46 | 47 | test-disable-prism: 48 | needs: ruby-versions 49 | runs-on: ubuntu-latest 50 | strategy: 51 | fail-fast: false 52 | matrix: 53 | ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }} 54 | steps: 55 | - name: Checkout code 56 | uses: actions/checkout@v6 57 | - name: Set up Ruby 58 | uses: ruby/setup-ruby@v1 59 | with: 60 | ruby-version: ${{ matrix.ruby }} 61 | bundler-cache: true 62 | - name: test 63 | run: SYNTAX_SUGGEST_DISABLE_PRISM=1 bin/rake test 64 | continue-on-error: ${{ matrix.ruby == 'head' }} 65 | -------------------------------------------------------------------------------- /lib/syntax_suggest/lex_value.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Value object for accessing lex values 5 | # 6 | # This lex: 7 | # 8 | # [1, 0], :on_ident, "describe", CMDARG 9 | # 10 | # Would translate into: 11 | # 12 | # lex.line # => 1 13 | # lex.type # => :on_indent 14 | # lex.token # => "describe" 15 | class LexValue 16 | attr_reader :line, :type, :token, :state 17 | 18 | def initialize(line, type, token, state, last_lex = nil) 19 | @line = line 20 | @type = type 21 | @token = token 22 | @state = state 23 | 24 | set_kw_end(last_lex) 25 | end 26 | 27 | private def set_kw_end(last_lex) 28 | @is_end = false 29 | @is_kw = false 30 | return if type != :on_kw 31 | 32 | return if last_lex && last_lex.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953 33 | 34 | case token 35 | when "if", "unless", "while", "until" 36 | # Only count if/unless when it's not a "trailing" if/unless 37 | # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375 38 | @is_kw = true unless expr_label? 39 | when "def", "case", "for", "begin", "class", "module", "do" 40 | @is_kw = true 41 | when "end" 42 | @is_end = true 43 | end 44 | end 45 | 46 | def fname? 47 | state.allbits?(Ripper::EXPR_FNAME) 48 | end 49 | 50 | def ignore_newline? 51 | type == :on_ignored_nl 52 | end 53 | 54 | def is_end? 55 | @is_end 56 | end 57 | 58 | def is_kw? 59 | @is_kw 60 | end 61 | 62 | def expr_beg? 63 | state.anybits?(Ripper::EXPR_BEG) 64 | end 65 | 66 | def expr_label? 67 | state.allbits?(Ripper::EXPR_LABEL) 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/fixtures/this_project_extra_def.rb.txt: -------------------------------------------------------------------------------- 1 | module SyntaxErrorSearch 2 | # Used for formatting invalid blocks 3 | class DisplayInvalidBlocks 4 | attr_reader :filename 5 | 6 | def initialize(block_array, io: $stderr, filename: nil) 7 | @filename = filename 8 | @io = io 9 | @blocks = block_array 10 | @lines = @blocks.map(&:lines).flatten 11 | @digit_count = @lines.last.line_number.to_s.length 12 | @code_lines = @blocks.first.code_lines 13 | 14 | @invalid_line_hash = @lines.each_with_object({}) {|line, h| h[line] = true} 15 | end 16 | 17 | def call 18 | @io.puts <<~EOM 19 | 20 | SyntaxSuggest: A syntax error was detected 21 | 22 | This code has an unmatched `end` this is caused by either 23 | missing a syntax keyword (`def`, `do`, etc.) or inclusion 24 | of an extra `end` line: 25 | EOM 26 | 27 | @io.puts(<<~EOM) if filename 28 | file: #{filename} 29 | EOM 30 | 31 | @io.puts <<~EOM 32 | #{code_with_filename} 33 | EOM 34 | end 35 | 36 | def filename 37 | 38 | def code_with_filename 39 | string = String.new("") 40 | string << "```\n" 41 | string << "#".rjust(@digit_count) + " filename: #{filename}\n\n" if filename 42 | string << code_with_lines 43 | string << "```\n" 44 | string 45 | end 46 | 47 | def code_with_lines 48 | @code_lines.map do |line| 49 | next if line.hidden? 50 | number = line.line_number.to_s.rjust(@digit_count) 51 | if line.empty? 52 | "#{number.to_s}#{line}" 53 | else 54 | string = String.new 55 | string << "\e[1;3m" if @invalid_line_hash[line] # Bold, italics 56 | string << "#{number.to_s} " 57 | string << line.to_s 58 | string << "\e[0m" 59 | string 60 | end 61 | end.join 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /lib/syntax_suggest/capture/falling_indent_lines.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | module Capture 5 | # Shows the context around code provided by "falling" indentation 6 | # 7 | # If this is the original code lines: 8 | # 9 | # class OH 10 | # def hello 11 | # it "foo" do 12 | # end 13 | # end 14 | # 15 | # And this is the line that is captured 16 | # 17 | # it "foo" do 18 | # 19 | # It will yield its surrounding context: 20 | # 21 | # class OH 22 | # def hello 23 | # end 24 | # end 25 | # 26 | # Example: 27 | # 28 | # FallingIndentLines.new( 29 | # block: block, 30 | # code_lines: @code_lines 31 | # ).call do |line| 32 | # @lines_to_output << line 33 | # end 34 | # 35 | class FallingIndentLines 36 | def initialize(code_lines:, block:) 37 | @lines = nil 38 | @scanner = ScanHistory.new(code_lines: code_lines, block: block) 39 | @original_indent = block.current_indent 40 | end 41 | 42 | def call(&yieldable) 43 | last_indent_up = @original_indent 44 | last_indent_down = @original_indent 45 | 46 | @scanner.commit_if_changed 47 | @scanner.scan( 48 | up: ->(line, _, _) { 49 | next true if line.empty? 50 | 51 | if line.indent < last_indent_up 52 | yieldable.call(line) 53 | last_indent_up = line.indent 54 | end 55 | true 56 | }, 57 | down: ->(line, _, _) { 58 | next true if line.empty? 59 | 60 | if line.indent < last_indent_down 61 | yieldable.call(line) 62 | last_indent_down = line.indent 63 | end 64 | true 65 | } 66 | ) 67 | @scanner.stash_changes 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /lib/syntax_suggest/lex_all.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Ripper.lex is not guaranteed to lex the entire source document 5 | # 6 | # This class guarantees the whole document is lex-ed by iteratively 7 | # lexing the document where ripper stopped. 8 | # 9 | # Prism likely doesn't have the same problem. Once ripper support is removed 10 | # we can likely reduce the complexity here if not remove the whole concept. 11 | # 12 | # Example usage: 13 | # 14 | # lex = LexAll.new(source: source) 15 | # lex.each do |value| 16 | # puts value.line 17 | # end 18 | class LexAll 19 | include Enumerable 20 | 21 | def initialize(source:, source_lines: nil) 22 | @lex = self.class.lex(source, 1) 23 | lineno = @lex.last[0][0] + 1 24 | source_lines ||= source.lines 25 | last_lineno = source_lines.length 26 | 27 | until lineno >= last_lineno 28 | lines = source_lines[lineno..] 29 | 30 | @lex.concat( 31 | self.class.lex(lines.join, lineno + 1) 32 | ) 33 | 34 | lineno = @lex.last[0].first + 1 35 | end 36 | 37 | last_lex = nil 38 | @lex.map! { |elem| 39 | last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex) 40 | } 41 | end 42 | 43 | if SyntaxSuggest.use_prism_parser? 44 | def self.lex(source, line_number) 45 | Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] } 46 | end 47 | else 48 | def self.lex(source, line_number) 49 | Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos) 50 | end 51 | end 52 | 53 | def to_a 54 | @lex 55 | end 56 | 57 | def each 58 | return @lex.each unless block_given? 59 | @lex.each do |x| 60 | yield x 61 | end 62 | end 63 | 64 | def [](index) 65 | @lex[index] 66 | end 67 | 68 | def last 69 | @lex.last 70 | end 71 | end 72 | end 73 | 74 | require_relative "lex_value" 75 | -------------------------------------------------------------------------------- /spec/fixtures/derailed_require_tree.rb.txt: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Tree structure used to store and sort require memory costs 4 | # RequireTree.new('get_process_mem') 5 | module DerailedBenchmarks 6 | class RequireTree 7 | REQUIRED_BY = {} 8 | 9 | attr_reader :name 10 | attr_writer :cost 11 | attr_accessor :parent 12 | 13 | def initialize(name) 14 | @name = name 15 | @children = {} 16 | @cost = 0 17 | 18 | def self.reset! 19 | REQUIRED_BY.clear 20 | if defined?(Kernel::REQUIRE_STACK) 21 | Kernel::REQUIRE_STACK.clear 22 | 23 | Kernel::REQUIRE_STACK.push(TOP_REQUIRE) 24 | end 25 | end 26 | 27 | def <<(tree) 28 | @children[tree.name.to_s] = tree 29 | tree.parent = self 30 | (REQUIRED_BY[tree.name.to_s] ||= []) << self.name 31 | end 32 | 33 | def [](name) 34 | @children[name.to_s] 35 | end 36 | 37 | # Returns array of child nodes 38 | def children 39 | @children.values 40 | end 41 | 42 | def cost 43 | @cost || 0 44 | end 45 | 46 | # Returns sorted array of child nodes from Largest to Smallest 47 | def sorted_children 48 | children.sort { |c1, c2| c2.cost <=> c1.cost } 49 | end 50 | 51 | def to_string 52 | str = String.new("#{name}: #{cost.round(4)} MiB") 53 | if parent && REQUIRED_BY[self.name.to_s] 54 | names = REQUIRED_BY[self.name.to_s].uniq - [parent.name.to_s] 55 | if names.any? 56 | str << " (Also required by: #{ names.first(2).join(", ") }" 57 | str << ", and #{names.count - 2} others" if names.count > 3 58 | str << ")" 59 | end 60 | end 61 | str 62 | end 63 | 64 | # Recursively prints all child nodes 65 | def print_sorted_children(level = 0, out = STDOUT) 66 | return if cost < ENV['CUT_OFF'].to_f 67 | out.puts " " * level + self.to_string 68 | level += 1 69 | sorted_children.each do |child| 70 | child.print_sorted_children(level, out) 71 | end 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /lib/syntax_suggest/display_code_with_line_numbers.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Outputs code with highlighted lines 5 | # 6 | # Whatever is passed to this class will be rendered 7 | # even if it is "marked invisible" any filtering of 8 | # output should be done before calling this class. 9 | # 10 | # DisplayCodeWithLineNumbers.new( 11 | # lines: lines, 12 | # highlight_lines: [lines[2], lines[3]] 13 | # ).call 14 | # # => 15 | # 1 16 | # 2 def cat 17 | # > 3 Dir.chdir 18 | # > 4 end 19 | # 5 end 20 | # 6 21 | class DisplayCodeWithLineNumbers 22 | TERMINAL_HIGHLIGHT = "\e[1;3m" # Bold, italics 23 | TERMINAL_END = "\e[0m" 24 | 25 | def initialize(lines:, highlight_lines: [], terminal: false) 26 | @lines = Array(lines).sort 27 | @terminal = terminal 28 | @highlight_line_hash = Array(highlight_lines).each_with_object({}) { |line, h| h[line] = true } 29 | @digit_count = @lines.last&.line_number.to_s.length 30 | end 31 | 32 | def call 33 | @lines.map do |line| 34 | format_line(line) 35 | end.join 36 | end 37 | 38 | private def format_line(code_line) 39 | # Handle trailing slash lines 40 | code_line.original.lines.map.with_index do |contents, i| 41 | format( 42 | empty: code_line.empty?, 43 | number: (code_line.number + i).to_s, 44 | contents: contents, 45 | highlight: @highlight_line_hash[code_line] 46 | ) 47 | end.join 48 | end 49 | 50 | private def format(contents:, number:, empty:, highlight: false) 51 | string = +"" 52 | string << if highlight 53 | "> " 54 | else 55 | " " 56 | end 57 | 58 | string << number.rjust(@digit_count).to_s 59 | if empty 60 | string << contents 61 | else 62 | string << " " 63 | string << TERMINAL_HIGHLIGHT if @terminal && highlight 64 | string << contents 65 | string << TERMINAL_END if @terminal 66 | end 67 | string 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/syntax_suggest/display_invalid_blocks.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "capture_code_context" 4 | require_relative "display_code_with_line_numbers" 5 | 6 | module SyntaxSuggest 7 | # Used for formatting invalid blocks 8 | class DisplayInvalidBlocks 9 | attr_reader :filename 10 | 11 | def initialize(code_lines:, blocks:, io: $stderr, filename: nil, terminal: DEFAULT_VALUE) 12 | @io = io 13 | @blocks = Array(blocks) 14 | @filename = filename 15 | @code_lines = code_lines 16 | 17 | @terminal = (terminal == DEFAULT_VALUE) ? io.isatty : terminal 18 | end 19 | 20 | def document_ok? 21 | @blocks.none? { |b| !b.hidden? } 22 | end 23 | 24 | def call 25 | if document_ok? 26 | return self 27 | end 28 | 29 | if filename 30 | @io.puts("--> #{filename}") 31 | @io.puts 32 | end 33 | @blocks.each do |block| 34 | display_block(block) 35 | end 36 | 37 | self 38 | end 39 | 40 | private def display_block(block) 41 | # Build explanation 42 | explain = ExplainSyntax.new( 43 | code_lines: block.lines 44 | ).call 45 | 46 | # Enhance code output 47 | # Also handles several ambiguious cases 48 | lines = CaptureCodeContext.new( 49 | blocks: block, 50 | code_lines: @code_lines 51 | ).call 52 | 53 | # Build code output 54 | document = DisplayCodeWithLineNumbers.new( 55 | lines: lines, 56 | terminal: @terminal, 57 | highlight_lines: block.lines 58 | ).call 59 | 60 | # Output syntax error explanation 61 | explain.errors.each do |e| 62 | @io.puts e 63 | end 64 | @io.puts 65 | 66 | # Output code 67 | @io.puts(document) 68 | end 69 | 70 | private def code_with_context 71 | lines = CaptureCodeContext.new( 72 | blocks: @blocks, 73 | code_lines: @code_lines 74 | ).call 75 | 76 | DisplayCodeWithLineNumbers.new( 77 | lines: lines, 78 | terminal: @terminal, 79 | highlight_lines: @invalid_lines 80 | ).call 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /spec/unit/code_block_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe CodeBlock do 7 | it "can detect if it's valid or not" do 8 | code_lines = code_line_array(<<~EOM) 9 | def foo 10 | puts 'lol' 11 | end 12 | EOM 13 | 14 | block = CodeBlock.new(lines: code_lines[1]) 15 | expect(block.valid?).to be_truthy 16 | end 17 | 18 | it "can be sorted in indentation order" do 19 | code_lines = code_line_array(<<~EOM) 20 | def foo 21 | puts 'lol' 22 | end 23 | EOM 24 | 25 | block_0 = CodeBlock.new(lines: code_lines[0]) 26 | block_1 = CodeBlock.new(lines: code_lines[1]) 27 | block_2 = CodeBlock.new(lines: code_lines[2]) 28 | 29 | expect(block_0 <=> block_0.dup).to eq(0) 30 | expect(block_1 <=> block_0).to eq(1) 31 | expect(block_1 <=> block_2).to eq(-1) 32 | 33 | array = [block_2, block_1, block_0].sort 34 | expect(array.last).to eq(block_2) 35 | 36 | block = CodeBlock.new(lines: CodeLine.new(line: " " * 8 + "foo", index: 4, lex: [])) 37 | array.prepend(block) 38 | expect(array.max).to eq(block) 39 | end 40 | 41 | it "knows it's current indentation level" do 42 | code_lines = code_line_array(<<~EOM) 43 | def foo 44 | puts 'lol' 45 | end 46 | EOM 47 | 48 | block = CodeBlock.new(lines: code_lines[1]) 49 | expect(block.current_indent).to eq(2) 50 | 51 | block = CodeBlock.new(lines: code_lines[0]) 52 | expect(block.current_indent).to eq(0) 53 | end 54 | 55 | it "knows it's current indentation level when mismatched indents" do 56 | code_lines = code_line_array(<<~EOM) 57 | def foo 58 | puts 'lol' 59 | end 60 | EOM 61 | 62 | block = CodeBlock.new(lines: [code_lines[1], code_lines[2]]) 63 | expect(block.current_indent).to eq(1) 64 | end 65 | 66 | it "before lines and after lines" do 67 | code_lines = code_line_array(<<~EOM) 68 | def foo 69 | bar; end 70 | end 71 | EOM 72 | 73 | block = CodeBlock.new(lines: code_lines[1]) 74 | expect(block.valid?).to be_falsey 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /spec/unit/pathname_from_message_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe "PathnameFromMessage" do 7 | it "handles filenames with colons in them" do 8 | Dir.mktmpdir do |dir| 9 | dir = Pathname(dir) 10 | 11 | file = dir.join("scr:atch.rb").tap { |p| FileUtils.touch(p) } 12 | 13 | message = "#{file}:2:in `require_relative': /private/tmp/bad.rb:1: syntax error, unexpected `end' (SyntaxError)" 14 | file = PathnameFromMessage.new(message).call.name 15 | 16 | expect(file).to be_truthy 17 | end 18 | end 19 | 20 | it "checks if the file exists" do 21 | Dir.mktmpdir do |dir| 22 | dir = Pathname(dir) 23 | 24 | file = dir.join("scratch.rb") 25 | # No touch, file does not exist 26 | expect(file.exist?).to be_falsey 27 | 28 | message = "#{file}:2:in `require_relative': /private/tmp/bad.rb:1: syntax error, unexpected `end' (SyntaxError)" 29 | io = StringIO.new 30 | file = PathnameFromMessage.new(message, io: io).call.name 31 | 32 | expect(io.string).to include(file.to_s) 33 | expect(file).to be_falsey 34 | end 35 | end 36 | 37 | it "does not output error message on syntax error inside of an (eval)" do 38 | message = "(eval):1: invalid multibyte char (UTF-8) (SyntaxError)\n" 39 | io = StringIO.new 40 | file = PathnameFromMessage.new(message, io: io).call.name 41 | 42 | expect(io.string).to eq("") 43 | expect(file).to be_falsey 44 | end 45 | 46 | it "does not output error message on syntax error inside of an (eval at __FILE__:__LINE__)" do 47 | message = "(eval at #{__FILE__}:#{__LINE__}):1: invalid multibyte char (UTF-8) (SyntaxError)\n" 48 | io = StringIO.new 49 | file = PathnameFromMessage.new(message, io: io).call.name 50 | 51 | expect(io.string).to eq("") 52 | expect(file).to be_falsey 53 | end 54 | 55 | it "does not output error message on syntax error inside of streamed code" do 56 | # An example of streamed code is: $ echo "def foo" | ruby 57 | message = "-:1: syntax error, unexpected end-of-input\n" 58 | io = StringIO.new 59 | file = PathnameFromMessage.new(message, io: io).call.name 60 | 61 | expect(io.string).to eq("") 62 | expect(file).to be_falsey 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /lib/syntax_suggest/capture/before_after_keyword_ends.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | module Capture 5 | # Shows surrounding kw/end pairs 6 | # 7 | # The purpose of showing these extra pairs is due to cases 8 | # of ambiguity when only one visible line is matched. 9 | # 10 | # For example: 11 | # 12 | # 1 class Dog 13 | # 2 def bark 14 | # 4 def eat 15 | # 5 end 16 | # 6 end 17 | # 18 | # In this case either line 2 could be missing an `end` or 19 | # line 4 was an extra line added by mistake (it happens). 20 | # 21 | # When we detect the above problem it shows the issue 22 | # as only being on line 2 23 | # 24 | # 2 def bark 25 | # 26 | # Showing "neighbor" keyword pairs gives extra context: 27 | # 28 | # 2 def bark 29 | # 4 def eat 30 | # 5 end 31 | # 32 | # 33 | # Example: 34 | # 35 | # lines = BeforeAfterKeywordEnds.new( 36 | # block: block, 37 | # code_lines: code_lines 38 | # ).call() 39 | # 40 | class BeforeAfterKeywordEnds 41 | def initialize(code_lines:, block:) 42 | @scanner = ScanHistory.new(code_lines: code_lines, block: block) 43 | @original_indent = block.current_indent 44 | end 45 | 46 | def call 47 | lines = [] 48 | 49 | @scanner.scan( 50 | up: ->(line, kw_count, end_count) { 51 | next true if line.empty? 52 | break if line.indent < @original_indent 53 | next true if line.indent != @original_indent 54 | 55 | # If we're going up and have one complete kw/end pair, stop 56 | if kw_count != 0 && kw_count == end_count 57 | lines << line 58 | break 59 | end 60 | 61 | lines << line if line.is_kw? || line.is_end? 62 | true 63 | }, 64 | down: ->(line, kw_count, end_count) { 65 | next true if line.empty? 66 | break if line.indent < @original_indent 67 | next true if line.indent != @original_indent 68 | 69 | # if we're going down and have one complete kw/end pair,stop 70 | if kw_count != 0 && kw_count == end_count 71 | lines << line 72 | break 73 | end 74 | 75 | lines << line if line.is_kw? || line.is_end? 76 | true 77 | } 78 | ) 79 | @scanner.stash_changes 80 | 81 | lines 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/syntax_suggest/priority_queue.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Holds elements in a priority heap on insert 5 | # 6 | # Instead of constantly calling `sort!`, put 7 | # the element where it belongs the first time 8 | # around 9 | # 10 | # Example: 11 | # 12 | # queue = PriorityQueue.new 13 | # queue << 33 14 | # queue << 44 15 | # queue << 1 16 | # 17 | # puts queue.peek # => 44 18 | # 19 | class PriorityQueue 20 | attr_reader :elements 21 | 22 | def initialize 23 | @elements = [] 24 | end 25 | 26 | def <<(element) 27 | @elements << element 28 | bubble_up(last_index, element) 29 | end 30 | 31 | def pop 32 | exchange(0, last_index) 33 | max = @elements.pop 34 | bubble_down(0) 35 | max 36 | end 37 | 38 | def length 39 | @elements.length 40 | end 41 | 42 | def empty? 43 | @elements.empty? 44 | end 45 | 46 | def peek 47 | @elements.first 48 | end 49 | 50 | def to_a 51 | @elements 52 | end 53 | 54 | # Used for testing, extremely not performant 55 | def sorted 56 | out = [] 57 | elements = @elements.dup 58 | while (element = pop) 59 | out << element 60 | end 61 | @elements = elements 62 | out.reverse 63 | end 64 | 65 | private def last_index 66 | @elements.size - 1 67 | end 68 | 69 | private def bubble_up(index, element) 70 | return if index <= 0 71 | 72 | parent_index = (index - 1) / 2 73 | parent = @elements[parent_index] 74 | 75 | return if (parent <=> element) >= 0 76 | 77 | exchange(index, parent_index) 78 | bubble_up(parent_index, element) 79 | end 80 | 81 | private def bubble_down(index) 82 | child_index = (index * 2) + 1 83 | 84 | return if child_index > last_index 85 | 86 | not_the_last_element = child_index < last_index 87 | left_element = @elements[child_index] 88 | right_element = @elements[child_index + 1] 89 | 90 | child_index += 1 if not_the_last_element && (right_element <=> left_element) == 1 91 | 92 | return if (@elements[index] <=> @elements[child_index]) >= 0 93 | 94 | exchange(index, child_index) 95 | bubble_down(child_index) 96 | end 97 | 98 | def exchange(source, target) 99 | a = @elements[source] 100 | b = @elements[target] 101 | @elements[source] = b 102 | @elements[target] = a 103 | end 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/setup" 4 | require "syntax_suggest/api" 5 | 6 | begin 7 | require "benchmark" 8 | rescue LoadError 9 | end 10 | require "tempfile" 11 | 12 | RSpec.configure do |config| 13 | # Enable flags like --only-failures and --next-failure 14 | config.example_status_persistence_file_path = ".rspec_status" 15 | 16 | # Disable RSpec exposing methods globally on `Module` and `main` 17 | config.disable_monkey_patching! 18 | 19 | config.expect_with :rspec do |c| 20 | c.syntax = :expect 21 | end 22 | 23 | if config.color_mode == :automatic 24 | if config.color_enabled? && ((ENV["TERM"] == "dumb") || ENV["NO_COLOR"]&.slice(0)) 25 | config.color_mode = :off 26 | end 27 | end 28 | end 29 | 30 | # Used for debugging modifications to 31 | # display output 32 | def debug_display(output) 33 | return unless ENV["DEBUG_DISPLAY"] 34 | puts 35 | puts output 36 | puts 37 | end 38 | 39 | def spec_dir 40 | Pathname(__dir__) 41 | end 42 | 43 | def lib_dir 44 | if ruby_core? 45 | root_dir.join("../lib") 46 | else 47 | root_dir.join("lib") 48 | end 49 | end 50 | 51 | def root_dir 52 | spec_dir.join("..") 53 | end 54 | 55 | def fixtures_dir 56 | spec_dir.join("fixtures") 57 | end 58 | 59 | def ruby_core? 60 | !root_dir.join("syntax_suggest.gemspec").exist? 61 | end 62 | 63 | def code_line_array(source) 64 | SyntaxSuggest::CleanDocument.new(source: source).call.lines 65 | end 66 | 67 | autoload :RubyProf, "ruby-prof" 68 | 69 | def debug_perf 70 | raise "No block given" unless block_given? 71 | 72 | if ENV["DEBUG_PERF"] 73 | out = nil 74 | result = RubyProf.profile do 75 | out = yield 76 | end 77 | 78 | dir = SyntaxSuggest.record_dir("tmp") 79 | printer = RubyProf::MultiPrinter.new(result, [:flat, :graph, :graph_html, :tree, :call_tree, :stack, :dot]) 80 | printer.print(path: dir, profile: "profile") 81 | 82 | out 83 | else 84 | yield 85 | end 86 | end 87 | 88 | def run!(cmd, raise_on_nonzero_exit: true) 89 | out = `#{cmd} 2>&1` 90 | raise "Command: #{cmd} failed: #{out}" if !$?.success? && raise_on_nonzero_exit 91 | out 92 | end 93 | 94 | # Allows us to write cleaner tests since <<~EOM block quotes 95 | # strip off all leading indentation and we need it to be preserved 96 | # sometimes. 97 | class String 98 | def indent(number) 99 | lines.map do |line| 100 | if line.chomp.empty? 101 | line 102 | else 103 | " " * number + line 104 | end 105 | end.join 106 | end 107 | end 108 | -------------------------------------------------------------------------------- /lib/syntax_suggest/code_block.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Multiple lines form a singular CodeBlock 5 | # 6 | # Source code is made of multiple CodeBlocks. 7 | # 8 | # Example: 9 | # 10 | # code_block.to_s # => 11 | # # def foo 12 | # # puts "foo" 13 | # # end 14 | # 15 | # code_block.valid? # => true 16 | # code_block.in_valid? # => false 17 | # 18 | # 19 | class CodeBlock 20 | UNSET = Object.new.freeze 21 | attr_reader :lines, :starts_at, :ends_at 22 | 23 | def initialize(lines: []) 24 | @lines = Array(lines) 25 | @valid = UNSET 26 | @deleted = false 27 | @starts_at = @lines.first.number 28 | @ends_at = @lines.last.number 29 | end 30 | 31 | def delete 32 | @deleted = true 33 | end 34 | 35 | def deleted? 36 | @deleted 37 | end 38 | 39 | def visible_lines 40 | @lines.select(&:visible?).select(&:not_empty?) 41 | end 42 | 43 | def mark_invisible 44 | @lines.map(&:mark_invisible) 45 | end 46 | 47 | def is_end? 48 | to_s.strip == "end" 49 | end 50 | 51 | def hidden? 52 | @lines.all?(&:hidden?) 53 | end 54 | 55 | # This is used for frontier ordering, we are searching from 56 | # the largest indentation to the smallest. This allows us to 57 | # populate an array with multiple code blocks then call `sort!` 58 | # on it without having to specify the sorting criteria 59 | def <=>(other) 60 | out = current_indent <=> other.current_indent 61 | return out if out != 0 62 | 63 | # Stable sort 64 | starts_at <=> other.starts_at 65 | end 66 | 67 | def current_indent 68 | @current_indent ||= lines.select(&:not_empty?).map(&:indent).min || 0 69 | end 70 | 71 | def invalid? 72 | !valid? 73 | end 74 | 75 | def valid? 76 | if @valid == UNSET 77 | # Performance optimization 78 | # 79 | # If all the lines were previously hidden 80 | # and we expand to capture additional empty 81 | # lines then the result cannot be invalid 82 | # 83 | # That means there's no reason to re-check all 84 | # lines with the parser (which is expensive). 85 | # Benchmark in commit message 86 | @valid = if lines.all? { |l| l.hidden? || l.empty? } 87 | true 88 | else 89 | SyntaxSuggest.valid?(lines.map(&:original).join) 90 | end 91 | else 92 | @valid 93 | end 94 | end 95 | 96 | def to_s 97 | @lines.join 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/syntax_suggest/core_ext.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Ruby 3.2+ has a cleaner way to hook into Ruby that doesn't use `require` 4 | if SyntaxError.method_defined?(:detailed_message) 5 | module SyntaxSuggest 6 | # SyntaxSuggest.module_for_detailed_message [Private] 7 | # 8 | # Used to monkeypatch SyntaxError via Module.prepend 9 | def self.module_for_detailed_message 10 | Module.new { 11 | def detailed_message(highlight: true, syntax_suggest: true, **kwargs) 12 | return super unless syntax_suggest 13 | 14 | require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) 15 | 16 | message = super 17 | 18 | if path 19 | file = Pathname.new(path) 20 | io = SyntaxSuggest::MiniStringIO.new 21 | 22 | SyntaxSuggest.call( 23 | io: io, 24 | source: file.read, 25 | filename: file, 26 | terminal: highlight 27 | ) 28 | annotation = io.string 29 | 30 | annotation += "\n" unless annotation.end_with?("\n") 31 | 32 | annotation + message 33 | else 34 | message 35 | end 36 | rescue => e 37 | if ENV["SYNTAX_SUGGEST_DEBUG"] 38 | $stderr.warn(e.message) 39 | $stderr.warn(e.backtrace) 40 | end 41 | 42 | # Ignore internal errors 43 | message 44 | end 45 | } 46 | end 47 | end 48 | 49 | SyntaxError.prepend(SyntaxSuggest.module_for_detailed_message) 50 | else 51 | autoload :Pathname, "pathname" 52 | 53 | #-- 54 | # Monkey patch kernel to ensure that all `require` calls call the same 55 | # method 56 | #++ 57 | module Kernel 58 | # :stopdoc: 59 | 60 | module_function 61 | 62 | alias_method :syntax_suggest_original_require, :require 63 | alias_method :syntax_suggest_original_require_relative, :require_relative 64 | alias_method :syntax_suggest_original_load, :load 65 | 66 | def load(file, wrap = false) 67 | syntax_suggest_original_load(file) 68 | rescue SyntaxError => e 69 | require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) 70 | 71 | SyntaxSuggest.handle_error(e) 72 | end 73 | 74 | def require(file) 75 | syntax_suggest_original_require(file) 76 | rescue SyntaxError => e 77 | require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) 78 | 79 | SyntaxSuggest.handle_error(e) 80 | end 81 | 82 | def require_relative(file) 83 | if Pathname.new(file).absolute? 84 | syntax_suggest_original_require file 85 | else 86 | relative_from = caller_locations(1..1).first 87 | relative_from_path = relative_from.absolute_path || relative_from.path 88 | syntax_suggest_original_require File.expand_path("../#{file}", relative_from_path) 89 | end 90 | rescue SyntaxError => e 91 | require "syntax_suggest/api" unless defined?(SyntaxSuggest::DEFAULT_VALUE) 92 | 93 | SyntaxSuggest.handle_error(e) 94 | end 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/syntax_suggest/explain_syntax.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "left_right_lex_count" 4 | 5 | if !SyntaxSuggest.use_prism_parser? 6 | require_relative "ripper_errors" 7 | end 8 | 9 | module SyntaxSuggest 10 | class GetParseErrors 11 | def self.errors(source) 12 | if SyntaxSuggest.use_prism_parser? 13 | Prism.parse(source).errors.map(&:message) 14 | else 15 | RipperErrors.new(source).call.errors 16 | end 17 | end 18 | end 19 | 20 | # Explains syntax errors based on their source 21 | # 22 | # example: 23 | # 24 | # source = "def foo; puts 'lol'" # Note missing end 25 | # explain ExplainSyntax.new( 26 | # code_lines: CodeLine.from_source(source) 27 | # ).call 28 | # explain.errors.first 29 | # # => "Unmatched keyword, missing `end' ?" 30 | # 31 | # When the error cannot be determined by lexical counting 32 | # then the parser is run against the input and the raw 33 | # errors are returned. 34 | # 35 | # Example: 36 | # 37 | # source = "1 * " # Note missing a second number 38 | # explain ExplainSyntax.new( 39 | # code_lines: CodeLine.from_source(source) 40 | # ).call 41 | # explain.errors.first 42 | # # => "syntax error, unexpected end-of-input" 43 | class ExplainSyntax 44 | INVERSE = { 45 | "{" => "}", 46 | "}" => "{", 47 | "[" => "]", 48 | "]" => "[", 49 | "(" => ")", 50 | ")" => "(", 51 | "|" => "|" 52 | }.freeze 53 | 54 | def initialize(code_lines:) 55 | @code_lines = code_lines 56 | @left_right = LeftRightLexCount.new 57 | @missing = nil 58 | end 59 | 60 | def call 61 | @code_lines.each do |line| 62 | line.lex.each do |lex| 63 | @left_right.count_lex(lex) 64 | end 65 | end 66 | 67 | self 68 | end 69 | 70 | # Returns an array of missing elements 71 | # 72 | # For example this: 73 | # 74 | # ExplainSyntax.new(code_lines: lines).missing 75 | # # => ["}"] 76 | # 77 | # Would indicate that the source is missing 78 | # a `}` character in the source code 79 | def missing 80 | @missing ||= @left_right.missing 81 | end 82 | 83 | # Converts a missing string to 84 | # an human understandable explanation. 85 | # 86 | # Example: 87 | # 88 | # explain.why("}") 89 | # # => "Unmatched `{', missing `}' ?" 90 | # 91 | def why(miss) 92 | case miss 93 | when "keyword" 94 | "Unmatched `end', missing keyword (`do', `def`, `if`, etc.) ?" 95 | when "end" 96 | "Unmatched keyword, missing `end' ?" 97 | else 98 | inverse = INVERSE.fetch(miss) { 99 | raise "Unknown explain syntax char or key: #{miss.inspect}" 100 | } 101 | "Unmatched `#{inverse}', missing `#{miss}' ?" 102 | end 103 | end 104 | 105 | # Returns an array of syntax error messages 106 | # 107 | # If no missing pairs are found it falls back 108 | # on the original error messages 109 | def errors 110 | if missing.empty? 111 | return GetParseErrors.errors(@code_lines.map(&:original).join).uniq 112 | end 113 | 114 | missing.map { |miss| why(miss) } 115 | end 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /spec/unit/priority_queue_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | class CurrentIndex 7 | attr_reader :current_indent 8 | 9 | def initialize(value) 10 | @current_indent = value 11 | end 12 | 13 | def <=>(other) 14 | @current_indent <=> other.current_indent 15 | end 16 | 17 | def inspect 18 | @current_indent 19 | end 20 | end 21 | 22 | RSpec.describe CodeFrontier do 23 | it "works" do 24 | q = PriorityQueue.new 25 | q << 1 26 | q << 2 27 | expect(q.elements).to eq([2, 1]) 28 | 29 | q << 3 30 | expect(q.elements).to eq([3, 1, 2]) 31 | 32 | expect(q.pop).to eq(3) 33 | expect(q.pop).to eq(2) 34 | expect(q.pop).to eq(1) 35 | expect(q.pop).to eq(nil) 36 | 37 | array = [] 38 | q = PriorityQueue.new 39 | array.reverse_each do |v| 40 | q << v 41 | end 42 | expect(q.elements).to eq(array) 43 | 44 | array = [100, 36, 17, 19, 25, 0, 3, 1, 7, 2] 45 | array.reverse_each do |v| 46 | q << v 47 | end 48 | 49 | expect(q.pop).to eq(100) 50 | expect(q.elements).to eq([36, 25, 19, 17, 0, 1, 7, 2, 3]) 51 | 52 | # expected [36, 25, 19, 17, 0, 1, 7, 2, 3] 53 | expect(q.pop).to eq(36) 54 | expect(q.pop).to eq(25) 55 | expect(q.pop).to eq(19) 56 | expect(q.pop).to eq(17) 57 | expect(q.pop).to eq(7) 58 | expect(q.pop).to eq(3) 59 | expect(q.pop).to eq(2) 60 | expect(q.pop).to eq(1) 61 | expect(q.pop).to eq(0) 62 | expect(q.pop).to eq(nil) 63 | end 64 | 65 | it "priority queue" do 66 | frontier = PriorityQueue.new 67 | frontier << CurrentIndex.new(0) 68 | frontier << CurrentIndex.new(1) 69 | 70 | expect(frontier.sorted.map(&:current_indent)).to eq([0, 1]) 71 | 72 | frontier << CurrentIndex.new(1) 73 | expect(frontier.sorted.map(&:current_indent)).to eq([0, 1, 1]) 74 | 75 | frontier << CurrentIndex.new(0) 76 | expect(frontier.sorted.map(&:current_indent)).to eq([0, 0, 1, 1]) 77 | 78 | frontier << CurrentIndex.new(10) 79 | expect(frontier.sorted.map(&:current_indent)).to eq([0, 0, 1, 1, 10]) 80 | 81 | frontier << CurrentIndex.new(2) 82 | expect(frontier.sorted.map(&:current_indent)).to eq([0, 0, 1, 1, 2, 10]) 83 | 84 | frontier = PriorityQueue.new 85 | values = [18, 18, 0, 18, 0, 18, 18, 18, 18, 16, 18, 8, 18, 8, 8, 8, 16, 6, 0, 0, 16, 16, 4, 14, 14, 12, 12, 12, 10, 12, 12, 12, 12, 8, 10, 10, 8, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 8, 10, 6, 6, 6, 6, 6, 6, 8, 10, 8, 8, 10, 8, 10, 8, 10, 8, 6, 8, 8, 6, 8, 6, 6, 8, 0, 8, 0, 0, 8, 8, 0, 8, 0, 8, 8, 0, 8, 8, 8, 0, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 6, 8, 6, 6, 6, 6, 8, 6, 8, 6, 6, 4, 4, 6, 6, 4, 6, 4, 6, 6, 4, 6, 4, 4, 6, 6, 6, 6, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 6, 6, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 0, 0, 6, 6, 2] 86 | 87 | values.each do |v| 88 | value = CurrentIndex.new(v) 89 | frontier << value # CurrentIndex.new(v) 90 | end 91 | 92 | expect(frontier.sorted.map(&:current_indent)).to eq(values.sort) 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /spec/fixtures/routes.rb.txt: -------------------------------------------------------------------------------- 1 | Rails.application.routes.draw do 2 | constraints -> { Rails.application.config.non_production } do 3 | namespace :foo do 4 | resource :bar 5 | end 6 | end 7 | constraints -> { Rails.application.config.non_production } do 8 | namespace :bar do 9 | resource :baz 10 | end 11 | end 12 | constraints -> { Rails.application.config.non_production } do 13 | namespace :bar do 14 | resource :baz 15 | end 16 | end 17 | constraints -> { Rails.application.config.non_production } do 18 | namespace :bar do 19 | resource :baz 20 | end 21 | end 22 | constraints -> { Rails.application.config.non_production } do 23 | namespace :bar do 24 | resource :baz 25 | end 26 | end 27 | constraints -> { Rails.application.config.non_production } do 28 | namespace :bar do 29 | resource :baz 30 | end 31 | end 32 | constraints -> { Rails.application.config.non_production } do 33 | namespace :bar do 34 | resource :baz 35 | end 36 | end 37 | constraints -> { Rails.application.config.non_production } do 38 | namespace :bar do 39 | resource :baz 40 | end 41 | end 42 | constraints -> { Rails.application.config.non_production } do 43 | namespace :bar do 44 | resource :baz 45 | end 46 | end 47 | constraints -> { Rails.application.config.non_production } do 48 | namespace :bar do 49 | resource :baz 50 | end 51 | end 52 | constraints -> { Rails.application.config.non_production } do 53 | namespace :bar do 54 | resource :baz 55 | end 56 | end 57 | constraints -> { Rails.application.config.non_production } do 58 | namespace :bar do 59 | resource :baz 60 | end 61 | end 62 | constraints -> { Rails.application.config.non_production } do 63 | namespace :bar do 64 | resource :baz 65 | end 66 | end 67 | constraints -> { Rails.application.config.non_production } do 68 | namespace :bar do 69 | resource :baz 70 | end 71 | end 72 | constraints -> { Rails.application.config.non_production } do 73 | namespace :bar do 74 | resource :baz 75 | end 76 | end 77 | constraints -> { Rails.application.config.non_production } do 78 | namespace :bar do 79 | resource :baz 80 | end 81 | end 82 | constraints -> { Rails.application.config.non_production } do 83 | namespace :bar do 84 | resource :baz 85 | end 86 | end 87 | constraints -> { Rails.application.config.non_production } do 88 | namespace :bar do 89 | resource :baz 90 | end 91 | end 92 | constraints -> { Rails.application.config.non_production } do 93 | namespace :bar do 94 | resource :baz 95 | end 96 | end 97 | constraints -> { Rails.application.config.non_production } do 98 | namespace :bar do 99 | resource :baz 100 | end 101 | end 102 | constraints -> { Rails.application.config.non_production } do 103 | namespace :bar do 104 | resource :baz 105 | end 106 | end 107 | constraints -> { Rails.application.config.non_production } do 108 | namespace :bar do 109 | resource :baz 110 | end 111 | end 112 | 113 | namespace :admin do 114 | resource :session 115 | 116 | match "/foobar(*path)", via: :all, to: redirect { |_params, req| 117 | uri = URI(req.path.gsub("foobar", "foobaz")) 118 | uri.query = req.query_string.presence 119 | uri.to_s 120 | } 121 | end 122 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at richard.schneeman+foo@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [https://contributor-covenant.org/version/1/4/code-of-conduct/][version] 72 | 73 | [homepage]: https://contributor-covenant.org 74 | [version]: https://contributor-covenant.org/version/1/4/code-of-conduct/ 75 | -------------------------------------------------------------------------------- /lib/syntax_suggest/scan_history.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Scans up/down from the given block 5 | # 6 | # You can try out a change, stash it, or commit it to save for later 7 | # 8 | # Example: 9 | # 10 | # scanner = ScanHistory.new(code_lines: code_lines, block: block) 11 | # scanner.scan( 12 | # up: ->(_, _, _) { true }, 13 | # down: ->(_, _, _) { true } 14 | # ) 15 | # scanner.changed? # => true 16 | # expect(scanner.lines).to eq(code_lines) 17 | # 18 | # scanner.stash_changes 19 | # 20 | # expect(scanner.lines).to_not eq(code_lines) 21 | class ScanHistory 22 | attr_reader :before_index, :after_index 23 | 24 | def initialize(code_lines:, block:) 25 | @code_lines = code_lines 26 | @history = [block] 27 | refresh_index 28 | end 29 | 30 | def commit_if_changed 31 | if changed? 32 | @history << CodeBlock.new(lines: @code_lines[before_index..after_index]) 33 | end 34 | 35 | self 36 | end 37 | 38 | # Discards any changes that have not been committed 39 | def stash_changes 40 | refresh_index 41 | self 42 | end 43 | 44 | # Discard changes that have not been committed and revert the last commit 45 | # 46 | # Cannot revert the first commit 47 | def revert_last_commit 48 | if @history.length > 1 49 | @history.pop 50 | refresh_index 51 | end 52 | 53 | self 54 | end 55 | 56 | def changed? 57 | @before_index != current.lines.first.index || 58 | @after_index != current.lines.last.index 59 | end 60 | 61 | # Iterates up and down 62 | # 63 | # Returns line, kw_count, end_count for each iteration 64 | def scan(up:, down:) 65 | kw_count = 0 66 | end_count = 0 67 | 68 | up_index = before_lines.reverse_each.take_while do |line| 69 | kw_count += 1 if line.is_kw? 70 | end_count += 1 if line.is_end? 71 | up.call(line, kw_count, end_count) 72 | end.last&.index 73 | 74 | kw_count = 0 75 | end_count = 0 76 | 77 | down_index = after_lines.each.take_while do |line| 78 | kw_count += 1 if line.is_kw? 79 | end_count += 1 if line.is_end? 80 | down.call(line, kw_count, end_count) 81 | end.last&.index 82 | 83 | @before_index = if up_index && up_index < @before_index 84 | up_index 85 | else 86 | @before_index 87 | end 88 | 89 | @after_index = if down_index && down_index > @after_index 90 | down_index 91 | else 92 | @after_index 93 | end 94 | 95 | self 96 | end 97 | 98 | def next_up 99 | return nil if @before_index <= 0 100 | 101 | @code_lines[@before_index - 1] 102 | end 103 | 104 | def next_down 105 | return nil if @after_index >= @code_lines.length 106 | 107 | @code_lines[@after_index + 1] 108 | end 109 | 110 | def lines 111 | @code_lines[@before_index..@after_index] 112 | end 113 | 114 | private def before_lines 115 | @code_lines[0...@before_index] || [] 116 | end 117 | 118 | # Returns an array of all the CodeLines that exist after 119 | # the currently scanned block 120 | private def after_lines 121 | @code_lines[@after_index.next..] || [] 122 | end 123 | 124 | private def current 125 | @history.last 126 | end 127 | 128 | private def refresh_index 129 | @before_index = current.lines.first.index 130 | @after_index = current.lines.last.index 131 | self 132 | end 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /spec/unit/api_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | begin 5 | require "ruby-prof" 6 | rescue LoadError 7 | end 8 | 9 | module SyntaxSuggest 10 | RSpec.describe "Top level SyntaxSuggest api" do 11 | it "doesn't load prism if env var is set" do 12 | skip("SYNTAX_SUGGEST_DISABLE_PRISM not set") unless ENV["SYNTAX_SUGGEST_DISABLE_PRISM"] 13 | 14 | expect(SyntaxSuggest.use_prism_parser?).to be_falsey 15 | end 16 | 17 | it "has a `handle_error` interface" do 18 | fake_error = Object.new 19 | def fake_error.message 20 | "#{__FILE__}:216: unterminated string meets end of file " 21 | end 22 | 23 | def fake_error.is_a?(v) 24 | true 25 | end 26 | 27 | io = StringIO.new 28 | SyntaxSuggest.handle_error( 29 | fake_error, 30 | re_raise: false, 31 | io: io 32 | ) 33 | 34 | expect(io.string.strip).to eq("") 35 | end 36 | 37 | it "raises original error with warning if a non-syntax error is passed" do 38 | error = NameError.new("blerg") 39 | io = StringIO.new 40 | expect { 41 | SyntaxSuggest.handle_error( 42 | error, 43 | re_raise: false, 44 | io: io 45 | ) 46 | }.to raise_error { |e| 47 | expect(io.string).to include("Must pass a SyntaxError") 48 | expect(e).to eq(error) 49 | } 50 | end 51 | 52 | it "raises original error with warning if file is not found" do 53 | fake_error = SyntaxError.new 54 | def fake_error.message 55 | "#does/not/exist/lol/doesnotexist:216: unterminated string meets end of file " 56 | end 57 | 58 | io = StringIO.new 59 | expect { 60 | SyntaxSuggest.handle_error( 61 | fake_error, 62 | re_raise: false, 63 | io: io 64 | ) 65 | }.to raise_error { |e| 66 | expect(io.string).to include("Could not find filename") 67 | expect(e).to eq(fake_error) 68 | } 69 | end 70 | 71 | it "respects highlight API" do 72 | skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") 73 | 74 | core_ext_file = lib_dir.join("syntax_suggest").join("core_ext.rb") 75 | require_relative core_ext_file 76 | 77 | error_klass = Class.new do 78 | def path 79 | fixtures_dir.join("this_project_extra_def.rb.txt") 80 | end 81 | 82 | def detailed_message(**kwargs) 83 | "error" 84 | end 85 | end 86 | error_klass.prepend(SyntaxSuggest.module_for_detailed_message) 87 | error = error_klass.new 88 | 89 | expect(error.detailed_message(highlight: true)).to include(SyntaxSuggest::DisplayCodeWithLineNumbers::TERMINAL_HIGHLIGHT) 90 | expect(error.detailed_message(highlight: false)).to_not include(SyntaxSuggest::DisplayCodeWithLineNumbers::TERMINAL_HIGHLIGHT) 91 | end 92 | 93 | it "can be disabled via falsey kwarg" do 94 | skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") 95 | 96 | core_ext_file = lib_dir.join("syntax_suggest").join("core_ext.rb") 97 | require_relative core_ext_file 98 | 99 | error_klass = Class.new do 100 | def path 101 | fixtures_dir.join("this_project_extra_def.rb.txt") 102 | end 103 | 104 | def detailed_message(**kwargs) 105 | "error" 106 | end 107 | end 108 | error_klass.prepend(SyntaxSuggest.module_for_detailed_message) 109 | error = error_klass.new 110 | 111 | expect(error.detailed_message(syntax_suggest: true)).to_not eq(error.detailed_message(syntax_suggest: false)) 112 | end 113 | end 114 | end 115 | -------------------------------------------------------------------------------- /spec/unit/scan_history_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe ScanHistory do 7 | it "retains commits" do 8 | source = <<~EOM 9 | class OH # 0 10 | def lol # 1 11 | print 'lol # 2 12 | end # 3 13 | 14 | def hello # 5 15 | it "foo" do # 6 16 | end # 7 17 | 18 | def yolo # 8 19 | print 'haha' # 9 20 | end # 10 21 | end 22 | EOM 23 | 24 | code_lines = CleanDocument.new(source: source).call.lines 25 | block = CodeBlock.new(lines: code_lines[6]) 26 | 27 | scanner = ScanHistory.new(code_lines: code_lines, block: block) 28 | scanner.scan(up: ->(_, _, _) { true }, down: ->(_, _, _) { true }) 29 | 30 | expect(scanner.changed?).to be_truthy 31 | scanner.commit_if_changed 32 | expect(scanner.changed?).to be_falsey 33 | 34 | expect(scanner.lines).to eq(code_lines) 35 | 36 | scanner.stash_changes # Assert does nothing if changes are already committed 37 | expect(scanner.lines).to eq(code_lines) 38 | 39 | scanner.revert_last_commit 40 | 41 | expect(scanner.lines.join).to eq(code_lines[6].to_s) 42 | end 43 | 44 | it "is stashable" do 45 | source = <<~EOM 46 | class OH # 0 47 | def lol # 1 48 | print 'lol # 2 49 | end # 3 50 | 51 | def hello # 5 52 | it "foo" do # 6 53 | end # 7 54 | 55 | def yolo # 8 56 | print 'haha' # 9 57 | end # 10 58 | end 59 | EOM 60 | 61 | code_lines = CleanDocument.new(source: source).call.lines 62 | block = CodeBlock.new(lines: code_lines[6]) 63 | 64 | scanner = ScanHistory.new(code_lines: code_lines, block: block) 65 | scanner.scan(up: ->(_, _, _) { true }, down: ->(_, _, _) { true }) 66 | 67 | expect(scanner.lines).to eq(code_lines) 68 | expect(scanner.changed?).to be_truthy 69 | expect(scanner.next_up).to be_falsey 70 | expect(scanner.next_down).to be_falsey 71 | 72 | scanner.stash_changes 73 | 74 | expect(scanner.changed?).to be_falsey 75 | 76 | expect(scanner.next_up).to eq(code_lines[5]) 77 | expect(scanner.lines.join).to eq(code_lines[6].to_s) 78 | expect(scanner.next_down).to eq(code_lines[7]) 79 | end 80 | 81 | it "doesnt change if you dont't change it" do 82 | source = <<~EOM 83 | class OH # 0 84 | def lol # 1 85 | print 'lol # 2 86 | end # 3 87 | 88 | def hello # 5 89 | it "foo" do # 6 90 | end # 7 91 | 92 | def yolo # 8 93 | print 'haha' # 9 94 | end # 10 95 | end 96 | EOM 97 | 98 | code_lines = CleanDocument.new(source: source).call.lines 99 | block = CodeBlock.new(lines: code_lines[6]) 100 | 101 | scanner = ScanHistory.new(code_lines: code_lines, block: block) 102 | 103 | lines = scanner.lines 104 | expect(scanner.changed?).to be_falsey 105 | expect(scanner.next_up).to eq(code_lines[5]) 106 | expect(scanner.next_down).to eq(code_lines[7]) 107 | 108 | expect(scanner.stash_changes.lines).to eq(lines) 109 | expect(scanner.revert_last_commit.lines).to eq(lines) 110 | 111 | expect(scanner.scan(up: ->(_, _, _) { false }, down: ->(_, _, _) { false }).lines).to eq(lines) 112 | end 113 | end 114 | end 115 | -------------------------------------------------------------------------------- /lib/syntax_suggest/cli.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "pathname" 4 | require "optparse" 5 | 6 | module SyntaxSuggest 7 | # All the logic of the exe/syntax_suggest CLI in one handy spot 8 | # 9 | # Cli.new(argv: ["--help"]).call 10 | # Cli.new(argv: [".rb"]).call 11 | # Cli.new(argv: [".rb", "--record=tmp"]).call 12 | # Cli.new(argv: [".rb", "--terminal"]).call 13 | # 14 | class Cli 15 | attr_accessor :options 16 | 17 | # ARGV is Everything passed to the executable, does not include executable name 18 | # 19 | # All other intputs are dependency injection for testing 20 | def initialize(argv:, exit_obj: Kernel, io: $stdout, env: ENV) 21 | @options = {} 22 | @parser = nil 23 | options[:record_dir] = env["SYNTAX_SUGGEST_RECORD_DIR"] 24 | options[:record_dir] = "tmp" if env["DEBUG"] 25 | options[:terminal] = SyntaxSuggest::DEFAULT_VALUE 26 | 27 | @io = io 28 | @argv = argv 29 | @exit_obj = exit_obj 30 | end 31 | 32 | def call 33 | if @argv.empty? 34 | # Display help if raw command 35 | parser.parse! %w[--help] 36 | return 37 | else 38 | # Mutates @argv 39 | parse 40 | return if options[:exit] 41 | end 42 | 43 | file_name = @argv.first 44 | if file_name.nil? 45 | @io.puts "No file given" 46 | @exit_obj.exit(1) 47 | return 48 | end 49 | 50 | file = Pathname(file_name) 51 | if !file.exist? 52 | @io.puts "file not found: #{file.expand_path} " 53 | @exit_obj.exit(1) 54 | return 55 | end 56 | 57 | @io.puts "Record dir: #{options[:record_dir]}" if options[:record_dir] 58 | 59 | display = SyntaxSuggest.call( 60 | io: @io, 61 | source: file.read, 62 | filename: file.expand_path, 63 | terminal: options.fetch(:terminal, SyntaxSuggest::DEFAULT_VALUE), 64 | record_dir: options[:record_dir] 65 | ) 66 | 67 | if display.document_ok? 68 | @io.puts "Syntax OK" 69 | @exit_obj.exit(0) 70 | else 71 | @exit_obj.exit(1) 72 | end 73 | end 74 | 75 | def parse 76 | parser.parse!(@argv) 77 | 78 | self 79 | end 80 | 81 | def parser 82 | @parser ||= OptionParser.new do |opts| 83 | opts.banner = <<~EOM 84 | Usage: syntax_suggest [options] 85 | 86 | Parses a ruby source file and searches for syntax error(s) such as 87 | unexpected `end', expecting end-of-input. 88 | 89 | Example: 90 | 91 | $ syntax_suggest dog.rb 92 | 93 | # ... 94 | 95 | > 10 defdog 96 | > 15 end 97 | 98 | ENV options: 99 | 100 | SYNTAX_SUGGEST_RECORD_DIR= 101 | 102 | Records the steps used to search for a syntax error 103 | to the given directory 104 | 105 | Options: 106 | EOM 107 | 108 | opts.version = SyntaxSuggest::VERSION 109 | 110 | opts.on("--help", "Help - displays this message") do |v| 111 | @io.puts opts 112 | options[:exit] = true 113 | @exit_obj.exit 114 | end 115 | 116 | opts.on("--record ", "Records the steps used to search for a syntax error to the given directory") do |v| 117 | options[:record_dir] = v 118 | end 119 | 120 | opts.on("--terminal", "Enable terminal highlighting") do |v| 121 | options[:terminal] = true 122 | end 123 | 124 | opts.on("--no-terminal", "Disable terminal highlighting") do |v| 125 | options[:terminal] = false 126 | end 127 | end 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /spec/unit/code_frontier_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe CodeFrontier do 7 | it "detect_bad_blocks" do 8 | code_lines = code_line_array(<<~EOM) 9 | describe "lol" do 10 | end 11 | end 12 | 13 | it "lol" do 14 | end 15 | end 16 | EOM 17 | 18 | frontier = CodeFrontier.new(code_lines: code_lines) 19 | blocks = [] 20 | blocks << CodeBlock.new(lines: code_lines[1]) 21 | blocks << CodeBlock.new(lines: code_lines[5]) 22 | blocks.each do |b| 23 | frontier << b 24 | end 25 | 26 | expect(frontier.detect_invalid_blocks.sort).to eq(blocks.sort) 27 | end 28 | 29 | it "self.combination" do 30 | expect( 31 | CodeFrontier.combination([:a, :b, :c, :d]) 32 | ).to eq( 33 | [ 34 | [:a], [:b], [:c], [:d], 35 | [:a, :b], 36 | [:a, :c], 37 | [:a, :d], 38 | [:b, :c], 39 | [:b, :d], 40 | [:c, :d], 41 | [:a, :b, :c], 42 | [:a, :b, :d], 43 | [:a, :c, :d], 44 | [:b, :c, :d], 45 | [:a, :b, :c, :d] 46 | ] 47 | ) 48 | end 49 | 50 | it "doesn't duplicate blocks" do 51 | code_lines = code_line_array(<<~EOM) 52 | def foo 53 | puts "lol" 54 | puts "lol" 55 | puts "lol" 56 | end 57 | EOM 58 | 59 | frontier = CodeFrontier.new(code_lines: code_lines) 60 | frontier << CodeBlock.new(lines: [code_lines[2]]) 61 | expect(frontier.count).to eq(1) 62 | 63 | frontier << CodeBlock.new(lines: [code_lines[1], code_lines[2], code_lines[3]]) 64 | # expect(frontier.count).to eq(1) 65 | expect(frontier.pop.to_s).to eq(<<~EOM.indent(2)) 66 | puts "lol" 67 | puts "lol" 68 | puts "lol" 69 | EOM 70 | 71 | expect(frontier.pop).to be_nil 72 | 73 | code_lines = code_line_array(<<~EOM) 74 | def foo 75 | puts "lol" 76 | puts "lol" 77 | puts "lol" 78 | end 79 | EOM 80 | 81 | frontier = CodeFrontier.new(code_lines: code_lines) 82 | frontier << CodeBlock.new(lines: [code_lines[2]]) 83 | expect(frontier.count).to eq(1) 84 | 85 | frontier << CodeBlock.new(lines: [code_lines[3]]) 86 | expect(frontier.count).to eq(2) 87 | expect(frontier.pop.to_s).to eq(<<~EOM.indent(2)) 88 | puts "lol" 89 | EOM 90 | end 91 | 92 | it "detects if multiple syntax errors are found" do 93 | code_lines = code_line_array(<<~EOM) 94 | def foo 95 | end 96 | end 97 | EOM 98 | 99 | frontier = CodeFrontier.new(code_lines: code_lines) 100 | 101 | frontier << CodeBlock.new(lines: code_lines[1]) 102 | block = frontier.pop 103 | expect(block.to_s).to eq(<<~EOM.indent(2)) 104 | end 105 | EOM 106 | frontier << block 107 | 108 | expect(frontier.holds_all_syntax_errors?).to be_truthy 109 | end 110 | 111 | it "detects if it has not captured all syntax errors" do 112 | code_lines = code_line_array(<<~EOM) 113 | def foo 114 | puts "lol" 115 | end 116 | 117 | describe "lol" 118 | end 119 | 120 | it "lol" 121 | end 122 | EOM 123 | 124 | frontier = CodeFrontier.new(code_lines: code_lines) 125 | frontier << CodeBlock.new(lines: [code_lines[1]]) 126 | block = frontier.pop 127 | expect(block.to_s).to eq(<<~EOM.indent(2)) 128 | puts "lol" 129 | EOM 130 | frontier << block 131 | 132 | expect(frontier.holds_all_syntax_errors?).to be_falsey 133 | end 134 | end 135 | end 136 | -------------------------------------------------------------------------------- /lib/syntax_suggest/code_search.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Searches code for a syntax error 5 | # 6 | # There are three main phases in the algorithm: 7 | # 8 | # 1. Sanitize/format input source 9 | # 2. Search for invalid blocks 10 | # 3. Format invalid blocks into something meaninful 11 | # 12 | # This class handles the part. 13 | # 14 | # The bulk of the heavy lifting is done in: 15 | # 16 | # - CodeFrontier (Holds information for generating blocks and determining if we can stop searching) 17 | # - ParseBlocksFromLine (Creates blocks into the frontier) 18 | # - BlockExpand (Expands existing blocks to search more code) 19 | # 20 | # ## Syntax error detection 21 | # 22 | # When the frontier holds the syntax error, we can stop searching 23 | # 24 | # search = CodeSearch.new(<<~EOM) 25 | # def dog 26 | # def lol 27 | # end 28 | # EOM 29 | # 30 | # search.call 31 | # 32 | # search.invalid_blocks.map(&:to_s) # => 33 | # # => ["def lol\n"] 34 | # 35 | class CodeSearch 36 | private 37 | 38 | attr_reader :frontier 39 | 40 | public 41 | 42 | attr_reader :invalid_blocks, :record_dir, :code_lines 43 | 44 | def initialize(source, record_dir: DEFAULT_VALUE) 45 | record_dir = if record_dir == DEFAULT_VALUE 46 | (ENV["SYNTAX_SUGGEST_RECORD_DIR"] || ENV["SYNTAX_SUGGEST_DEBUG"]) ? "tmp" : nil 47 | else 48 | record_dir 49 | end 50 | 51 | if record_dir 52 | @record_dir = SyntaxSuggest.record_dir(record_dir) 53 | @write_count = 0 54 | end 55 | 56 | @tick = 0 57 | @source = source 58 | @name_tick = Hash.new { |hash, k| hash[k] = 0 } 59 | @invalid_blocks = [] 60 | 61 | @code_lines = CleanDocument.new(source: source).call.lines 62 | 63 | @frontier = CodeFrontier.new(code_lines: @code_lines) 64 | @block_expand = BlockExpand.new(code_lines: @code_lines) 65 | @parse_blocks_from_indent_line = ParseBlocksFromIndentLine.new(code_lines: @code_lines) 66 | end 67 | 68 | # Used for debugging 69 | def record(block:, name: "record") 70 | return unless @record_dir 71 | @name_tick[name] += 1 72 | filename = "#{@write_count += 1}-#{name}-#{@name_tick[name]}-(#{block.starts_at}__#{block.ends_at}).txt" 73 | if ENV["SYNTAX_SUGGEST_DEBUG"] 74 | puts "\n\n==== #{filename} ====" 75 | puts "\n```#{block.starts_at}..#{block.ends_at}" 76 | puts block 77 | puts "```" 78 | puts " block indent: #{block.current_indent}" 79 | end 80 | @record_dir.join(filename).open(mode: "a") do |f| 81 | document = DisplayCodeWithLineNumbers.new( 82 | lines: @code_lines.select(&:visible?), 83 | terminal: false, 84 | highlight_lines: block.lines 85 | ).call 86 | 87 | f.write(" Block lines: #{block.starts_at..block.ends_at} (#{name}) \n\n#{document}") 88 | end 89 | end 90 | 91 | def push(block, name:) 92 | record(block: block, name: name) 93 | 94 | block.mark_invisible if block.valid? 95 | frontier << block 96 | end 97 | 98 | # Parses the most indented lines into blocks that are marked 99 | # and added to the frontier 100 | def create_blocks_from_untracked_lines 101 | max_indent = frontier.next_indent_line&.indent 102 | 103 | while (line = frontier.next_indent_line) && (line.indent == max_indent) 104 | @parse_blocks_from_indent_line.each_neighbor_block(frontier.next_indent_line) do |block| 105 | push(block, name: "add") 106 | end 107 | end 108 | end 109 | 110 | # Given an already existing block in the frontier, expand it to see 111 | # if it contains our invalid syntax 112 | def expand_existing 113 | block = frontier.pop 114 | return unless block 115 | 116 | record(block: block, name: "before-expand") 117 | 118 | block = @block_expand.call(block) 119 | push(block, name: "expand") 120 | end 121 | 122 | # Main search loop 123 | def call 124 | until frontier.holds_all_syntax_errors? 125 | @tick += 1 126 | 127 | if frontier.expand? 128 | expand_existing 129 | else 130 | create_blocks_from_untracked_lines 131 | end 132 | end 133 | 134 | @invalid_blocks.concat(frontier.detect_invalid_blocks) 135 | @invalid_blocks.sort_by! { |block| block.starts_at } 136 | self 137 | end 138 | end 139 | end 140 | -------------------------------------------------------------------------------- /spec/unit/display_invalid_blocks_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe DisplayInvalidBlocks do 7 | it "works with valid code" do 8 | syntax_string = <<~EOM 9 | class OH 10 | def hello 11 | end 12 | def hai 13 | end 14 | end 15 | EOM 16 | 17 | search = CodeSearch.new(syntax_string) 18 | search.call 19 | 20 | io = StringIO.new 21 | display = DisplayInvalidBlocks.new( 22 | io: io, 23 | blocks: search.invalid_blocks, 24 | terminal: false, 25 | code_lines: search.code_lines 26 | ) 27 | display.call 28 | expect(io.string).to include("") 29 | end 30 | 31 | it "selectively prints to terminal if input is a tty by default" do 32 | source = <<~EOM 33 | class OH 34 | def hello 35 | def hai 36 | end 37 | end 38 | EOM 39 | 40 | code_lines = CleanDocument.new(source: source).call.lines 41 | 42 | io = StringIO.new 43 | def io.isatty 44 | true 45 | end 46 | 47 | block = CodeBlock.new(lines: code_lines[1]) 48 | display = DisplayInvalidBlocks.new( 49 | io: io, 50 | blocks: block, 51 | code_lines: code_lines 52 | ) 53 | display.call 54 | expect(io.string).to include([ 55 | "> 2 ", 56 | DisplayCodeWithLineNumbers::TERMINAL_HIGHLIGHT, 57 | " def hello" 58 | ].join) 59 | 60 | io = StringIO.new 61 | def io.isatty 62 | false 63 | end 64 | 65 | block = CodeBlock.new(lines: code_lines[1]) 66 | display = DisplayInvalidBlocks.new( 67 | io: io, 68 | blocks: block, 69 | code_lines: code_lines 70 | ) 71 | display.call 72 | expect(io.string).to include("> 2 def hello") 73 | end 74 | 75 | it "outputs to io when using `call`" do 76 | source = <<~EOM 77 | class OH 78 | def hello 79 | def hai 80 | end 81 | end 82 | EOM 83 | 84 | code_lines = CleanDocument.new(source: source).call.lines 85 | 86 | io = StringIO.new 87 | block = CodeBlock.new(lines: code_lines[1]) 88 | display = DisplayInvalidBlocks.new( 89 | io: io, 90 | blocks: block, 91 | terminal: false, 92 | code_lines: code_lines 93 | ) 94 | display.call 95 | expect(io.string).to include("> 2 def hello") 96 | end 97 | 98 | it " wraps code with github style codeblocks" do 99 | source = <<~EOM 100 | class OH 101 | def hello 102 | 103 | def hai 104 | end 105 | end 106 | EOM 107 | 108 | code_lines = CleanDocument.new(source: source).call.lines 109 | block = CodeBlock.new(lines: code_lines[1]) 110 | io = StringIO.new 111 | DisplayInvalidBlocks.new( 112 | io: io, 113 | blocks: block, 114 | terminal: false, 115 | code_lines: code_lines 116 | ).call 117 | expect(io.string).to include(<<~EOM) 118 | 1 class OH 119 | > 2 def hello 120 | 4 def hai 121 | 5 end 122 | 6 end 123 | EOM 124 | end 125 | 126 | it "shows terminal characters" do 127 | code_lines = code_line_array(<<~EOM) 128 | class OH 129 | def hello 130 | def hai 131 | end 132 | end 133 | EOM 134 | 135 | io = StringIO.new 136 | block = CodeBlock.new(lines: code_lines[1]) 137 | DisplayInvalidBlocks.new( 138 | io: io, 139 | blocks: block, 140 | terminal: false, 141 | code_lines: code_lines 142 | ).call 143 | 144 | expect(io.string).to include([ 145 | " 1 class OH", 146 | "> 2 def hello", 147 | " 3 def hai", 148 | " 4 end", 149 | " 5 end", 150 | "" 151 | ].join($/)) 152 | 153 | block = CodeBlock.new(lines: code_lines[1]) 154 | io = StringIO.new 155 | DisplayInvalidBlocks.new( 156 | io: io, 157 | blocks: block, 158 | terminal: true, 159 | code_lines: code_lines 160 | ).call 161 | 162 | expect(io.string).to include( 163 | [ 164 | " 1 class OH", 165 | ["> 2 ", DisplayCodeWithLineNumbers::TERMINAL_HIGHLIGHT, " def hello"].join, 166 | " 3 def hai", 167 | " 4 end", 168 | " 5 end", 169 | "" 170 | ].join($/ + DisplayCodeWithLineNumbers::TERMINAL_END) 171 | ) 172 | end 173 | end 174 | end 175 | -------------------------------------------------------------------------------- /lib/syntax_suggest/left_right_lex_count.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Find mis-matched syntax based on lexical count 5 | # 6 | # Used for detecting missing pairs of elements 7 | # each keyword needs an end, each '{' needs a '}' 8 | # etc. 9 | # 10 | # Example: 11 | # 12 | # left_right = LeftRightLexCount.new 13 | # left_right.count_kw 14 | # left_right.missing.first 15 | # # => "end" 16 | # 17 | # left_right = LeftRightLexCount.new 18 | # source = "{ a: b, c: d" # Note missing '}' 19 | # LexAll.new(source: source).each do |lex| 20 | # left_right.count_lex(lex) 21 | # end 22 | # left_right.missing.first 23 | # # => "}" 24 | class LeftRightLexCount 25 | def initialize 26 | @kw_count = 0 27 | @end_count = 0 28 | 29 | @count_for_char = { 30 | "{" => 0, 31 | "}" => 0, 32 | "[" => 0, 33 | "]" => 0, 34 | "(" => 0, 35 | ")" => 0, 36 | "|" => 0 37 | } 38 | end 39 | 40 | def count_kw 41 | @kw_count += 1 42 | end 43 | 44 | def count_end 45 | @end_count += 1 46 | end 47 | 48 | # Count source code characters 49 | # 50 | # Example: 51 | # 52 | # left_right = LeftRightLexCount.new 53 | # left_right.count_lex(LexValue.new(1, :on_lbrace, "{", Ripper::EXPR_BEG)) 54 | # left_right.count_for_char("{") 55 | # # => 1 56 | # left_right.count_for_char("}") 57 | # # => 0 58 | def count_lex(lex) 59 | case lex.type 60 | when :on_tstring_content 61 | # ^^^ 62 | # Means it's a string or a symbol `"{"` rather than being 63 | # part of a data structure (like a hash) `{ a: b }` 64 | # ignore it. 65 | when :on_words_beg, :on_symbos_beg, :on_qwords_beg, 66 | :on_qsymbols_beg, :on_regexp_beg, :on_tstring_beg 67 | # ^^^ 68 | # Handle shorthand syntaxes like `%Q{ i am a string }` 69 | # 70 | # The start token will be the full thing `%Q{` but we 71 | # need to count it as if it's a `{`. Any token 72 | # can be used 73 | char = lex.token[-1] 74 | @count_for_char[char] += 1 if @count_for_char.key?(char) 75 | when :on_embexpr_beg 76 | # ^^^ 77 | # Embedded string expressions like `"#{foo} <-embed"` 78 | # are parsed with chars: 79 | # 80 | # `#{` as :on_embexpr_beg 81 | # `}` as :on_embexpr_end 82 | # 83 | # We cannot ignore both :on_emb_expr_beg and :on_embexpr_end 84 | # because sometimes the lexer thinks something is an embed 85 | # string end, when it is not like `lol = }` (no clue why). 86 | # 87 | # When we see `#{` count it as a `{` or we will 88 | # have a mis-match count. 89 | # 90 | case lex.token 91 | when "\#{" 92 | @count_for_char["{"] += 1 93 | end 94 | else 95 | @end_count += 1 if lex.is_end? 96 | @kw_count += 1 if lex.is_kw? 97 | @count_for_char[lex.token] += 1 if @count_for_char.key?(lex.token) 98 | end 99 | end 100 | 101 | def count_for_char(char) 102 | @count_for_char[char] 103 | end 104 | 105 | # Returns an array of missing syntax characters 106 | # or `"end"` or `"keyword"` 107 | # 108 | # left_right.missing 109 | # # => ["}"] 110 | def missing 111 | out = missing_pairs 112 | out << missing_pipe 113 | out << missing_keyword_end 114 | out.compact! 115 | out 116 | end 117 | 118 | PAIRS = { 119 | "{" => "}", 120 | "[" => "]", 121 | "(" => ")" 122 | }.freeze 123 | 124 | # Opening characters like `{` need closing characters # like `}`. 125 | # 126 | # When a mis-match count is detected, suggest the 127 | # missing member. 128 | # 129 | # For example if there are 3 `}` and only two `{` 130 | # return `"{"` 131 | private def missing_pairs 132 | PAIRS.map do |(left, right)| 133 | case @count_for_char[left] <=> @count_for_char[right] 134 | when 1 135 | right 136 | when 0 137 | nil 138 | when -1 139 | left 140 | end 141 | end 142 | end 143 | 144 | # Keywords need ends and ends need keywords 145 | # 146 | # If we have more keywords, there's a missing `end` 147 | # if we have more `end`-s, there's a missing keyword 148 | private def missing_keyword_end 149 | case @kw_count <=> @end_count 150 | when 1 151 | "end" 152 | when 0 153 | nil 154 | when -1 155 | "keyword" 156 | end 157 | end 158 | 159 | # Pipes come in pairs. 160 | # If there's an odd number of pipes then we 161 | # are missing one 162 | private def missing_pipe 163 | if @count_for_char["|"].odd? 164 | "|" 165 | end 166 | end 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /spec/unit/code_line_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe CodeLine do 7 | it "bug in keyword detection" do 8 | lines = CodeLine.from_source(<<~EOM) 9 | def to_json(*opts) 10 | { 11 | type: :module, 12 | }.to_json(*opts) 13 | end 14 | EOM 15 | expect(lines.count(&:is_kw?)).to eq(1) 16 | expect(lines.count(&:is_end?)).to eq(1) 17 | end 18 | 19 | it "supports endless method definitions" do 20 | skip("Unsupported ruby version") unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3") 21 | 22 | line = CodeLine.from_source(<<~EOM).first 23 | def square(x) = x * x 24 | EOM 25 | 26 | expect(line.is_kw?).to be_falsey 27 | expect(line.is_end?).to be_falsey 28 | end 29 | 30 | it "retains original line value, after being marked invisible" do 31 | line = CodeLine.from_source(<<~EOM).first 32 | puts "lol" 33 | EOM 34 | expect(line.line).to match('puts "lol"') 35 | line.mark_invisible 36 | expect(line.line).to eq("") 37 | expect(line.original).to match('puts "lol"') 38 | end 39 | 40 | it "knows which lines can be joined" do 41 | code_lines = CodeLine.from_source(<<~EOM) 42 | user = User. 43 | where(name: 'schneems'). 44 | first 45 | puts user.name 46 | EOM 47 | 48 | # Indicates line 1 can join 2, 2 can join 3, but 3 won't join it's next line 49 | expect(code_lines.map(&:ignore_newline_not_beg?)).to eq([true, true, false, false]) 50 | end 51 | 52 | it "trailing if" do 53 | code_lines = CodeLine.from_source(<<~EOM) 54 | puts "lol" if foo 55 | if foo 56 | end 57 | EOM 58 | 59 | expect(code_lines.map(&:is_kw?)).to eq([false, true, false]) 60 | end 61 | 62 | it "trailing unless" do 63 | code_lines = CodeLine.from_source(<<~EOM) 64 | puts "lol" unless foo 65 | unless foo 66 | end 67 | EOM 68 | 69 | expect(code_lines.map(&:is_kw?)).to eq([false, true, false]) 70 | end 71 | 72 | it "trailing slash" do 73 | code_lines = CodeLine.from_source(<<~'EOM') 74 | it "trailing s" \ 75 | "lash" do 76 | EOM 77 | 78 | expect(code_lines.map(&:trailing_slash?)).to eq([true, false]) 79 | 80 | code_lines = CodeLine.from_source(<<~'EOM') 81 | amazing_print: ->(obj) { obj.ai + "\n" }, 82 | EOM 83 | expect(code_lines.map(&:trailing_slash?)).to eq([false]) 84 | end 85 | 86 | it "knows it's got an end" do 87 | line = CodeLine.from_source(" end").first 88 | 89 | expect(line.is_end?).to be_truthy 90 | expect(line.is_kw?).to be_falsey 91 | end 92 | 93 | it "knows it's got a keyword" do 94 | line = CodeLine.from_source(" if").first 95 | 96 | expect(line.is_end?).to be_falsey 97 | expect(line.is_kw?).to be_truthy 98 | end 99 | 100 | it "ignores marked lines" do 101 | code_lines = CodeLine.from_source(<<~EOM) 102 | def foo 103 | Array(value) |x| 104 | end 105 | end 106 | EOM 107 | 108 | expect(SyntaxSuggest.valid?(code_lines)).to be_falsey 109 | expect(code_lines.join).to eq(<<~EOM) 110 | def foo 111 | Array(value) |x| 112 | end 113 | end 114 | EOM 115 | 116 | expect(code_lines[0].visible?).to be_truthy 117 | expect(code_lines[3].visible?).to be_truthy 118 | 119 | code_lines[0].mark_invisible 120 | code_lines[3].mark_invisible 121 | 122 | expect(code_lines[0].visible?).to be_falsey 123 | expect(code_lines[3].visible?).to be_falsey 124 | 125 | expect(code_lines.join).to eq(<<~EOM.indent(2)) 126 | Array(value) |x| 127 | end 128 | EOM 129 | expect(SyntaxSuggest.valid?(code_lines)).to be_falsey 130 | end 131 | 132 | it "knows empty lines" do 133 | code_lines = CodeLine.from_source(<<~EOM) 134 | # Not empty 135 | 136 | # Not empty 137 | EOM 138 | 139 | expect(code_lines.map(&:empty?)).to eq([false, true, false]) 140 | expect(code_lines.map(&:not_empty?)).to eq([true, false, true]) 141 | expect(code_lines.map { |l| SyntaxSuggest.valid?(l) }).to eq([true, true, true]) 142 | end 143 | 144 | it "counts indentations" do 145 | code_lines = CodeLine.from_source(<<~EOM) 146 | def foo 147 | Array(value) |x| 148 | puts 'lol' 149 | end 150 | end 151 | EOM 152 | 153 | expect(code_lines.map(&:indent)).to eq([0, 2, 4, 2, 0]) 154 | end 155 | 156 | it "doesn't count empty lines as having an indentation" do 157 | code_lines = CodeLine.from_source(<<~EOM) 158 | 159 | 160 | EOM 161 | 162 | expect(code_lines.map(&:indent)).to eq([0, 0]) 163 | end 164 | end 165 | end 166 | -------------------------------------------------------------------------------- /spec/unit/around_block_scan_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe AroundBlockScan do 7 | it "continues scan from last location even if scan is false" do 8 | source = <<~EOM 9 | print 'omg' 10 | print 'lol' 11 | print 'haha' 12 | EOM 13 | code_lines = CodeLine.from_source(source) 14 | block = CodeBlock.new(lines: code_lines[1]) 15 | expand = AroundBlockScan.new(code_lines: code_lines, block: block) 16 | .scan_neighbors_not_empty 17 | 18 | expect(expand.code_block.to_s).to eq(source) 19 | expand.scan_while { |line| false } 20 | 21 | expect(expand.code_block.to_s).to eq(source) 22 | end 23 | 24 | it "scan_adjacent_indent works on first or last line" do 25 | source_string = <<~EOM 26 | def foo 27 | if [options.output_format_tty, options.output_format_block].include?(nil) 28 | raise("Bad output mode '\#{v}'; each must be one of \#{lookups.output_formats.keys}.") 29 | end 30 | end 31 | EOM 32 | 33 | code_lines = code_line_array(source_string) 34 | block = CodeBlock.new(lines: code_lines[4]) 35 | expand = AroundBlockScan.new(code_lines: code_lines, block: block) 36 | .scan_adjacent_indent 37 | 38 | expect(expand.code_block.to_s).to eq(<<~EOM) 39 | def foo 40 | if [options.output_format_tty, options.output_format_block].include?(nil) 41 | raise("Bad output mode '\#{v}'; each must be one of \#{lookups.output_formats.keys}.") 42 | end 43 | end 44 | EOM 45 | end 46 | 47 | it "expands indentation" do 48 | source_string = <<~EOM 49 | def foo 50 | if [options.output_format_tty, options.output_format_block].include?(nil) 51 | raise("Bad output mode '\#{v}'; each must be one of \#{lookups.output_formats.keys}.") 52 | end 53 | end 54 | EOM 55 | 56 | code_lines = code_line_array(source_string) 57 | block = CodeBlock.new(lines: code_lines[2]) 58 | expand = AroundBlockScan.new(code_lines: code_lines, block: block) 59 | .stop_after_kw 60 | .scan_adjacent_indent 61 | 62 | expect(expand.code_block.to_s).to eq(<<~EOM.indent(2)) 63 | if [options.output_format_tty, options.output_format_block].include?(nil) 64 | raise("Bad output mode '\#{v}'; each must be one of \#{lookups.output_formats.keys}.") 65 | end 66 | EOM 67 | end 68 | 69 | it "can stop before hitting another end" do 70 | source_string = <<~EOM 71 | def lol 72 | end 73 | def foo 74 | puts "lol" 75 | end 76 | EOM 77 | 78 | code_lines = code_line_array(source_string) 79 | block = CodeBlock.new(lines: code_lines[3]) 80 | expand = AroundBlockScan.new(code_lines: code_lines, block: block) 81 | expand.stop_after_kw 82 | expand.scan_while { true } 83 | 84 | expect(expand.code_block.to_s).to eq(<<~EOM) 85 | def foo 86 | puts "lol" 87 | end 88 | EOM 89 | end 90 | 91 | it "captures multiple empty and hidden lines" do 92 | source_string = <<~EOM 93 | def foo 94 | Foo.call 95 | 96 | puts "lol" 97 | 98 | end 99 | end 100 | EOM 101 | 102 | code_lines = code_line_array(source_string) 103 | block = CodeBlock.new(lines: code_lines[3]) 104 | expand = AroundBlockScan.new(code_lines: code_lines, block: block) 105 | expand.scan_while { true } 106 | 107 | expect(expand.lines.first.index).to eq(0) 108 | expect(expand.lines.last.index).to eq(6) 109 | expect(expand.code_block.to_s).to eq(source_string) 110 | end 111 | 112 | it "only takes what you ask" do 113 | source_string = <<~EOM 114 | def foo 115 | Foo.call 116 | 117 | puts "lol" 118 | 119 | end 120 | end 121 | EOM 122 | 123 | code_lines = code_line_array(source_string) 124 | block = CodeBlock.new(lines: code_lines[3]) 125 | expand = AroundBlockScan.new(code_lines: code_lines, block: block) 126 | expand.scan_while { |line| line.not_empty? } 127 | 128 | expect(expand.code_block.to_s).to eq(<<~EOM.indent(4)) 129 | puts "lol" 130 | EOM 131 | end 132 | 133 | it "skips what you want" do 134 | source_string = <<~EOM 135 | def foo 136 | Foo.call 137 | 138 | puts "haha" 139 | # hide me 140 | 141 | puts "lol" 142 | 143 | end 144 | end 145 | EOM 146 | 147 | code_lines = code_line_array(source_string) 148 | code_lines[4].mark_invisible 149 | 150 | block = CodeBlock.new(lines: code_lines[3]) 151 | expand = AroundBlockScan.new(code_lines: code_lines, block: block) 152 | expand.force_add_empty 153 | expand.force_add_hidden 154 | expand.scan_neighbors_not_empty 155 | 156 | expect(expand.code_block.to_s).to eq(<<~EOM.indent(4)) 157 | 158 | puts "haha" 159 | 160 | puts "lol" 161 | 162 | EOM 163 | end 164 | end 165 | end 166 | -------------------------------------------------------------------------------- /spec/unit/block_expand_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe BlockExpand do 7 | it "empty line in methods" do 8 | source_string = <<~EOM 9 | class Dog # index 0 10 | def bark # index 1 11 | 12 | end # index 3 13 | 14 | def sit # index 5 15 | print "sit" # index 6 16 | end # index 7 17 | end # index 8 18 | end # extra end 19 | EOM 20 | 21 | code_lines = code_line_array(source_string) 22 | 23 | sit = code_lines[4..7] 24 | sit.each(&:mark_invisible) 25 | 26 | block = CodeBlock.new(lines: sit) 27 | expansion = BlockExpand.new(code_lines: code_lines) 28 | block = expansion.expand_neighbors(block) 29 | 30 | expect(block.to_s).to eq(<<~EOM.indent(2)) 31 | def bark # index 1 32 | 33 | end # index 3 34 | EOM 35 | end 36 | 37 | it "captures multiple empty and hidden lines" do 38 | source_string = <<~EOM 39 | def foo 40 | Foo.call 41 | 42 | 43 | puts "lol" 44 | 45 | # hidden 46 | end 47 | end 48 | EOM 49 | 50 | code_lines = code_line_array(source_string) 51 | 52 | code_lines[6].mark_invisible 53 | 54 | block = CodeBlock.new(lines: [code_lines[3]]) 55 | expansion = BlockExpand.new(code_lines: code_lines) 56 | block = expansion.call(block) 57 | 58 | expect(block.to_s).to eq(<<~EOM.indent(4)) 59 | 60 | 61 | puts "lol" 62 | 63 | EOM 64 | end 65 | 66 | it "captures multiple empty lines" do 67 | source_string = <<~EOM 68 | def foo 69 | Foo.call 70 | 71 | 72 | puts "lol" 73 | 74 | end 75 | end 76 | EOM 77 | 78 | code_lines = code_line_array(source_string) 79 | block = CodeBlock.new(lines: [code_lines[3]]) 80 | expansion = BlockExpand.new(code_lines: code_lines) 81 | block = expansion.call(block) 82 | 83 | expect(block.to_s).to eq(<<~EOM.indent(4)) 84 | 85 | 86 | puts "lol" 87 | 88 | EOM 89 | end 90 | 91 | it "expands neighbors then indentation" do 92 | source_string = <<~EOM 93 | def foo 94 | Foo.call 95 | puts "hey" 96 | puts "lol" 97 | puts "sup" 98 | end 99 | end 100 | EOM 101 | 102 | code_lines = code_line_array(source_string) 103 | block = CodeBlock.new(lines: [code_lines[3]]) 104 | expansion = BlockExpand.new(code_lines: code_lines) 105 | block = expansion.call(block) 106 | 107 | expect(block.to_s).to eq(<<~EOM.indent(4)) 108 | puts "hey" 109 | puts "lol" 110 | puts "sup" 111 | EOM 112 | 113 | block = expansion.call(block) 114 | 115 | expect(block.to_s).to eq(<<~EOM.indent(2)) 116 | Foo.call 117 | puts "hey" 118 | puts "lol" 119 | puts "sup" 120 | end 121 | EOM 122 | end 123 | 124 | it "handles else code" do 125 | source_string = <<~EOM 126 | Foo.call 127 | if blerg 128 | puts "lol" 129 | else 130 | puts "haha" 131 | end 132 | end 133 | EOM 134 | 135 | code_lines = code_line_array(source_string) 136 | block = CodeBlock.new(lines: [code_lines[2]]) 137 | expansion = BlockExpand.new(code_lines: code_lines) 138 | block = expansion.call(block) 139 | 140 | expect(block.to_s).to eq(<<~EOM.indent(2)) 141 | if blerg 142 | puts "lol" 143 | else 144 | puts "haha" 145 | end 146 | EOM 147 | end 148 | 149 | it "expand until next boundary (indentation)" do 150 | source_string = <<~EOM 151 | describe "what" do 152 | Foo.call 153 | end 154 | 155 | describe "hi" 156 | Bar.call do 157 | Foo.call 158 | end 159 | end 160 | 161 | it "blerg" do 162 | end 163 | EOM 164 | 165 | code_lines = code_line_array(source_string) 166 | 167 | block = CodeBlock.new( 168 | lines: code_lines[6] 169 | ) 170 | 171 | expansion = BlockExpand.new(code_lines: code_lines) 172 | block = expansion.call(block) 173 | 174 | expect(block.to_s).to eq(<<~EOM.indent(2)) 175 | Bar.call do 176 | Foo.call 177 | end 178 | EOM 179 | 180 | block = expansion.call(block) 181 | 182 | expect(block.to_s).to eq(<<~EOM) 183 | describe "hi" 184 | Bar.call do 185 | Foo.call 186 | end 187 | end 188 | EOM 189 | end 190 | 191 | it "expand until next boundary (empty lines)" do 192 | source_string = <<~EOM 193 | describe "what" do 194 | end 195 | 196 | describe "hi" 197 | end 198 | 199 | it "blerg" do 200 | end 201 | EOM 202 | 203 | code_lines = code_line_array(source_string) 204 | expansion = BlockExpand.new(code_lines: code_lines) 205 | 206 | block = CodeBlock.new(lines: code_lines[3]) 207 | block = expansion.call(block) 208 | 209 | expect(block.to_s).to eq(<<~EOM) 210 | 211 | describe "hi" 212 | end 213 | 214 | EOM 215 | 216 | block = expansion.call(block) 217 | 218 | expect(block.to_s).to eq(<<~EOM) 219 | describe "what" do 220 | end 221 | 222 | describe "hi" 223 | end 224 | 225 | it "blerg" do 226 | end 227 | EOM 228 | end 229 | end 230 | end 231 | -------------------------------------------------------------------------------- /lib/syntax_suggest/block_expand.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # This class is responsible for taking a code block that exists 5 | # at a far indentaion and then iteratively increasing the block 6 | # so that it captures everything within the same indentation block. 7 | # 8 | # def dog 9 | # puts "bow" 10 | # puts "wow" 11 | # end 12 | # 13 | # block = BlockExpand.new(code_lines: code_lines) 14 | # .call(CodeBlock.new(lines: code_lines[1])) 15 | # 16 | # puts block.to_s 17 | # # => puts "bow" 18 | # puts "wow" 19 | # 20 | # 21 | # Once a code block has captured everything at a given indentation level 22 | # then it will expand to capture surrounding indentation. 23 | # 24 | # block = BlockExpand.new(code_lines: code_lines) 25 | # .call(block) 26 | # 27 | # block.to_s 28 | # # => def dog 29 | # puts "bow" 30 | # puts "wow" 31 | # end 32 | # 33 | class BlockExpand 34 | def initialize(code_lines:) 35 | @code_lines = code_lines 36 | end 37 | 38 | # Main interface. Expand current indentation, before 39 | # expanding to a lower indentation 40 | def call(block) 41 | if (next_block = expand_neighbors(block)) 42 | next_block 43 | else 44 | expand_indent(block) 45 | end 46 | end 47 | 48 | # Expands code to the next lowest indentation 49 | # 50 | # For example: 51 | # 52 | # 1 def dog 53 | # 2 print "dog" 54 | # 3 end 55 | # 56 | # If a block starts on line 2 then it has captured all it's "neighbors" (code at 57 | # the same indentation or higher). To continue expanding, this block must capture 58 | # lines one and three which are at a different indentation level. 59 | # 60 | # This method allows fully expanded blocks to decrease their indentation level (so 61 | # they can expand to capture more code up and down). It does this conservatively 62 | # as there's no undo (currently). 63 | def expand_indent(block) 64 | now = AroundBlockScan.new(code_lines: @code_lines, block: block) 65 | .force_add_hidden 66 | .stop_after_kw 67 | .scan_adjacent_indent 68 | 69 | now.lookahead_balance_one_line 70 | 71 | now.code_block 72 | end 73 | 74 | # A neighbor is code that is at or above the current indent line. 75 | # 76 | # First we build a block with all neighbors. If we can't go further 77 | # then we decrease the indentation threshold and expand via indentation 78 | # i.e. `expand_indent` 79 | # 80 | # Handles two general cases. 81 | # 82 | # ## Case #1: Check code inside of methods/classes/etc. 83 | # 84 | # It's important to note, that not everything in a given indentation level can be parsed 85 | # as valid code even if it's part of valid code. For example: 86 | # 87 | # 1 hash = { 88 | # 2 name: "richard", 89 | # 3 dog: "cinco", 90 | # 4 } 91 | # 92 | # In this case lines 2 and 3 will be neighbors, but they're invalid until `expand_indent` 93 | # is called on them. 94 | # 95 | # When we are adding code within a method or class (at the same indentation level), 96 | # use the empty lines to denote the programmer intended logical chunks. 97 | # Stop and check each one. For example: 98 | # 99 | # 1 def dog 100 | # 2 print "dog" 101 | # 3 102 | # 4 hash = { 103 | # 5 end 104 | # 105 | # If we did not stop parsing at empty newlines then the block might mistakenly grab all 106 | # the contents (lines 2, 3, and 4) and report them as being problems, instead of only 107 | # line 4. 108 | # 109 | # ## Case #2: Expand/grab other logical blocks 110 | # 111 | # Once the search algorithm has converted all lines into blocks at a given indentation 112 | # it will then `expand_indent`. Once the blocks that generates are expanded as neighbors 113 | # we then begin seeing neighbors being other logical blocks i.e. a block's neighbors 114 | # may be another method or class (something with keywords/ends). 115 | # 116 | # For example: 117 | # 118 | # 1 def bark 119 | # 2 120 | # 3 end 121 | # 4 122 | # 5 def sit 123 | # 6 end 124 | # 125 | # In this case if lines 4, 5, and 6 are in a block when it tries to expand neighbors 126 | # it will expand up. If it stops after line 2 or 3 it may cause problems since there's a 127 | # valid kw/end pair, but the block will be checked without it. 128 | # 129 | # We try to resolve this edge case with `lookahead_balance_one_line` below. 130 | def expand_neighbors(block) 131 | now = AroundBlockScan.new(code_lines: @code_lines, block: block) 132 | 133 | # Initial scan 134 | now 135 | .force_add_hidden 136 | .stop_after_kw 137 | .scan_neighbors_not_empty 138 | 139 | # Slurp up empties 140 | now 141 | .scan_while { |line| line.empty? } 142 | 143 | # If next line is kw and it will balance us, take it 144 | expanded_lines = now 145 | .lookahead_balance_one_line 146 | .lines 147 | 148 | # Don't allocate a block if it won't be used 149 | # 150 | # If nothing was taken, return nil to indicate that status 151 | # used in `def call` to determine if 152 | # we need to expand up/out (`expand_indent`) 153 | if block.lines == expanded_lines 154 | nil 155 | else 156 | CodeBlock.new(lines: expanded_lines) 157 | end 158 | end 159 | 160 | # Manageable rspec errors 161 | def inspect 162 | "#" 163 | end 164 | end 165 | end 166 | -------------------------------------------------------------------------------- /spec/unit/capture_code_context_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe CaptureCodeContext do 7 | it "capture_before_after_kws two" do 8 | source = <<~EOM 9 | class OH 10 | 11 | def hello 12 | 13 | def hai 14 | end 15 | end 16 | EOM 17 | 18 | code_lines = CleanDocument.new(source: source).call.lines 19 | block = CodeBlock.new(lines: code_lines[2]) 20 | 21 | display = CaptureCodeContext.new( 22 | blocks: [block], 23 | code_lines: code_lines 24 | ) 25 | display.capture_before_after_kws(block) 26 | expect(display.sorted_lines.join).to eq(<<~EOM.indent(2)) 27 | def hello 28 | def hai 29 | end 30 | EOM 31 | end 32 | 33 | it "capture_before_after_kws" do 34 | source = <<~EOM 35 | def sit 36 | end 37 | 38 | def bark 39 | 40 | def eat 41 | end 42 | EOM 43 | 44 | code_lines = CleanDocument.new(source: source).call.lines 45 | block = CodeBlock.new(lines: code_lines[3]) 46 | 47 | display = CaptureCodeContext.new( 48 | blocks: [block], 49 | code_lines: code_lines 50 | ) 51 | 52 | lines = display.capture_before_after_kws(block).sort 53 | expect(lines.join).to eq(<<~EOM) 54 | def sit 55 | end 56 | def bark 57 | def eat 58 | end 59 | EOM 60 | end 61 | 62 | it "handles ambiguous end" do 63 | source = <<~EOM 64 | def call # 0 65 | print "lol" # 1 66 | end # one # 2 67 | end # two # 3 68 | EOM 69 | 70 | code_lines = CleanDocument.new(source: source).call.lines 71 | code_lines[0..2].each(&:mark_invisible) 72 | block = CodeBlock.new(lines: code_lines) 73 | 74 | display = CaptureCodeContext.new( 75 | blocks: [block], 76 | code_lines: code_lines 77 | ) 78 | lines = display.call 79 | 80 | lines = lines.sort.map(&:original) 81 | 82 | expect(lines.join).to eq(<<~EOM) 83 | def call # 0 84 | end # one # 2 85 | end # two # 3 86 | EOM 87 | end 88 | 89 | it "shows ends of captured block" do 90 | lines = fixtures_dir.join("rexe.rb.txt").read.lines 91 | lines.delete_at(148 - 1) 92 | source = lines.join 93 | 94 | code_lines = CleanDocument.new(source: source).call.lines 95 | 96 | code_lines[0..75].each(&:mark_invisible) 97 | code_lines[77..].each(&:mark_invisible) 98 | expect(code_lines.join.strip).to eq("class Lookups") 99 | 100 | block = CodeBlock.new(lines: code_lines[76..149]) 101 | 102 | display = CaptureCodeContext.new( 103 | blocks: [block], 104 | code_lines: code_lines 105 | ) 106 | lines = display.call 107 | 108 | lines = lines.sort.map(&:original) 109 | expect(lines.join).to include(<<~EOM.indent(2)) 110 | class Lookups 111 | def format_requires 112 | end 113 | EOM 114 | end 115 | 116 | it "shows ends of captured block" do 117 | source = <<~EOM 118 | class Dog 119 | def bark 120 | puts "woof" 121 | end 122 | EOM 123 | 124 | code_lines = CleanDocument.new(source: source).call.lines 125 | block = CodeBlock.new(lines: code_lines) 126 | code_lines[1..].each(&:mark_invisible) 127 | 128 | expect(block.to_s.strip).to eq("class Dog") 129 | 130 | display = CaptureCodeContext.new( 131 | blocks: [block], 132 | code_lines: code_lines 133 | ) 134 | lines = display.call.sort.map(&:original) 135 | expect(lines.join).to eq(<<~EOM) 136 | class Dog 137 | def bark 138 | end 139 | EOM 140 | end 141 | 142 | it "captures surrounding context on falling indent" do 143 | source = <<~EOM 144 | class Blerg 145 | end 146 | 147 | class OH 148 | 149 | def hello 150 | it "foo" do 151 | end 152 | end 153 | 154 | class Zerg 155 | end 156 | EOM 157 | code_lines = CleanDocument.new(source: source).call.lines 158 | block = CodeBlock.new(lines: code_lines[6]) 159 | 160 | expect(block.to_s.strip).to eq('it "foo" do') 161 | 162 | display = CaptureCodeContext.new( 163 | blocks: [block], 164 | code_lines: code_lines 165 | ) 166 | lines = display.call.sort.map(&:original) 167 | expect(lines.join).to eq(<<~EOM) 168 | class OH 169 | def hello 170 | it "foo" do 171 | end 172 | end 173 | EOM 174 | end 175 | 176 | it "captures surrounding context on same indent" do 177 | source = <<~EOM 178 | class Blerg 179 | end 180 | class OH 181 | 182 | def nope 183 | end 184 | 185 | def lol 186 | end 187 | 188 | end # here 189 | 190 | def haha 191 | end 192 | 193 | def nope 194 | end 195 | end 196 | 197 | class Zerg 198 | end 199 | EOM 200 | 201 | code_lines = CleanDocument.new(source: source).call.lines 202 | block = CodeBlock.new(lines: code_lines[7..10]) 203 | expect(block.to_s).to eq(<<~EOM.indent(2)) 204 | def lol 205 | end 206 | 207 | end # here 208 | EOM 209 | 210 | code_context = CaptureCodeContext.new( 211 | blocks: [block], 212 | code_lines: code_lines 213 | ) 214 | 215 | lines = code_context.call 216 | out = DisplayCodeWithLineNumbers.new( 217 | lines: lines 218 | ).call 219 | 220 | expect(out).to eq(<<~EOM.indent(2)) 221 | 3 class OH 222 | 8 def lol 223 | 9 end 224 | 11 end # here 225 | 18 end 226 | EOM 227 | end 228 | end 229 | end 230 | -------------------------------------------------------------------------------- /spec/unit/cli_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | class FakeExit 7 | def initialize 8 | @called = false 9 | @value = nil 10 | end 11 | 12 | def exit(value = nil) 13 | @called = true 14 | @value = value 15 | end 16 | 17 | def called? 18 | @called 19 | end 20 | 21 | attr_reader :value 22 | end 23 | 24 | RSpec.describe Cli do 25 | it "parses valid code" do 26 | Dir.mktmpdir do |dir| 27 | dir = Pathname(dir) 28 | file = dir.join("script.rb") 29 | file.write("puts 'lol'") 30 | 31 | io = StringIO.new 32 | exit_obj = FakeExit.new 33 | Cli.new( 34 | io: io, 35 | argv: [file.to_s], 36 | exit_obj: exit_obj 37 | ).call 38 | 39 | expect(exit_obj.called?).to be_truthy 40 | expect(exit_obj.value).to eq(0) 41 | expect(io.string.strip).to eq("Syntax OK") 42 | end 43 | end 44 | 45 | it "parses invalid code" do 46 | file = fixtures_dir.join("this_project_extra_def.rb.txt") 47 | 48 | io = StringIO.new 49 | exit_obj = FakeExit.new 50 | Cli.new( 51 | io: io, 52 | argv: [file.to_s], 53 | exit_obj: exit_obj 54 | ).call 55 | 56 | out = io.string 57 | debug_display(out) 58 | 59 | expect(exit_obj.called?).to be_truthy 60 | expect(exit_obj.value).to eq(1) 61 | expect(out.strip).to include("> 36 def filename") 62 | end 63 | 64 | it "parses valid code with flags" do 65 | Dir.mktmpdir do |dir| 66 | dir = Pathname(dir) 67 | file = dir.join("script.rb") 68 | file.write("puts 'lol'") 69 | 70 | io = StringIO.new 71 | exit_obj = FakeExit.new 72 | cli = Cli.new( 73 | io: io, 74 | argv: ["--terminal", file.to_s], 75 | exit_obj: exit_obj 76 | ) 77 | cli.call 78 | 79 | expect(exit_obj.called?).to be_truthy 80 | expect(exit_obj.value).to eq(0) 81 | expect(cli.options[:terminal]).to be_truthy 82 | expect(io.string.strip).to eq("Syntax OK") 83 | end 84 | end 85 | 86 | it "errors when no file given" do 87 | io = StringIO.new 88 | exit_obj = FakeExit.new 89 | cli = Cli.new( 90 | io: io, 91 | argv: ["--terminal"], 92 | exit_obj: exit_obj 93 | ) 94 | cli.call 95 | 96 | expect(exit_obj.called?).to be_truthy 97 | expect(exit_obj.value).to eq(1) 98 | expect(io.string.strip).to eq("No file given") 99 | end 100 | 101 | it "errors when file does not exist" do 102 | io = StringIO.new 103 | exit_obj = FakeExit.new 104 | cli = Cli.new( 105 | io: io, 106 | argv: ["lol-i-d-o-not-ex-ist-yololo.txtblerglol"], 107 | exit_obj: exit_obj 108 | ) 109 | cli.call 110 | 111 | expect(exit_obj.called?).to be_truthy 112 | expect(exit_obj.value).to eq(1) 113 | expect(io.string.strip).to include("file not found:") 114 | end 115 | 116 | # We cannot execute the parser here 117 | # because it calls `exit` and it will exit 118 | # our tests, however we can assert that the 119 | # parser has the right value for version 120 | it "-v version" do 121 | io = StringIO.new 122 | exit_obj = FakeExit.new 123 | parser = Cli.new( 124 | io: io, 125 | argv: ["-v"], 126 | exit_obj: exit_obj 127 | ).parser 128 | 129 | expect(parser.version).to include(SyntaxSuggest::VERSION.to_s) 130 | end 131 | 132 | it "SYNTAX_SUGGEST_RECORD_DIR" do 133 | io = StringIO.new 134 | exit_obj = FakeExit.new 135 | cli = Cli.new( 136 | io: io, 137 | argv: [], 138 | env: {"SYNTAX_SUGGEST_RECORD_DIR" => "hahaha"}, 139 | exit_obj: exit_obj 140 | ).parse 141 | 142 | expect(exit_obj.called?).to be_falsey 143 | expect(cli.options[:record_dir]).to eq("hahaha") 144 | end 145 | 146 | it "--record-dir=" do 147 | io = StringIO.new 148 | exit_obj = FakeExit.new 149 | cli = Cli.new( 150 | io: io, 151 | argv: ["--record=lol"], 152 | exit_obj: exit_obj 153 | ).parse 154 | 155 | expect(exit_obj.called?).to be_falsey 156 | expect(cli.options[:record_dir]).to eq("lol") 157 | end 158 | 159 | it "terminal default to respecting TTY" do 160 | io = StringIO.new 161 | exit_obj = FakeExit.new 162 | cli = Cli.new( 163 | io: io, 164 | argv: [], 165 | exit_obj: exit_obj 166 | ).parse 167 | 168 | expect(exit_obj.called?).to be_falsey 169 | expect(cli.options[:terminal]).to eq(SyntaxSuggest::DEFAULT_VALUE) 170 | end 171 | 172 | it "--terminal" do 173 | io = StringIO.new 174 | exit_obj = FakeExit.new 175 | cli = Cli.new( 176 | io: io, 177 | argv: ["--terminal"], 178 | exit_obj: exit_obj 179 | ).parse 180 | 181 | expect(exit_obj.called?).to be_falsey 182 | expect(cli.options[:terminal]).to be_truthy 183 | end 184 | 185 | it "--no-terminal" do 186 | io = StringIO.new 187 | exit_obj = FakeExit.new 188 | cli = Cli.new( 189 | io: io, 190 | argv: ["--no-terminal"], 191 | exit_obj: exit_obj 192 | ).parse 193 | 194 | expect(exit_obj.called?).to be_falsey 195 | expect(cli.options[:terminal]).to be_falsey 196 | end 197 | 198 | it "--help outputs help" do 199 | io = StringIO.new 200 | exit_obj = FakeExit.new 201 | Cli.new( 202 | io: io, 203 | argv: ["--help"], 204 | exit_obj: exit_obj 205 | ).call 206 | 207 | expect(exit_obj.called?).to be_truthy 208 | expect(io.string).to include("Usage: syntax_suggest [options]") 209 | end 210 | 211 | it " outputs help" do 212 | io = StringIO.new 213 | exit_obj = FakeExit.new 214 | Cli.new( 215 | io: io, 216 | argv: [], 217 | exit_obj: exit_obj 218 | ).call 219 | 220 | expect(exit_obj.called?).to be_truthy 221 | expect(io.string).to include("Usage: syntax_suggest [options]") 222 | end 223 | end 224 | end 225 | -------------------------------------------------------------------------------- /lib/syntax_suggest/code_frontier.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # The main function of the frontier is to hold the edges of our search and to 5 | # evaluate when we can stop searching. 6 | 7 | # There are three main phases in the algorithm: 8 | # 9 | # 1. Sanitize/format input source 10 | # 2. Search for invalid blocks 11 | # 3. Format invalid blocks into something meaningful 12 | # 13 | # The Code frontier is a critical part of the second step 14 | # 15 | # ## Knowing where we've been 16 | # 17 | # Once a code block is generated it is added onto the frontier. Then it will be 18 | # sorted by indentation and frontier can be filtered. Large blocks that fully enclose a 19 | # smaller block will cause the smaller block to be evicted. 20 | # 21 | # CodeFrontier#<<(block) # Adds block to frontier 22 | # CodeFrontier#pop # Removes block from frontier 23 | # 24 | # ## Knowing where we can go 25 | # 26 | # Internally the frontier keeps track of "unvisited" lines which are exposed via `next_indent_line` 27 | # when called, this method returns, a line of code with the highest indentation. 28 | # 29 | # The returned line of code can be used to build a CodeBlock and then that code block 30 | # is added back to the frontier. Then, the lines are removed from the 31 | # "unvisited" so we don't double-create the same block. 32 | # 33 | # CodeFrontier#next_indent_line # Shows next line 34 | # CodeFrontier#register_indent_block(block) # Removes lines from unvisited 35 | # 36 | # ## Knowing when to stop 37 | # 38 | # The frontier knows how to check the entire document for a syntax error. When blocks 39 | # are added onto the frontier, they're removed from the document. When all code containing 40 | # syntax errors has been added to the frontier, the document will be parsable without a 41 | # syntax error and the search can stop. 42 | # 43 | # CodeFrontier#holds_all_syntax_errors? # Returns true when frontier holds all syntax errors 44 | # 45 | # ## Filtering false positives 46 | # 47 | # Once the search is completed, the frontier may have multiple blocks that do not contain 48 | # the syntax error. To limit the result to the smallest subset of "invalid blocks" call: 49 | # 50 | # CodeFrontier#detect_invalid_blocks 51 | # 52 | class CodeFrontier 53 | def initialize(code_lines:, unvisited: UnvisitedLines.new(code_lines: code_lines)) 54 | @code_lines = code_lines 55 | @unvisited = unvisited 56 | @queue = PriorityEngulfQueue.new 57 | 58 | @check_next = true 59 | end 60 | 61 | def count 62 | @queue.length 63 | end 64 | 65 | # Performance optimization 66 | # 67 | # Parsing with ripper is expensive 68 | # If we know we don't have any blocks with invalid 69 | # syntax, then we know we cannot have found 70 | # the incorrect syntax yet. 71 | # 72 | # When an invalid block is added onto the frontier 73 | # check document state 74 | private def can_skip_check? 75 | check_next = @check_next 76 | @check_next = false 77 | 78 | if check_next 79 | false 80 | else 81 | true 82 | end 83 | end 84 | 85 | # Returns true if the document is valid with all lines 86 | # removed. By default it checks all blocks in present in 87 | # the frontier array, but can be used for arbitrary arrays 88 | # of codeblocks as well 89 | def holds_all_syntax_errors?(block_array = @queue, can_cache: true) 90 | return false if can_cache && can_skip_check? 91 | 92 | without_lines = block_array.to_a.flat_map do |block| 93 | block.lines 94 | end 95 | 96 | SyntaxSuggest.valid_without?( 97 | without_lines: without_lines, 98 | code_lines: @code_lines 99 | ) 100 | end 101 | 102 | # Returns a code block with the largest indentation possible 103 | def pop 104 | @queue.pop 105 | end 106 | 107 | def next_indent_line 108 | @unvisited.peek 109 | end 110 | 111 | def expand? 112 | return false if @queue.empty? 113 | return true if @unvisited.empty? 114 | 115 | frontier_indent = @queue.peek.current_indent 116 | unvisited_indent = next_indent_line.indent 117 | 118 | if ENV["SYNTAX_SUGGEST_DEBUG"] 119 | puts "```" 120 | puts @queue.peek 121 | puts "```" 122 | puts " @frontier indent: #{frontier_indent}" 123 | puts " @unvisited indent: #{unvisited_indent}" 124 | end 125 | 126 | # Expand all blocks before moving to unvisited lines 127 | frontier_indent >= unvisited_indent 128 | end 129 | 130 | # Keeps track of what lines have been added to blocks and which are not yet 131 | # visited. 132 | def register_indent_block(block) 133 | @unvisited.visit_block(block) 134 | self 135 | end 136 | 137 | # When one element fully encapsulates another we remove the smaller 138 | # block from the frontier. This prevents double expansions and all-around 139 | # weird behavior. However this guarantee is quite expensive to maintain 140 | def register_engulf_block(block) 141 | end 142 | 143 | # Add a block to the frontier 144 | # 145 | # This method ensures the frontier always remains sorted (in indentation order) 146 | # and that each code block's lines are removed from the indentation hash so we 147 | # don't re-evaluate the same line multiple times. 148 | def <<(block) 149 | @unvisited.visit_block(block) 150 | 151 | @queue.push(block) 152 | 153 | @check_next = true if block.invalid? 154 | 155 | self 156 | end 157 | 158 | # Example: 159 | # 160 | # combination([:a, :b, :c, :d]) 161 | # # => [[:a], [:b], [:c], [:d], [:a, :b], [:a, :c], [:a, :d], [:b, :c], [:b, :d], [:c, :d], [:a, :b, :c], [:a, :b, :d], [:a, :c, :d], [:b, :c, :d], [:a, :b, :c, :d]] 162 | def self.combination(array) 163 | guesses = [] 164 | 1.upto(array.length).each do |size| 165 | guesses.concat(array.combination(size).to_a) 166 | end 167 | guesses 168 | end 169 | 170 | # Given that we know our syntax error exists somewhere in our frontier, we want to find 171 | # the smallest possible set of blocks that contain all the syntax errors 172 | def detect_invalid_blocks 173 | self.class.combination(@queue.to_a.select(&:invalid?)).detect do |block_array| 174 | holds_all_syntax_errors?(block_array, can_cache: false) 175 | end || [] 176 | end 177 | end 178 | end 179 | -------------------------------------------------------------------------------- /spec/integration/syntax_suggest_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe "Integration tests that don't spawn a process (like using the cli)" do 7 | before(:each) do 8 | skip "Benchmark is not available" unless defined?(::Benchmark) 9 | end 10 | 11 | it "does not timeout on massive files" do 12 | next unless ENV["SYNTAX_SUGGEST_TIMEOUT"] 13 | 14 | file = fixtures_dir.join("syntax_tree.rb.txt") 15 | lines = file.read.lines 16 | lines.delete_at(768 - 1) 17 | 18 | io = StringIO.new 19 | 20 | benchmark = Benchmark.measure do 21 | debug_perf do 22 | SyntaxSuggest.call( 23 | io: io, 24 | source: lines.join, 25 | filename: file 26 | ) 27 | end 28 | end 29 | 30 | debug_display(io.string) 31 | debug_display(benchmark) 32 | 33 | expect(io.string).to include(<<~EOM) 34 | 6 class SyntaxTree < Ripper 35 | 170 def self.parse(source) 36 | 174 end 37 | > 754 def on_args_add(arguments, argument) 38 | > 776 class ArgsAddBlock 39 | > 810 end 40 | 9233 end 41 | EOM 42 | end 43 | 44 | it "re-checks all block code, not just what's visible issues/95" do 45 | file = fixtures_dir.join("ruby_buildpack.rb.txt") 46 | io = StringIO.new 47 | 48 | debug_perf do 49 | benchmark = Benchmark.measure do 50 | SyntaxSuggest.call( 51 | io: io, 52 | source: file.read, 53 | filename: file 54 | ) 55 | end 56 | debug_display(io.string) 57 | debug_display(benchmark) 58 | end 59 | 60 | expect(io.string).to_not include("def ruby_install_binstub_path") 61 | expect(io.string).to include(<<~EOM) 62 | > 1067 def add_yarn_binary 63 | > 1068 return [] if yarn_preinstalled? 64 | > 1069 | 65 | > 1075 end 66 | EOM 67 | end 68 | 69 | it "returns good results on routes.rb" do 70 | source = fixtures_dir.join("routes.rb.txt").read 71 | 72 | io = StringIO.new 73 | SyntaxSuggest.call( 74 | io: io, 75 | source: source 76 | ) 77 | debug_display(io.string) 78 | 79 | expect(io.string).to include(<<~EOM) 80 | 1 Rails.application.routes.draw do 81 | > 113 namespace :admin do 82 | > 116 match "/foobar(*path)", via: :all, to: redirect { |_params, req| 83 | > 120 } 84 | 121 end 85 | EOM 86 | end 87 | 88 | it "handles multi-line-methods issues/64" do 89 | source = fixtures_dir.join("webmock.rb.txt").read 90 | 91 | io = StringIO.new 92 | SyntaxSuggest.call( 93 | io: io, 94 | source: source 95 | ) 96 | debug_display(io.string) 97 | 98 | expect(io.string).to include(<<~EOM) 99 | 1 describe "webmock tests" do 100 | 22 it "body" do 101 | 27 query = Cutlass::FunctionQuery.new( 102 | > 28 port: port 103 | > 29 body: body 104 | 30 ).call 105 | 34 end 106 | 35 end 107 | EOM 108 | end 109 | 110 | it "handles derailed output issues/50" do 111 | source = fixtures_dir.join("derailed_require_tree.rb.txt").read 112 | 113 | io = StringIO.new 114 | SyntaxSuggest.call( 115 | io: io, 116 | source: source 117 | ) 118 | debug_display(io.string) 119 | 120 | expect(io.string).to include(<<~EOM) 121 | 5 module DerailedBenchmarks 122 | 6 class RequireTree 123 | > 13 def initialize(name) 124 | > 18 def self.reset! 125 | > 25 end 126 | 73 end 127 | 74 end 128 | EOM 129 | end 130 | 131 | it "handles heredocs" do 132 | lines = fixtures_dir.join("rexe.rb.txt").read.lines 133 | lines.delete_at(85 - 1) 134 | io = StringIO.new 135 | SyntaxSuggest.call( 136 | io: io, 137 | source: lines.join 138 | ) 139 | 140 | out = io.string 141 | debug_display(out) 142 | 143 | expect(out).to include(<<~EOM) 144 | 16 class Rexe 145 | > 77 class Lookups 146 | > 78 def input_modes 147 | > 148 end 148 | 551 end 149 | EOM 150 | end 151 | 152 | it "rexe" do 153 | lines = fixtures_dir.join("rexe.rb.txt").read.lines 154 | lines.delete_at(148 - 1) 155 | source = lines.join 156 | 157 | io = StringIO.new 158 | SyntaxSuggest.call( 159 | io: io, 160 | source: source 161 | ) 162 | out = io.string 163 | expect(out).to include(<<~EOM) 164 | 16 class Rexe 165 | > 77 class Lookups 166 | > 140 def format_requires 167 | > 148 end 168 | 551 end 169 | EOM 170 | end 171 | 172 | it "ambiguous end" do 173 | source = <<~EOM 174 | def call # 0 175 | print "lol" # 1 176 | end # one # 2 177 | end # two # 3 178 | EOM 179 | io = StringIO.new 180 | SyntaxSuggest.call( 181 | io: io, 182 | source: source 183 | ) 184 | out = io.string 185 | expect(out).to include(<<~EOM) 186 | > 1 def call # 0 187 | > 3 end # one # 2 188 | > 4 end # two # 3 189 | EOM 190 | end 191 | 192 | it "simple regression" do 193 | source = <<~EOM 194 | class Dog 195 | def bark 196 | puts "woof" 197 | end 198 | EOM 199 | io = StringIO.new 200 | SyntaxSuggest.call( 201 | io: io, 202 | source: source 203 | ) 204 | out = io.string 205 | expect(out).to include(<<~EOM) 206 | > 1 class Dog 207 | > 2 def bark 208 | > 4 end 209 | EOM 210 | end 211 | 212 | it "empty else" do 213 | source = <<~EOM 214 | class Foo 215 | def foo 216 | if cond? 217 | foo 218 | else 219 | 220 | end 221 | end 222 | 223 | # ... 224 | 225 | def bar 226 | if @recv 227 | end_is_missing_here 228 | end 229 | end 230 | EOM 231 | 232 | io = StringIO.new 233 | SyntaxSuggest.call( 234 | io: io, 235 | source: source 236 | ) 237 | out = io.string 238 | expect(out).to include(<<~EOM) 239 | end_is_missing_here 240 | EOM 241 | end 242 | end 243 | end 244 | -------------------------------------------------------------------------------- /spec/integration/ruby_command_line_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | ruby = ENV.fetch("RUBY", "ruby") 7 | RSpec.describe "Requires with ruby cli" do 8 | it "namespaces all monkeypatched methods" do 9 | Dir.mktmpdir do |dir| 10 | tmpdir = Pathname(dir) 11 | script = tmpdir.join("script.rb") 12 | script.write <<~EOM 13 | puts Kernel.private_methods 14 | EOM 15 | 16 | syntax_suggest_methods_file = tmpdir.join("syntax_suggest_methods.txt") 17 | api_only_methods_file = tmpdir.join("api_only_methods.txt") 18 | kernel_methods_file = tmpdir.join("kernel_methods.txt") 19 | 20 | d_pid = Process.spawn("#{ruby} -I#{lib_dir} -rsyntax_suggest #{script} 2>&1 > #{syntax_suggest_methods_file}") 21 | k_pid = Process.spawn("#{ruby} #{script} 2>&1 >> #{kernel_methods_file}") 22 | r_pid = Process.spawn("#{ruby} -I#{lib_dir} -rsyntax_suggest/api #{script} 2>&1 > #{api_only_methods_file}") 23 | 24 | Process.wait(k_pid) 25 | Process.wait(d_pid) 26 | Process.wait(r_pid) 27 | 28 | kernel_methods_array = kernel_methods_file.read.strip.lines.map(&:strip) 29 | syntax_suggest_methods_array = syntax_suggest_methods_file.read.strip.lines.map(&:strip) 30 | api_only_methods_array = api_only_methods_file.read.strip.lines.map(&:strip) 31 | 32 | # In ruby 3.1.0-preview1 the `timeout` file is already required 33 | # we can remove it if it exists to normalize the output for 34 | # all ruby versions 35 | [syntax_suggest_methods_array, kernel_methods_array, api_only_methods_array].each do |array| 36 | array.delete("timeout") 37 | end 38 | 39 | methods = (syntax_suggest_methods_array - kernel_methods_array).sort 40 | if methods.any? 41 | expect(methods).to eq(["syntax_suggest_original_load", "syntax_suggest_original_require", "syntax_suggest_original_require_relative"]) 42 | end 43 | 44 | methods = (api_only_methods_array - kernel_methods_array).sort 45 | expect(methods).to eq([]) 46 | end 47 | end 48 | 49 | # Since Ruby 3.2 includes syntax_suggest as a default gem, we might accidentally 50 | # be requiring the default gem instead of this library under test. Assert that's 51 | # not the case 52 | it "tests current version of syntax_suggest" do 53 | Dir.mktmpdir do |dir| 54 | tmpdir = Pathname(dir) 55 | script = tmpdir.join("script.rb") 56 | contents = <<~'EOM' 57 | puts "suggest_version is #{SyntaxSuggest::VERSION}" 58 | EOM 59 | script.write(contents) 60 | 61 | out = `#{ruby} -I#{lib_dir} -rsyntax_suggest/version #{script} 2>&1` 62 | 63 | expect(out).to include("suggest_version is #{SyntaxSuggest::VERSION}").once 64 | end 65 | end 66 | 67 | it "detects require error and adds a message with auto mode" do 68 | Dir.mktmpdir do |dir| 69 | tmpdir = Pathname(dir) 70 | script = tmpdir.join("script.rb") 71 | script.write <<~EOM 72 | describe "things" do 73 | it "blerg" do 74 | end 75 | 76 | it "flerg" 77 | end 78 | 79 | it "zlerg" do 80 | end 81 | end 82 | EOM 83 | 84 | require_rb = tmpdir.join("require.rb") 85 | require_rb.write <<~EOM 86 | load "#{script.expand_path}" 87 | EOM 88 | 89 | out = `#{ruby} -I#{lib_dir} -rsyntax_suggest #{require_rb} 2>&1` 90 | 91 | expect($?.success?).to be_falsey 92 | expect(out).to include('> 5 it "flerg"').once 93 | end 94 | end 95 | 96 | it "gem can be tested when executing on Ruby with default gem included" do 97 | skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") 98 | 99 | out = `#{ruby} -I#{lib_dir} -rsyntax_suggest -e "puts SyntaxError.instance_method(:detailed_message).source_location" 2>&1` 100 | 101 | expect($?.success?).to be_truthy 102 | expect(out).to include(lib_dir.join("syntax_suggest").join("core_ext.rb").to_s).once 103 | end 104 | 105 | it "annotates a syntax error in Ruby 3.2+ when require is not used" do 106 | skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") 107 | 108 | Dir.mktmpdir do |dir| 109 | tmpdir = Pathname(dir) 110 | script = tmpdir.join("script.rb") 111 | script.write <<~EOM 112 | describe "things" do 113 | it "blerg" do 114 | end 115 | 116 | it "flerg" 117 | end 118 | 119 | it "zlerg" do 120 | end 121 | end 122 | EOM 123 | 124 | out = `#{ruby} -I#{lib_dir} -rsyntax_suggest #{script} 2>&1` 125 | 126 | expect($?.success?).to be_falsey 127 | expect(out).to include('> 5 it "flerg"').once 128 | end 129 | end 130 | 131 | it "does not load internals into memory if no syntax error" do 132 | Dir.mktmpdir do |dir| 133 | tmpdir = Pathname(dir) 134 | script = tmpdir.join("script.rb") 135 | script.write <<~EOM 136 | class Dog 137 | end 138 | 139 | if defined?(SyntaxSuggest::DEFAULT_VALUE) 140 | puts "SyntaxSuggest is loaded" 141 | else 142 | puts "SyntaxSuggest is NOT loaded" 143 | end 144 | EOM 145 | 146 | require_rb = tmpdir.join("require.rb") 147 | require_rb.write <<~EOM 148 | load "#{script.expand_path}" 149 | EOM 150 | 151 | out = `#{ruby} -I#{lib_dir} -rsyntax_suggest #{require_rb} 2>&1` 152 | 153 | expect($?.success?).to be_truthy 154 | expect(out).to include("SyntaxSuggest is NOT loaded").once 155 | end 156 | end 157 | 158 | it "ignores eval" do 159 | Dir.mktmpdir do |dir| 160 | tmpdir = Pathname(dir) 161 | script = tmpdir.join("script.rb") 162 | script.write <<~EOM 163 | $stderr = STDOUT 164 | eval("def lol") 165 | EOM 166 | 167 | out = `#{ruby} -I#{lib_dir} -rsyntax_suggest #{script} 2>&1` 168 | 169 | expect($?.success?).to be_falsey 170 | expect(out).to match(/\(eval.*\):1/) 171 | 172 | expect(out).to_not include("SyntaxSuggest") 173 | expect(out).to_not include("Could not find filename") 174 | end 175 | end 176 | 177 | it "does not say 'syntax ok' when a syntax error fires" do 178 | Dir.mktmpdir do |dir| 179 | tmpdir = Pathname(dir) 180 | script = tmpdir.join("script.rb") 181 | script.write <<~EOM 182 | break 183 | EOM 184 | 185 | out = `#{ruby} -I#{lib_dir} -rsyntax_suggest -e "require_relative '#{script}'" 2>&1` 186 | 187 | expect($?.success?).to be_falsey 188 | expect(out.downcase).to_not include("syntax ok") 189 | expect(out).to include("Invalid break") 190 | end 191 | end 192 | end 193 | end 194 | -------------------------------------------------------------------------------- /spec/unit/explain_syntax_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe "ExplainSyntax" do 7 | it "handles shorthand syntaxes with non-bracket characters" do 8 | source = <<~EOM 9 | %Q* lol 10 | EOM 11 | 12 | explain = ExplainSyntax.new( 13 | code_lines: CodeLine.from_source(source) 14 | ).call 15 | 16 | expect(explain.missing).to eq([]) 17 | expect(explain.errors.join.strip).to_not be_empty 18 | end 19 | 20 | it "handles %w[]" do 21 | source = <<~EOM 22 | node.is_a?(Op) && %w[| ||].include?(node.value) && 23 | EOM 24 | 25 | explain = ExplainSyntax.new( 26 | code_lines: CodeLine.from_source(source) 27 | ).call 28 | 29 | expect(explain.missing).to eq([]) 30 | end 31 | 32 | it "doesn't falsely identify strings or symbols as critical chars" do 33 | source = <<~EOM 34 | a = ['(', '{', '[', '|'] 35 | EOM 36 | 37 | explain = ExplainSyntax.new( 38 | code_lines: CodeLine.from_source(source) 39 | ).call 40 | 41 | expect(explain.missing).to eq([]) 42 | 43 | source = <<~EOM 44 | a = [:'(', :'{', :'[', :'|'] 45 | EOM 46 | 47 | explain = ExplainSyntax.new( 48 | code_lines: CodeLine.from_source(source) 49 | ).call 50 | 51 | expect(explain.missing).to eq([]) 52 | end 53 | 54 | it "finds missing |" do 55 | source = <<~EOM 56 | Foo.call do | 57 | end 58 | EOM 59 | 60 | explain = ExplainSyntax.new( 61 | code_lines: CodeLine.from_source(source) 62 | ).call 63 | 64 | expect(explain.missing).to eq(["|"]) 65 | expect(explain.errors).to eq([explain.why("|")]) 66 | end 67 | 68 | it "finds missing {" do 69 | source = <<~EOM 70 | class Cat 71 | lol = { 72 | end 73 | EOM 74 | 75 | explain = ExplainSyntax.new( 76 | code_lines: CodeLine.from_source(source) 77 | ).call 78 | 79 | expect(explain.missing).to eq(["}"]) 80 | expect(explain.errors).to eq([explain.why("}")]) 81 | end 82 | 83 | it "finds missing }" do 84 | source = <<~EOM 85 | def foo 86 | lol = "foo" => :bar } 87 | end 88 | EOM 89 | 90 | explain = ExplainSyntax.new( 91 | code_lines: CodeLine.from_source(source) 92 | ).call 93 | 94 | expect(explain.missing).to eq(["{"]) 95 | expect(explain.errors).to eq([explain.why("{")]) 96 | end 97 | 98 | it "finds missing [" do 99 | source = <<~EOM 100 | class Cat 101 | lol = [ 102 | end 103 | EOM 104 | 105 | explain = ExplainSyntax.new( 106 | code_lines: CodeLine.from_source(source) 107 | ).call 108 | 109 | expect(explain.missing).to eq(["]"]) 110 | expect(explain.errors).to eq([explain.why("]")]) 111 | end 112 | 113 | it "finds missing ]" do 114 | source = <<~EOM 115 | def foo 116 | lol = ] 117 | end 118 | EOM 119 | 120 | explain = ExplainSyntax.new( 121 | code_lines: CodeLine.from_source(source) 122 | ).call 123 | 124 | expect(explain.missing).to eq(["["]) 125 | expect(explain.errors).to eq([explain.why("[")]) 126 | end 127 | 128 | it "finds missing (" do 129 | source = "def initialize; ); end" 130 | 131 | explain = ExplainSyntax.new( 132 | code_lines: CodeLine.from_source(source) 133 | ).call 134 | 135 | expect(explain.missing).to eq(["("]) 136 | expect(explain.errors).to eq([explain.why("(")]) 137 | end 138 | 139 | it "finds missing )" do 140 | source = "def initialize; (; end" 141 | 142 | explain = ExplainSyntax.new( 143 | code_lines: CodeLine.from_source(source) 144 | ).call 145 | 146 | expect(explain.missing).to eq([")"]) 147 | expect(explain.errors).to eq([explain.why(")")]) 148 | end 149 | 150 | it "finds missing keyword" do 151 | source = <<~EOM 152 | class Cat 153 | end 154 | end 155 | EOM 156 | 157 | explain = ExplainSyntax.new( 158 | code_lines: CodeLine.from_source(source) 159 | ).call 160 | 161 | expect(explain.missing).to eq(["keyword"]) 162 | expect(explain.errors).to eq([explain.why("keyword")]) 163 | end 164 | 165 | it "finds missing end" do 166 | source = <<~EOM 167 | class Cat 168 | def meow 169 | end 170 | EOM 171 | 172 | explain = ExplainSyntax.new( 173 | code_lines: CodeLine.from_source(source) 174 | ).call 175 | 176 | expect(explain.missing).to eq(["end"]) 177 | expect(explain.errors).to eq([explain.why("end")]) 178 | end 179 | 180 | it "falls back to ripper on unknown errors" do 181 | source = <<~EOM 182 | class Cat 183 | def meow 184 | 1 * 185 | end 186 | end 187 | EOM 188 | 189 | explain = ExplainSyntax.new( 190 | code_lines: CodeLine.from_source(source) 191 | ).call 192 | 193 | expect(explain.missing).to eq([]) 194 | expect(explain.errors).to eq(GetParseErrors.errors(source)) 195 | end 196 | 197 | it "handles an unexpected rescue" do 198 | source = <<~EOM 199 | def foo 200 | if bar 201 | "baz" 202 | else 203 | "foo" 204 | rescue FooBar 205 | nil 206 | end 207 | EOM 208 | 209 | explain = ExplainSyntax.new( 210 | code_lines: CodeLine.from_source(source) 211 | ).call 212 | 213 | expect(explain.missing).to eq(["end"]) 214 | end 215 | 216 | # String embeds are `"#{foo} <-- here` 217 | # 218 | # We need to count a `#{` as a `{` 219 | # otherwise it will report that we are 220 | # missing a curly when we are using valid 221 | # string embed syntax 222 | it "is not confused by valid string embed" do 223 | source = <<~'EOM' 224 | foo = "#{hello}" 225 | EOM 226 | 227 | explain = ExplainSyntax.new( 228 | code_lines: CodeLine.from_source(source) 229 | ).call 230 | expect(explain.missing).to eq([]) 231 | end 232 | 233 | # Missing string embed beginnings are not a 234 | # syntax error. i.e. `"foo}"` or `"{foo}` or "#foo}" 235 | # would just be strings with extra characters. 236 | # 237 | # However missing the end curly will trigger 238 | # an error: i.e. `"#{foo` 239 | # 240 | # String embed beginning is a `#{` rather than 241 | # a `{`, make sure we handle that case and 242 | # report the correct missing `}` diagnosis 243 | it "finds missing string embed end" do 244 | source = <<~'EOM' 245 | "#{foo 246 | EOM 247 | 248 | explain = ExplainSyntax.new( 249 | code_lines: CodeLine.from_source(source) 250 | ).call 251 | 252 | expect(explain.missing).to eq(["}"]) 253 | end 254 | end 255 | end 256 | -------------------------------------------------------------------------------- /lib/syntax_suggest/api.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "version" 4 | 5 | require "tmpdir" 6 | require "stringio" 7 | require "pathname" 8 | require "timeout" 9 | 10 | # We need Ripper loaded for `Prism.lex_compat` even if we're using Prism 11 | # for lexing and parsing 12 | require "ripper" 13 | 14 | # Prism is the new parser, replacing Ripper 15 | # 16 | # We need to "dual boot" both for now because syntax_suggest 17 | # supports older rubies that do not ship with syntax suggest. 18 | # 19 | # We also need the ability to control loading of this library 20 | # so we can test that both modes work correctly in CI. 21 | if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"]) 22 | warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}" 23 | else 24 | begin 25 | require "prism" 26 | rescue LoadError 27 | end 28 | end 29 | 30 | module SyntaxSuggest 31 | # Used to indicate a default value that cannot 32 | # be confused with another input. 33 | DEFAULT_VALUE = Object.new.freeze 34 | 35 | class Error < StandardError; end 36 | TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i 37 | 38 | # SyntaxSuggest.use_prism_parser? [Private] 39 | # 40 | # Tells us if the prism parser is available for use 41 | # or if we should fallback to `Ripper` 42 | def self.use_prism_parser? 43 | defined?(Prism) 44 | end 45 | 46 | # SyntaxSuggest.handle_error [Public] 47 | # 48 | # Takes a `SyntaxError` exception, uses the 49 | # error message to locate the file. Then the file 50 | # will be analyzed to find the location of the syntax 51 | # error and emit that location to stderr. 52 | # 53 | # Example: 54 | # 55 | # begin 56 | # require 'bad_file' 57 | # rescue => e 58 | # SyntaxSuggest.handle_error(e) 59 | # end 60 | # 61 | # By default it will re-raise the exception unless 62 | # `re_raise: false`. The message output location 63 | # can be configured using the `io: $stderr` input. 64 | # 65 | # If a valid filename cannot be determined, the original 66 | # exception will be re-raised (even with 67 | # `re_raise: false`). 68 | def self.handle_error(e, re_raise: true, io: $stderr) 69 | unless e.is_a?(SyntaxError) 70 | io.puts("SyntaxSuggest: Must pass a SyntaxError, got: #{e.class}") 71 | raise e 72 | end 73 | 74 | file = PathnameFromMessage.new(e.message, io: io).call.name 75 | raise e unless file 76 | 77 | io.sync = true 78 | 79 | call( 80 | io: io, 81 | source: file.read, 82 | filename: file 83 | ) 84 | 85 | raise e if re_raise 86 | end 87 | 88 | # SyntaxSuggest.call [Private] 89 | # 90 | # Main private interface 91 | def self.call(source:, filename: DEFAULT_VALUE, terminal: DEFAULT_VALUE, record_dir: DEFAULT_VALUE, timeout: TIMEOUT_DEFAULT, io: $stderr) 92 | search = nil 93 | filename = nil if filename == DEFAULT_VALUE 94 | Timeout.timeout(timeout) do 95 | record_dir ||= ENV["DEBUG"] ? "tmp" : nil 96 | search = CodeSearch.new(source, record_dir: record_dir).call 97 | end 98 | 99 | blocks = search.invalid_blocks 100 | DisplayInvalidBlocks.new( 101 | io: io, 102 | blocks: blocks, 103 | filename: filename, 104 | terminal: terminal, 105 | code_lines: search.code_lines 106 | ).call 107 | rescue Timeout::Error => e 108 | io.puts "Search timed out SYNTAX_SUGGEST_TIMEOUT=#{timeout}, run with SYNTAX_SUGGEST_DEBUG=1 for more info" 109 | io.puts e.backtrace.first(3).join($/) 110 | end 111 | 112 | # SyntaxSuggest.record_dir [Private] 113 | # 114 | # Used to generate a unique directory to record 115 | # search steps for debugging 116 | def self.record_dir(dir) 117 | time = Time.now.strftime("%Y-%m-%d-%H-%M-%s-%N") 118 | dir = Pathname(dir) 119 | dir.join(time).tap { |path| 120 | path.mkpath 121 | alias_dir = dir.join("last") 122 | FileUtils.rm_rf(alias_dir) if alias_dir.exist? 123 | FileUtils.ln_sf(time, alias_dir) 124 | } 125 | end 126 | 127 | # SyntaxSuggest.valid_without? [Private] 128 | # 129 | # This will tell you if the `code_lines` would be valid 130 | # if you removed the `without_lines`. In short it's a 131 | # way to detect if we've found the lines with syntax errors 132 | # in our document yet. 133 | # 134 | # code_lines = [ 135 | # CodeLine.new(line: "def foo\n", index: 0) 136 | # CodeLine.new(line: " def bar\n", index: 1) 137 | # CodeLine.new(line: "end\n", index: 2) 138 | # ] 139 | # 140 | # SyntaxSuggest.valid_without?( 141 | # without_lines: code_lines[1], 142 | # code_lines: code_lines 143 | # ) # => true 144 | # 145 | # SyntaxSuggest.valid?(code_lines) # => false 146 | def self.valid_without?(without_lines:, code_lines:) 147 | lines = code_lines - Array(without_lines).flatten 148 | 149 | lines.empty? || valid?(lines) 150 | end 151 | 152 | # SyntaxSuggest.invalid? [Private] 153 | # 154 | # Opposite of `SyntaxSuggest.valid?` 155 | if defined?(Prism) 156 | def self.invalid?(source) 157 | source = source.join if source.is_a?(Array) 158 | source = source.to_s 159 | 160 | Prism.parse(source).failure? 161 | end 162 | else 163 | def self.invalid?(source) 164 | source = source.join if source.is_a?(Array) 165 | source = source.to_s 166 | 167 | Ripper.new(source).tap(&:parse).error? 168 | end 169 | end 170 | 171 | # SyntaxSuggest.valid? [Private] 172 | # 173 | # Returns truthy if a given input source is valid syntax 174 | # 175 | # SyntaxSuggest.valid?(<<~EOM) # => true 176 | # def foo 177 | # end 178 | # EOM 179 | # 180 | # SyntaxSuggest.valid?(<<~EOM) # => false 181 | # def foo 182 | # def bar # Syntax error here 183 | # end 184 | # EOM 185 | # 186 | # You can also pass in an array of lines and they'll be 187 | # joined before evaluating 188 | # 189 | # SyntaxSuggest.valid?( 190 | # [ 191 | # "def foo\n", 192 | # "end\n" 193 | # ] 194 | # ) # => true 195 | # 196 | # SyntaxSuggest.valid?( 197 | # [ 198 | # "def foo\n", 199 | # " def bar\n", # Syntax error here 200 | # "end\n" 201 | # ] 202 | # ) # => false 203 | # 204 | # As an FYI the CodeLine class instances respond to `to_s` 205 | # so passing a CodeLine in as an object or as an array 206 | # will convert it to it's code representation. 207 | def self.valid?(source) 208 | !invalid?(source) 209 | end 210 | end 211 | 212 | # Integration 213 | require_relative "cli" 214 | 215 | # Core logic 216 | require_relative "code_search" 217 | require_relative "code_frontier" 218 | require_relative "explain_syntax" 219 | require_relative "clean_document" 220 | 221 | # Helpers 222 | require_relative "lex_all" 223 | require_relative "code_line" 224 | require_relative "code_block" 225 | require_relative "block_expand" 226 | require_relative "mini_stringio" 227 | require_relative "priority_queue" 228 | require_relative "unvisited_lines" 229 | require_relative "around_block_scan" 230 | require_relative "priority_engulf_queue" 231 | require_relative "pathname_from_message" 232 | require_relative "display_invalid_blocks" 233 | require_relative "parse_blocks_from_indent_line" 234 | -------------------------------------------------------------------------------- /lib/syntax_suggest/code_line.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Represents a single line of code of a given source file 5 | # 6 | # This object contains metadata about the line such as 7 | # amount of indentation, if it is empty or not, and 8 | # lexical data, such as if it has an `end` or a keyword 9 | # in it. 10 | # 11 | # Visibility of lines can be toggled off. Marking a line as invisible 12 | # indicates that it should not be used for syntax checks. 13 | # It's functionally the same as commenting it out. 14 | # 15 | # Example: 16 | # 17 | # line = CodeLine.from_source("def foo\n").first 18 | # line.number => 1 19 | # line.empty? # => false 20 | # line.visible? # => true 21 | # line.mark_invisible 22 | # line.visible? # => false 23 | # 24 | class CodeLine 25 | TRAILING_SLASH = ("\\" + $/).freeze 26 | 27 | # Returns an array of CodeLine objects 28 | # from the source string 29 | def self.from_source(source, lines: nil) 30 | lines ||= source.lines 31 | lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex } 32 | lines.map.with_index do |line, index| 33 | CodeLine.new( 34 | line: line, 35 | index: index, 36 | lex: lex_array_for_line[index + 1] 37 | ) 38 | end 39 | end 40 | 41 | attr_reader :line, :index, :lex, :line_number, :indent 42 | def initialize(line:, index:, lex:) 43 | @lex = lex 44 | @line = line 45 | @index = index 46 | @original = line 47 | @line_number = @index + 1 48 | strip_line = line.dup 49 | strip_line.lstrip! 50 | 51 | @indent = if (@empty = strip_line.empty?) 52 | line.length - 1 # Newline removed from strip_line is not "whitespace" 53 | else 54 | line.length - strip_line.length 55 | end 56 | 57 | set_kw_end 58 | end 59 | 60 | # Used for stable sort via indentation level 61 | # 62 | # Ruby's sort is not "stable" meaning that when 63 | # multiple elements have the same value, they are 64 | # not guaranteed to return in the same order they 65 | # were put in. 66 | # 67 | # So when multiple code lines have the same indentation 68 | # level, they're sorted by their index value which is unique 69 | # and consistent. 70 | # 71 | # This is mostly needed for consistency of the test suite 72 | def indent_index 73 | @indent_index ||= [indent, index] 74 | end 75 | alias_method :number, :line_number 76 | 77 | # Returns true if the code line is determined 78 | # to contain a keyword that matches with an `end` 79 | # 80 | # For example: `def`, `do`, `begin`, `ensure`, etc. 81 | def is_kw? 82 | @is_kw 83 | end 84 | 85 | # Returns true if the code line is determined 86 | # to contain an `end` keyword 87 | def is_end? 88 | @is_end 89 | end 90 | 91 | # Used to hide lines 92 | # 93 | # The search alorithm will group lines into blocks 94 | # then if those blocks are determined to represent 95 | # valid code they will be hidden 96 | def mark_invisible 97 | @line = "" 98 | end 99 | 100 | # Means the line was marked as "invisible" 101 | # Confusingly, "empty" lines are visible...they 102 | # just don't contain any source code other than a newline ("\n"). 103 | def visible? 104 | !line.empty? 105 | end 106 | 107 | # Opposite or `visible?` (note: different than `empty?`) 108 | def hidden? 109 | !visible? 110 | end 111 | 112 | # An `empty?` line is one that was originally left 113 | # empty in the source code, while a "hidden" line 114 | # is one that we've since marked as "invisible" 115 | def empty? 116 | @empty 117 | end 118 | 119 | # Opposite of `empty?` (note: different than `visible?`) 120 | def not_empty? 121 | !empty? 122 | end 123 | 124 | # Renders the given line 125 | # 126 | # Also allows us to represent source code as 127 | # an array of code lines. 128 | # 129 | # When we have an array of code line elements 130 | # calling `join` on the array will call `to_s` 131 | # on each element, which essentially converts 132 | # it back into it's original source string. 133 | def to_s 134 | line 135 | end 136 | 137 | # When the code line is marked invisible 138 | # we retain the original value of it's line 139 | # this is useful for debugging and for 140 | # showing extra context 141 | # 142 | # DisplayCodeWithLineNumbers will render 143 | # all lines given to it, not just visible 144 | # lines, it uses the original method to 145 | # obtain them. 146 | attr_reader :original 147 | 148 | # Comparison operator, needed for equality 149 | # and sorting 150 | def <=>(other) 151 | index <=> other.index 152 | end 153 | 154 | # [Not stable API] 155 | # 156 | # Lines that have a `on_ignored_nl` type token and NOT 157 | # a `BEG` type seem to be a good proxy for the ability 158 | # to join multiple lines into one. 159 | # 160 | # This predicate method is used to determine when those 161 | # two criteria have been met. 162 | # 163 | # The one known case this doesn't handle is: 164 | # 165 | # Ripper.lex <<~EOM 166 | # a && 167 | # b || 168 | # c 169 | # EOM 170 | # 171 | # For some reason this introduces `on_ignore_newline` but with BEG type 172 | def ignore_newline_not_beg? 173 | @ignore_newline_not_beg 174 | end 175 | 176 | # Determines if the given line has a trailing slash 177 | # 178 | # lines = CodeLine.from_source(<<~EOM) 179 | # it "foo" \ 180 | # EOM 181 | # expect(lines.first.trailing_slash?).to eq(true) 182 | # 183 | if SyntaxSuggest.use_prism_parser? 184 | def trailing_slash? 185 | last = @lex.last 186 | last&.type == :on_tstring_end 187 | end 188 | else 189 | def trailing_slash? 190 | last = @lex.last 191 | return false unless last 192 | return false unless last.type == :on_sp 193 | 194 | last.token == TRAILING_SLASH 195 | end 196 | end 197 | 198 | # Endless method detection 199 | # 200 | # From https://github.com/ruby/irb/commit/826ae909c9c93a2ddca6f9cfcd9c94dbf53d44ab 201 | # Detecting a "oneliner" seems to need a state machine. 202 | # This can be done by looking mostly at the "state" (last value): 203 | # 204 | # ENDFN -> BEG (token = '=' ) -> END 205 | # 206 | private def set_kw_end 207 | oneliner_count = 0 208 | in_oneliner_def = nil 209 | 210 | kw_count = 0 211 | end_count = 0 212 | 213 | @ignore_newline_not_beg = false 214 | @lex.each do |lex| 215 | kw_count += 1 if lex.is_kw? 216 | end_count += 1 if lex.is_end? 217 | 218 | if lex.type == :on_ignored_nl 219 | @ignore_newline_not_beg = !lex.expr_beg? 220 | end 221 | 222 | if in_oneliner_def.nil? 223 | in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN) 224 | elsif lex.state.allbits?(Ripper::EXPR_ENDFN) 225 | # Continue 226 | elsif lex.state.allbits?(Ripper::EXPR_BEG) 227 | in_oneliner_def = :BODY if lex.token == "=" 228 | elsif lex.state.allbits?(Ripper::EXPR_END) 229 | # We found an endless method, count it 230 | oneliner_count += 1 if in_oneliner_def == :BODY 231 | 232 | in_oneliner_def = nil 233 | else 234 | in_oneliner_def = nil 235 | end 236 | end 237 | 238 | kw_count -= oneliner_count 239 | 240 | @is_kw = (kw_count - end_count) > 0 241 | @is_end = (end_count - kw_count) > 0 242 | end 243 | end 244 | end 245 | -------------------------------------------------------------------------------- /lib/syntax_suggest/capture_code_context.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | module Capture 5 | end 6 | end 7 | 8 | require_relative "capture/falling_indent_lines" 9 | require_relative "capture/before_after_keyword_ends" 10 | 11 | module SyntaxSuggest 12 | # Turns a "invalid block(s)" into useful context 13 | # 14 | # There are three main phases in the algorithm: 15 | # 16 | # 1. Sanitize/format input source 17 | # 2. Search for invalid blocks 18 | # 3. Format invalid blocks into something meaningful 19 | # 20 | # This class handles the third part. 21 | # 22 | # The algorithm is very good at capturing all of a syntax 23 | # error in a single block in number 2, however the results 24 | # can contain ambiguities. Humans are good at pattern matching 25 | # and filtering and can mentally remove extraneous data, but 26 | # they can't add extra data that's not present. 27 | # 28 | # In the case of known ambiguious cases, this class adds context 29 | # back to the ambiguity so the programmer has full information. 30 | # 31 | # Beyond handling these ambiguities, it also captures surrounding 32 | # code context information: 33 | # 34 | # puts block.to_s # => "def bark" 35 | # 36 | # context = CaptureCodeContext.new( 37 | # blocks: block, 38 | # code_lines: code_lines 39 | # ) 40 | # 41 | # lines = context.call.map(&:original) 42 | # puts lines.join 43 | # # => 44 | # class Dog 45 | # def bark 46 | # end 47 | # 48 | class CaptureCodeContext 49 | attr_reader :code_lines 50 | 51 | def initialize(blocks:, code_lines:) 52 | @blocks = Array(blocks) 53 | @code_lines = code_lines 54 | @visible_lines = @blocks.map(&:visible_lines).flatten 55 | @lines_to_output = @visible_lines.dup 56 | end 57 | 58 | def call 59 | @blocks.each do |block| 60 | capture_first_kw_end_same_indent(block) 61 | capture_last_end_same_indent(block) 62 | capture_before_after_kws(block) 63 | capture_falling_indent(block) 64 | end 65 | 66 | sorted_lines 67 | end 68 | 69 | def sorted_lines 70 | @lines_to_output.select!(&:not_empty?) 71 | @lines_to_output.uniq! 72 | @lines_to_output.sort! 73 | 74 | @lines_to_output 75 | end 76 | 77 | # Shows the context around code provided by "falling" indentation 78 | # 79 | # Converts: 80 | # 81 | # it "foo" do 82 | # 83 | # into: 84 | # 85 | # class OH 86 | # def hello 87 | # it "foo" do 88 | # end 89 | # end 90 | # 91 | def capture_falling_indent(block) 92 | Capture::FallingIndentLines.new( 93 | block: block, 94 | code_lines: @code_lines 95 | ).call do |line| 96 | @lines_to_output << line 97 | end 98 | end 99 | 100 | # Shows surrounding kw/end pairs 101 | # 102 | # The purpose of showing these extra pairs is due to cases 103 | # of ambiguity when only one visible line is matched. 104 | # 105 | # For example: 106 | # 107 | # 1 class Dog 108 | # 2 def bark 109 | # 4 def eat 110 | # 5 end 111 | # 6 end 112 | # 113 | # In this case either line 2 could be missing an `end` or 114 | # line 4 was an extra line added by mistake (it happens). 115 | # 116 | # When we detect the above problem it shows the issue 117 | # as only being on line 2 118 | # 119 | # 2 def bark 120 | # 121 | # Showing "neighbor" keyword pairs gives extra context: 122 | # 123 | # 2 def bark 124 | # 4 def eat 125 | # 5 end 126 | # 127 | def capture_before_after_kws(block) 128 | return unless block.visible_lines.count == 1 129 | 130 | around_lines = Capture::BeforeAfterKeywordEnds.new( 131 | code_lines: @code_lines, 132 | block: block 133 | ).call 134 | 135 | around_lines -= block.lines 136 | 137 | @lines_to_output.concat(around_lines) 138 | end 139 | 140 | # When there is an invalid block with a keyword 141 | # missing an end right before another end, 142 | # it is unclear where which keyword is missing the 143 | # end 144 | # 145 | # Take this example: 146 | # 147 | # class Dog # 1 148 | # def bark # 2 149 | # puts "woof" # 3 150 | # end # 4 151 | # 152 | # However due to https://github.com/ruby/syntax_suggest/issues/32 153 | # the problem line will be identified as: 154 | # 155 | # > class Dog # 1 156 | # 157 | # Because lines 2, 3, and 4 are technically valid code and are expanded 158 | # first, deemed valid, and hidden. We need to un-hide the matching end 159 | # line 4. Also work backwards and if there's a mis-matched keyword, show it 160 | # too 161 | def capture_last_end_same_indent(block) 162 | return if block.visible_lines.length != 1 163 | return unless block.visible_lines.first.is_kw? 164 | 165 | visible_line = block.visible_lines.first 166 | lines = @code_lines[visible_line.index..block.lines.last.index] 167 | 168 | # Find first end with same indent 169 | # (this would return line 4) 170 | # 171 | # end # 4 172 | matching_end = lines.detect { |line| line.indent == block.current_indent && line.is_end? } 173 | return unless matching_end 174 | 175 | @lines_to_output << matching_end 176 | 177 | # Work backwards from the end to 178 | # see if there are mis-matched 179 | # keyword/end pairs 180 | # 181 | # Return the first mis-matched keyword 182 | # this would find line 2 183 | # 184 | # def bark # 2 185 | # puts "woof" # 3 186 | # end # 4 187 | end_count = 0 188 | kw_count = 0 189 | kw_line = @code_lines[visible_line.index..matching_end.index].reverse.detect do |line| 190 | end_count += 1 if line.is_end? 191 | kw_count += 1 if line.is_kw? 192 | 193 | !kw_count.zero? && kw_count >= end_count 194 | end 195 | return unless kw_line 196 | @lines_to_output << kw_line 197 | end 198 | 199 | # The logical inverse of `capture_last_end_same_indent` 200 | # 201 | # When there is an invalid block with an `end` 202 | # missing a keyword right after another `end`, 203 | # it is unclear where which end is missing the 204 | # keyword. 205 | # 206 | # Take this example: 207 | # 208 | # class Dog # 1 209 | # puts "woof" # 2 210 | # end # 3 211 | # end # 4 212 | # 213 | # the problem line will be identified as: 214 | # 215 | # > end # 4 216 | # 217 | # This happens because lines 1, 2, and 3 are technically valid code and are expanded 218 | # first, deemed valid, and hidden. We need to un-hide the matching keyword on 219 | # line 1. Also work backwards and if there's a mis-matched end, show it 220 | # too 221 | def capture_first_kw_end_same_indent(block) 222 | return if block.visible_lines.length != 1 223 | return unless block.visible_lines.first.is_end? 224 | 225 | visible_line = block.visible_lines.first 226 | lines = @code_lines[block.lines.first.index..visible_line.index] 227 | matching_kw = lines.reverse.detect { |line| line.indent == block.current_indent && line.is_kw? } 228 | return unless matching_kw 229 | 230 | @lines_to_output << matching_kw 231 | 232 | kw_count = 0 233 | end_count = 0 234 | orphan_end = @code_lines[matching_kw.index..visible_line.index].detect do |line| 235 | kw_count += 1 if line.is_kw? 236 | end_count += 1 if line.is_end? 237 | 238 | end_count >= kw_count 239 | end 240 | 241 | return unless orphan_end 242 | @lines_to_output << orphan_end 243 | end 244 | end 245 | end 246 | -------------------------------------------------------------------------------- /lib/syntax_suggest/around_block_scan.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "scan_history" 4 | 5 | module SyntaxSuggest 6 | # This class is useful for exploring contents before and after 7 | # a block 8 | # 9 | # It searches above and below the passed in block to match for 10 | # whatever criteria you give it: 11 | # 12 | # Example: 13 | # 14 | # def dog # 1 15 | # puts "bark" # 2 16 | # puts "bark" # 3 17 | # end # 4 18 | # 19 | # scan = AroundBlockScan.new( 20 | # code_lines: code_lines 21 | # block: CodeBlock.new(lines: code_lines[1]) 22 | # ) 23 | # 24 | # scan.scan_while { true } 25 | # 26 | # puts scan.before_index # => 0 27 | # puts scan.after_index # => 3 28 | # 29 | class AroundBlockScan 30 | def initialize(code_lines:, block:) 31 | @code_lines = code_lines 32 | @orig_indent = block.current_indent 33 | 34 | @stop_after_kw = false 35 | @force_add_empty = false 36 | @force_add_hidden = false 37 | @target_indent = nil 38 | 39 | @scanner = ScanHistory.new(code_lines: code_lines, block: block) 40 | end 41 | 42 | # When using this flag, `scan_while` will 43 | # bypass the block it's given and always add a 44 | # line that responds truthy to `CodeLine#hidden?` 45 | # 46 | # Lines are hidden when they've been evaluated by 47 | # the parser as part of a block and found to contain 48 | # valid code. 49 | def force_add_hidden 50 | @force_add_hidden = true 51 | self 52 | end 53 | 54 | # When using this flag, `scan_while` will 55 | # bypass the block it's given and always add a 56 | # line that responds truthy to `CodeLine#empty?` 57 | # 58 | # Empty lines contain no code, only whitespace such 59 | # as leading spaces a newline. 60 | def force_add_empty 61 | @force_add_empty = true 62 | self 63 | end 64 | 65 | # Tells `scan_while` to look for mismatched keyword/end-s 66 | # 67 | # When scanning up, if we see more keywords then end-s it will 68 | # stop. This might happen when scanning outside of a method body. 69 | # the first scan line up would be a keyword and this setting would 70 | # trigger a stop. 71 | # 72 | # When scanning down, stop if there are more end-s than keywords. 73 | def stop_after_kw 74 | @stop_after_kw = true 75 | self 76 | end 77 | 78 | # Main work method 79 | # 80 | # The scan_while method takes a block that yields lines above and 81 | # below the block. If the yield returns true, the @before_index 82 | # or @after_index are modified to include the matched line. 83 | # 84 | # In addition to yielding individual lines, the internals of this 85 | # object give a mini DSL to handle common situations such as 86 | # stopping if we've found a keyword/end mis-match in one direction 87 | # or the other. 88 | def scan_while 89 | stop_next_up = false 90 | stop_next_down = false 91 | 92 | @scanner.scan( 93 | up: ->(line, kw_count, end_count) { 94 | next false if stop_next_up 95 | next true if @force_add_hidden && line.hidden? 96 | next true if @force_add_empty && line.empty? 97 | 98 | if @stop_after_kw && kw_count > end_count 99 | stop_next_up = true 100 | end 101 | 102 | yield line 103 | }, 104 | down: ->(line, kw_count, end_count) { 105 | next false if stop_next_down 106 | next true if @force_add_hidden && line.hidden? 107 | next true if @force_add_empty && line.empty? 108 | 109 | if @stop_after_kw && end_count > kw_count 110 | stop_next_down = true 111 | end 112 | 113 | yield line 114 | } 115 | ) 116 | 117 | self 118 | end 119 | 120 | # Scanning is intentionally conservative because 121 | # we have no way of rolling back an aggressive block (at this time) 122 | # 123 | # If a block was stopped for some trivial reason, (like an empty line) 124 | # but the next line would have caused it to be balanced then we 125 | # can check that condition and grab just one more line either up or 126 | # down. 127 | # 128 | # For example, below if we're scanning up, line 2 might cause 129 | # the scanning to stop. This is because empty lines might 130 | # denote logical breaks where the user intended to chunk code 131 | # which is a good place to stop and check validity. Unfortunately 132 | # it also means we might have a "dangling" keyword or end. 133 | # 134 | # 1 def bark 135 | # 2 136 | # 3 end 137 | # 138 | # If lines 2 and 3 are in the block, then when this method is 139 | # run it would see it is unbalanced, but that acquiring line 1 140 | # would make it balanced, so that's what it does. 141 | def lookahead_balance_one_line 142 | kw_count = 0 143 | end_count = 0 144 | lines.each do |line| 145 | kw_count += 1 if line.is_kw? 146 | end_count += 1 if line.is_end? 147 | end 148 | 149 | return self if kw_count == end_count # nothing to balance 150 | 151 | @scanner.commit_if_changed # Rollback point if we don't find anything to optimize 152 | 153 | # Try to eat up empty lines 154 | @scanner.scan( 155 | up: ->(line, _, _) { line.hidden? || line.empty? }, 156 | down: ->(line, _, _) { line.hidden? || line.empty? } 157 | ) 158 | 159 | # More ends than keywords, check if we can balance expanding up 160 | next_up = @scanner.next_up 161 | next_down = @scanner.next_down 162 | case end_count - kw_count 163 | when 1 164 | if next_up&.is_kw? && next_up.indent >= @target_indent 165 | @scanner.scan( 166 | up: ->(line, _, _) { line == next_up }, 167 | down: ->(line, _, _) { false } 168 | ) 169 | @scanner.commit_if_changed 170 | end 171 | when -1 172 | if next_down&.is_end? && next_down.indent >= @target_indent 173 | @scanner.scan( 174 | up: ->(line, _, _) { false }, 175 | down: ->(line, _, _) { line == next_down } 176 | ) 177 | @scanner.commit_if_changed 178 | end 179 | end 180 | # Rollback any uncommitted changes 181 | @scanner.stash_changes 182 | 183 | self 184 | end 185 | 186 | # Finds code lines at the same or greater indentation and adds them 187 | # to the block 188 | def scan_neighbors_not_empty 189 | @target_indent = @orig_indent 190 | scan_while { |line| line.not_empty? && line.indent >= @target_indent } 191 | end 192 | 193 | # Scan blocks based on indentation of next line above/below block 194 | # 195 | # Determines indentaion of the next line above/below the current block. 196 | # 197 | # Normally this is called when a block has expanded to capture all "neighbors" 198 | # at the same (or greater) indentation and needs to expand out. For example 199 | # the `def/end` lines surrounding a method. 200 | def scan_adjacent_indent 201 | before_after_indent = [] 202 | 203 | before_after_indent << (@scanner.next_up&.indent || 0) 204 | before_after_indent << (@scanner.next_down&.indent || 0) 205 | 206 | @target_indent = before_after_indent.min 207 | scan_while { |line| line.not_empty? && line.indent >= @target_indent } 208 | 209 | self 210 | end 211 | 212 | # Return the currently matched lines as a `CodeBlock` 213 | # 214 | # When a `CodeBlock` is created it will gather metadata about 215 | # itself, so this is not a free conversion. Avoid allocating 216 | # more CodeBlock's than needed 217 | def code_block 218 | CodeBlock.new(lines: lines) 219 | end 220 | 221 | # Returns the lines matched by the current scan as an 222 | # array of CodeLines 223 | def lines 224 | @scanner.lines 225 | end 226 | 227 | # Manageable rspec errors 228 | def inspect 229 | "#<#{self.class}:0x0000123843lol >" 230 | end 231 | end 232 | end 233 | -------------------------------------------------------------------------------- /spec/unit/clean_document_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "../spec_helper" 4 | 5 | module SyntaxSuggest 6 | RSpec.describe CleanDocument do 7 | it "heredocs" do 8 | source = fixtures_dir.join("this_project_extra_def.rb.txt").read 9 | code_lines = CleanDocument.new(source: source).call.lines 10 | 11 | expect(code_lines[18 - 1].to_s).to eq(<<-EOL) 12 | @io.puts <<~EOM 13 | 14 | SyntaxSuggest: A syntax error was detected 15 | 16 | This code has an unmatched `end` this is caused by either 17 | missing a syntax keyword (`def`, `do`, etc.) or inclusion 18 | of an extra `end` line: 19 | EOM 20 | EOL 21 | expect(code_lines[18].to_s).to eq("") 22 | 23 | expect(code_lines[27 - 1].to_s).to eq(<<-'EOL') 24 | @io.puts(<<~EOM) if filename 25 | file: #{filename} 26 | EOM 27 | EOL 28 | expect(code_lines[27].to_s).to eq("") 29 | 30 | expect(code_lines[31 - 1].to_s).to eq(<<-'EOL') 31 | @io.puts <<~EOM 32 | #{code_with_filename} 33 | EOM 34 | EOL 35 | expect(code_lines[31].to_s).to eq("") 36 | end 37 | 38 | it "joins: multi line methods" do 39 | source = <<~EOM 40 | User 41 | .where(name: 'schneems') 42 | .first 43 | EOM 44 | 45 | doc = CleanDocument.new(source: source).join_consecutive! 46 | 47 | expect(doc.lines[0].to_s).to eq(source) 48 | expect(doc.lines[1].to_s).to eq("") 49 | expect(doc.lines[2].to_s).to eq("") 50 | expect(doc.lines[3]).to eq(nil) 51 | 52 | lines = doc.lines 53 | expect( 54 | DisplayCodeWithLineNumbers.new( 55 | lines: lines 56 | ).call 57 | ).to eq(<<~EOM.indent(2)) 58 | 1 User 59 | 2 .where(name: 'schneems') 60 | 3 .first 61 | EOM 62 | 63 | expect( 64 | DisplayCodeWithLineNumbers.new( 65 | lines: lines, 66 | highlight_lines: lines[0] 67 | ).call 68 | ).to eq(<<~EOM) 69 | > 1 User 70 | > 2 .where(name: 'schneems') 71 | > 3 .first 72 | EOM 73 | end 74 | 75 | it "joins multi-line chained methods when separated by comments" do 76 | source = <<~EOM 77 | User. 78 | # comment 79 | where(name: 'schneems'). 80 | # another comment 81 | first 82 | EOM 83 | 84 | doc = CleanDocument.new(source: source).join_consecutive! 85 | code_lines = doc.lines 86 | 87 | expect(code_lines[0].to_s.count($/)).to eq(5) 88 | code_lines[1..].each do |line| 89 | expect(line.to_s.strip.length).to eq(0) 90 | end 91 | end 92 | 93 | it "helper method: take_while_including" do 94 | source = <<~EOM 95 | User 96 | .where(name: 'schneems') 97 | .first 98 | EOM 99 | 100 | doc = CleanDocument.new(source: source) 101 | 102 | lines = doc.take_while_including { |line| !line.to_s.include?("where") } 103 | expect(lines.count).to eq(2) 104 | end 105 | 106 | it "comments: removes comments" do 107 | source = <<~EOM 108 | # lol 109 | puts "what" 110 | # yolo 111 | EOM 112 | 113 | lines = CleanDocument.new(source: source).lines 114 | expect(lines[0].to_s).to eq($/) 115 | expect(lines[1].to_s).to eq('puts "what"' + $/) 116 | expect(lines[2].to_s).to eq($/) 117 | end 118 | 119 | it "trailing slash: does not join trailing do" do 120 | # Some keywords and syntaxes trigger the "ignored line" 121 | # lex output, we ignore them by filtering by BEG 122 | # 123 | # The `do` keyword is one of these: 124 | # https://gist.github.com/schneems/6a7d7f988d3329fb3bd4b5be3e2efc0c 125 | source = <<~EOM 126 | foo do 127 | puts "lol" 128 | end 129 | EOM 130 | 131 | doc = CleanDocument.new(source: source).join_consecutive! 132 | 133 | expect(doc.lines[0].to_s).to eq(source.lines[0]) 134 | expect(doc.lines[1].to_s).to eq(source.lines[1]) 135 | expect(doc.lines[2].to_s).to eq(source.lines[2]) 136 | end 137 | 138 | it "trailing slash: formats output" do 139 | source = <<~'EOM' 140 | context "timezones workaround" do 141 | it "should receive a time in UTC format and return the time with the"\ 142 | "office's UTC offset subtracted from it" do 143 | travel_to DateTime.new(2020, 10, 1, 10, 0, 0) do 144 | office = build(:office) 145 | end 146 | end 147 | end 148 | EOM 149 | 150 | code_lines = CleanDocument.new(source: source).call.lines 151 | expect( 152 | DisplayCodeWithLineNumbers.new( 153 | lines: code_lines.select(&:visible?) 154 | ).call 155 | ).to eq(<<~'EOM'.indent(2)) 156 | 1 context "timezones workaround" do 157 | 2 it "should receive a time in UTC format and return the time with the"\ 158 | 3 "office's UTC offset subtracted from it" do 159 | 4 travel_to DateTime.new(2020, 10, 1, 10, 0, 0) do 160 | 5 office = build(:office) 161 | 6 end 162 | 7 end 163 | 8 end 164 | EOM 165 | 166 | expect( 167 | DisplayCodeWithLineNumbers.new( 168 | lines: code_lines.select(&:visible?), 169 | highlight_lines: code_lines[1] 170 | ).call 171 | ).to eq(<<~'EOM') 172 | 1 context "timezones workaround" do 173 | > 2 it "should receive a time in UTC format and return the time with the"\ 174 | > 3 "office's UTC offset subtracted from it" do 175 | 4 travel_to DateTime.new(2020, 10, 1, 10, 0, 0) do 176 | 5 office = build(:office) 177 | 6 end 178 | 7 end 179 | 8 end 180 | EOM 181 | end 182 | 183 | it "trailing slash: basic detection" do 184 | source = <<~'EOM' 185 | it "trailing s" \ 186 | "lash" do 187 | EOM 188 | 189 | code_lines = CleanDocument.new(source: source).call.lines 190 | 191 | expect(code_lines[0]).to_not be_hidden 192 | expect(code_lines[1]).to be_hidden 193 | 194 | expect( 195 | code_lines.join 196 | ).to eq(code_lines.map(&:original).join) 197 | end 198 | 199 | it "trailing slash: joins multiple lines" do 200 | source = <<~'EOM' 201 | it "should " \ 202 | "keep " \ 203 | "going " do 204 | end 205 | EOM 206 | 207 | doc = CleanDocument.new(source: source).join_trailing_slash! 208 | expect(doc.lines[0].to_s).to eq(source.lines[0..2].join) 209 | expect(doc.lines[1].to_s).to eq("") 210 | expect(doc.lines[2].to_s).to eq("") 211 | expect(doc.lines[3].to_s).to eq(source.lines[3]) 212 | 213 | lines = doc.lines 214 | expect( 215 | DisplayCodeWithLineNumbers.new( 216 | lines: lines 217 | ).call 218 | ).to eq(<<~'EOM'.indent(2)) 219 | 1 it "should " \ 220 | 2 "keep " \ 221 | 3 "going " do 222 | 4 end 223 | EOM 224 | 225 | expect( 226 | DisplayCodeWithLineNumbers.new( 227 | lines: lines, 228 | highlight_lines: lines[0] 229 | ).call 230 | ).to eq(<<~'EOM') 231 | > 1 it "should " \ 232 | > 2 "keep " \ 233 | > 3 "going " do 234 | 4 end 235 | EOM 236 | end 237 | 238 | it "trailing slash: no false positives" do 239 | source = <<~'EOM' 240 | def formatters 241 | @formatters ||= { 242 | amazing_print: ->(obj) { obj.ai + "\n" }, 243 | inspect: ->(obj) { obj.inspect + "\n" }, 244 | json: ->(obj) { obj.to_json }, 245 | marshal: ->(obj) { Marshal.dump(obj) }, 246 | none: ->(_obj) { nil }, 247 | pretty_json: ->(obj) { JSON.pretty_generate(obj) }, 248 | pretty_print: ->(obj) { obj.pretty_inspect }, 249 | puts: ->(obj) { require 'stringio'; sio = StringIO.new; sio.puts(obj); sio.string }, 250 | to_s: ->(obj) { obj.to_s + "\n" }, 251 | yaml: ->(obj) { obj.to_yaml }, 252 | } 253 | end 254 | EOM 255 | 256 | code_lines = CleanDocument.new(source: source).call.lines 257 | expect(code_lines.join).to eq(code_lines.join) 258 | end 259 | end 260 | end 261 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SyntaxSuggest 2 | 3 | An error in your code forces you to stop. SyntaxSuggest helps you find those errors to get you back on your way faster. 4 | 5 | ``` 6 | Unmatched `end', missing keyword (`do', `def`, `if`, etc.) ? 7 | 8 | 1 class Dog 9 | > 2 defbark 10 | > 4 end 11 | 5 end 12 | ``` 13 | 14 | > This project was previously named `dead_end` as it attempted to find dangling `end` keywords. The name was changed to be more descriptive and welcoming as part of the effort to merge it into [Ruby 3.2](https://bugs.ruby-lang.org/issues/18159#note-29). 15 | 16 | ## Installation in your codebase 17 | 18 | To automatically annotate errors when they happen, add this to your Gemfile: 19 | 20 | ```ruby 21 | gem 'syntax_suggest' 22 | ``` 23 | 24 | And then execute: 25 | 26 | $ bundle install 27 | 28 | If your application is not calling `Bundler.require` then you must manually add a require: 29 | 30 | ```ruby 31 | require "syntax_suggest" 32 | ``` 33 | 34 | If you're using rspec add this to your `.rspec` file: 35 | 36 | ``` 37 | --require syntax_suggest 38 | ``` 39 | 40 | > This is needed because people can execute a single test file via `bundle exec rspec path/to/file_spec.rb` and if that file has a syntax error, it won't load `spec_helper.rb` to trigger any requires. 41 | 42 | ## Install the CLI 43 | 44 | To get the CLI and manually search for syntax errors (but not automatically annotate them), you can manually install the gem: 45 | 46 | $ gem install syntax_suggest 47 | 48 | This gives you the CLI command `$ syntax_suggest` for more info run `$ syntax_suggest --help`. 49 | 50 | ## Editor integration 51 | 52 | An extension is available for VSCode: 53 | 54 | - Extension: https://marketplace.visualstudio.com/items?itemName=Zombocom.dead-end-vscode 55 | - GitHub: https://github.com/zombocom/dead_end-vscode 56 | 57 | ## What syntax errors does it handle? 58 | 59 | Syntax suggest will fire against all syntax errors and can isolate any syntax error. In addition, syntax_suggest attempts to produce human readable descriptions of what needs to be done to resolve the issue. For example: 60 | 61 | - Missing `end`: 62 | 63 | 71 | 72 | ``` 73 | Unmatched keyword, missing `end' ? 74 | 75 | > 1 class Dog 76 | > 2 def bark 77 | > 4 end 78 | ``` 79 | 80 | - Missing keyword 81 | 92 | 93 | ``` 94 | Unmatched `end', missing keyword (`do', `def`, `if`, etc.) ? 95 | 96 | 1 class Dog 97 | 2 def speak 98 | > 3 @sounds.each |sound| 99 | > 5 end 100 | 6 end 101 | 7 end 102 | ``` 103 | 104 | - Missing pair characters (like `{}`, `[]`, `()` , or `||`) 105 | 115 | 116 | ``` 117 | Unmatched `(', missing `)' ? 118 | 119 | 1 class Dog 120 | > 2 def speak(sound 121 | > 4 end 122 | 5 end 123 | ``` 124 | 125 | - Any ambiguous or unknown errors will be annotated by the original parser error output: 126 | 127 | 134 | 135 | ``` 136 | Expected an expression after the operator 137 | 138 | 1 class Dog 139 | 2 def meals_last_month 140 | > 3 puts 3 * 141 | 4 end 142 | 5 end 143 | ``` 144 | 145 | ## How is it better than `ruby -wc`? 146 | 147 | Ruby allows you to syntax check a file with warnings using `ruby -wc`. This emits a parser error instead of a human focused error. Ruby's parse errors attempt to narrow down the location and can tell you if there is a glaring indentation error involving `end`. 148 | 149 | The `syntax_suggest` algorithm doesn't just guess at the location of syntax errors, it re-parses the document to prove that it captured them. 150 | 151 | This library focuses on the human side of syntax errors. It cares less about why the document could not be parsed (computer problem) and more on what the programmer needs (human problem) to fix the problem. 152 | 153 | ## Sounds cool, but why isn't this baked into Ruby directly? 154 | 155 | We are now talking about it https://bugs.ruby-lang.org/issues/18159#change-93682. 156 | 157 | ## Artificial Inteligence? 158 | 159 | This library uses a goal-seeking algorithm for syntax error detection similar to that of a path-finding search. For more information [read the blog post about how it works under the hood](https://schneems.com/2020/12/01/squash-unexpectedend-errors-with-syntaxsearch/). 160 | 161 | ## How does it detect syntax error locations? 162 | 163 | We know that source code that does not contain a syntax error can be parsed. We also know that code with a syntax error contains both valid code and invalid code. If you remove the invalid code, then we can programatically determine that the code we removed contained a syntax error. We can do this detection by generating small code blocks and searching for which blocks need to be removed to generate valid source code. 164 | 165 | Since there can be multiple syntax errors in a document it's not good enough to check individual code blocks, we've got to check multiple at the same time. We will keep creating and adding new blocks to our search until we detect that our "frontier" (which contains all of our blocks) contains the syntax error. After this, we can stop our search and instead focus on filtering to find the smallest subset of blocks that contain the syntax error. 166 | 167 | Here's an example: 168 | 169 | ![](assets/syntax_search.gif) 170 | 171 | ## Use internals 172 | 173 | To use the `syntax_suggest` gem without monkeypatching you can `require 'syntax_suggest/api'`. This will allow you to load `syntax_suggest` and use its internals without mutating `require`. 174 | 175 | Stable internal interface(s): 176 | 177 | - `SyntaxSuggest.handle_error(e)` 178 | 179 | Any other entrypoints are subject to change without warning. If you want to use an internal interface from `syntax_suggest` not on this list, open an issue to explain your use case. 180 | 181 | ## Development 182 | 183 | ### Handling conflicts with the default gem 184 | 185 | Because `syntax_suggest` is a default gem you can get conflicts when working on this project with Ruby 3.2+. To fix conflicts you can disable loading `syntax_suggest` as a default gem by using then environment variable `RUBYOPT` with the value `--disable=syntax_suggest`. The `RUBYOPT` environment variable works the same as if we had entered those flags directly in the ruby cli (i.e. `ruby --disable=syntax_suggest` is the same as `RUBYOPT="--disable=syntax_suggest" ruby`). It's needed because we don't always directly execute Ruby and RUBYOPT will be picked up when other commands load ruby (`rspec`, `rake`, or `bundle` etc.). 186 | 187 | There are some binstubs that already have this done for you. Instead of running `bundle exec rake` you can run `bin/rake`. Binstubs provided: 188 | 189 | - `bin/bundle` 190 | - `bin/rake` 191 | - `bin/rspec` 192 | 193 | ### Installation 194 | 195 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 196 | 197 | To install this gem onto your local machine, run `bin/rake install`. To release a new version, update the version number in `version.rb`, and then run `bin/rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 198 | 199 | ### How to debug changes to output display 200 | 201 | You can see changes to output against a variety of invalid code by running specs and using the `DEBUG_DISPLAY=1` environment variable. For example: 202 | 203 | ``` 204 | $ DEBUG_DISPLAY=1 bundle exec rspec spec/ --format=failures 205 | ``` 206 | 207 | ### Run profiler 208 | 209 | You can output profiler data to the `tmp` directory by running: 210 | 211 | ``` 212 | $ DEBUG_PERF=1 bundle exec rspec spec/integration/syntax_suggest_spec.rb 213 | ``` 214 | 215 | Some outputs are in text format, some are html, the raw marshaled data is available in `raw.rb.marshal`. See https://ruby-prof.github.io/#reports for more info. One interesting one, is the "kcachegrind" interface. To view this on mac: 216 | 217 | ``` 218 | $ brew install qcachegrind 219 | ``` 220 | 221 | Open: 222 | 223 | ``` 224 | $ qcachegrind tmp/last/profile.callgrind.out. 225 | ``` 226 | 227 | ## Environment variables 228 | 229 | - `SYNTAX_SUGGEST_DEBUG` - Enables debug output to STDOUT/STDERR and/or disk at `./tmp`. The contents of debugging output are not stable and may change. If you would like stability, please open an issue to explain your use case. 230 | - `SYNTAX_SUGGEST_TIMEOUT` - Changes the default timeout value to the number set (in seconds). 231 | 232 | ## Contributing 233 | 234 | Bug reports and pull requests are welcome on GitHub at https://github.com/zombocom/syntax_suggest. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/zombocom/syntax_suggest/blob/main/CODE_OF_CONDUCT.md). 235 | 236 | 237 | ## License 238 | 239 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 240 | 241 | ## Code of Conduct 242 | 243 | Everyone interacting in the SyntaxSuggest project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/zombocom/syntax_suggest/blob/main/CODE_OF_CONDUCT.md). 244 | -------------------------------------------------------------------------------- /lib/syntax_suggest/clean_document.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module SyntaxSuggest 4 | # Parses and sanitizes source into a lexically aware document 5 | # 6 | # Internally the document is represented by an array with each 7 | # index containing a CodeLine correlating to a line from the source code. 8 | # 9 | # There are three main phases in the algorithm: 10 | # 11 | # 1. Sanitize/format input source 12 | # 2. Search for invalid blocks 13 | # 3. Format invalid blocks into something meaningful 14 | # 15 | # This class handles the first part. 16 | # 17 | # The reason this class exists is to format input source 18 | # for better/easier/cleaner exploration. 19 | # 20 | # The CodeSearch class operates at the line level so 21 | # we must be careful to not introduce lines that look 22 | # valid by themselves, but when removed will trigger syntax errors 23 | # or strange behavior. 24 | # 25 | # ## Join Trailing slashes 26 | # 27 | # Code with a trailing slash is logically treated as a single line: 28 | # 29 | # 1 it "code can be split" \ 30 | # 2 "across multiple lines" do 31 | # 32 | # In this case removing line 2 would add a syntax error. We get around 33 | # this by internally joining the two lines into a single "line" object 34 | # 35 | # ## Logically Consecutive lines 36 | # 37 | # Code that can be broken over multiple 38 | # lines such as method calls are on different lines: 39 | # 40 | # 1 User. 41 | # 2 where(name: "schneems"). 42 | # 3 first 43 | # 44 | # Removing line 2 can introduce a syntax error. To fix this, all lines 45 | # are joined into one. 46 | # 47 | # ## Heredocs 48 | # 49 | # A heredoc is an way of defining a multi-line string. They can cause many 50 | # problems. If left as a single line, the parser would try to parse the contents 51 | # as ruby code rather than as a string. Even without this problem, we still 52 | # hit an issue with indentation: 53 | # 54 | # 1 foo = <<~HEREDOC 55 | # 2 "Be yourself; everyone else is already taken."" 56 | # 3 ― Oscar Wilde 57 | # 4 puts "I look like ruby code" # but i'm still a heredoc 58 | # 5 HEREDOC 59 | # 60 | # If we didn't join these lines then our algorithm would think that line 4 61 | # is separate from the rest, has a higher indentation, then look at it first 62 | # and remove it. 63 | # 64 | # If the code evaluates line 5 by itself it will think line 5 is a constant, 65 | # remove it, and introduce a syntax errror. 66 | # 67 | # All of these problems are fixed by joining the whole heredoc into a single 68 | # line. 69 | # 70 | # ## Comments and whitespace 71 | # 72 | # Comments can throw off the way the lexer tells us that the line 73 | # logically belongs with the next line. This is valid ruby but 74 | # results in a different lex output than before: 75 | # 76 | # 1 User. 77 | # 2 where(name: "schneems"). 78 | # 3 # Comment here 79 | # 4 first 80 | # 81 | # To handle this we can replace comment lines with empty lines 82 | # and then re-lex the source. This removal and re-lexing preserves 83 | # line index and document size, but generates an easier to work with 84 | # document. 85 | # 86 | class CleanDocument 87 | def initialize(source:) 88 | lines = clean_sweep(source: source) 89 | @document = CodeLine.from_source(lines.join, lines: lines) 90 | end 91 | 92 | # Call all of the document "cleaners" 93 | # and return self 94 | def call 95 | join_trailing_slash! 96 | join_consecutive! 97 | join_heredoc! 98 | 99 | self 100 | end 101 | 102 | # Return an array of CodeLines in the 103 | # document 104 | def lines 105 | @document 106 | end 107 | 108 | # Renders the document back to a string 109 | def to_s 110 | @document.join 111 | end 112 | 113 | # Remove comments 114 | # 115 | # replace with empty newlines 116 | # 117 | # source = <<~'EOM' 118 | # # Comment 1 119 | # puts "hello" 120 | # # Comment 2 121 | # puts "world" 122 | # EOM 123 | # 124 | # lines = CleanDocument.new(source: source).lines 125 | # expect(lines[0].to_s).to eq("\n") 126 | # expect(lines[1].to_s).to eq("puts "hello") 127 | # expect(lines[2].to_s).to eq("\n") 128 | # expect(lines[3].to_s).to eq("puts "world") 129 | # 130 | # Important: This must be done before lexing. 131 | # 132 | # After this change is made, we lex the document because 133 | # removing comments can change how the doc is parsed. 134 | # 135 | # For example: 136 | # 137 | # values = LexAll.new(source: <<~EOM)) 138 | # User. 139 | # # comment 140 | # where(name: 'schneems') 141 | # EOM 142 | # expect( 143 | # values.count {|v| v.type == :on_ignored_nl} 144 | # ).to eq(1) 145 | # 146 | # After the comment is removed: 147 | # 148 | # values = LexAll.new(source: <<~EOM)) 149 | # User. 150 | # 151 | # where(name: 'schneems') 152 | # EOM 153 | # expect( 154 | # values.count {|v| v.type == :on_ignored_nl} 155 | # ).to eq(2) 156 | # 157 | def clean_sweep(source:) 158 | # Match comments, but not HEREDOC strings with #{variable} interpolation 159 | # https://rubular.com/r/HPwtW9OYxKUHXQ 160 | source.lines.map do |line| 161 | if line.match?(/^\s*#([^{].*|)$/) 162 | $/ 163 | else 164 | line 165 | end 166 | end 167 | end 168 | 169 | # Smushes all heredoc lines into one line 170 | # 171 | # source = <<~'EOM' 172 | # foo = <<~HEREDOC 173 | # lol 174 | # hehehe 175 | # HEREDOC 176 | # EOM 177 | # 178 | # lines = CleanDocument.new(source: source).join_heredoc!.lines 179 | # expect(lines[0].to_s).to eq(source) 180 | # expect(lines[1].to_s).to eq("") 181 | def join_heredoc! 182 | start_index_stack = [] 183 | heredoc_beg_end_index = [] 184 | lines.each do |line| 185 | line.lex.each do |lex_value| 186 | case lex_value.type 187 | when :on_heredoc_beg 188 | start_index_stack << line.index 189 | when :on_heredoc_end 190 | start_index = start_index_stack.pop 191 | end_index = line.index 192 | heredoc_beg_end_index << [start_index, end_index] 193 | end 194 | end 195 | end 196 | 197 | heredoc_groups = heredoc_beg_end_index.map { |start_index, end_index| @document[start_index..end_index] } 198 | 199 | join_groups(heredoc_groups) 200 | self 201 | end 202 | 203 | # Smushes logically "consecutive" lines 204 | # 205 | # source = <<~'EOM' 206 | # User. 207 | # where(name: 'schneems'). 208 | # first 209 | # EOM 210 | # 211 | # lines = CleanDocument.new(source: source).join_consecutive!.lines 212 | # expect(lines[0].to_s).to eq(source) 213 | # expect(lines[1].to_s).to eq("") 214 | # 215 | # The one known case this doesn't handle is: 216 | # 217 | # Ripper.lex <<~EOM 218 | # a && 219 | # b || 220 | # c 221 | # EOM 222 | # 223 | # For some reason this introduces `on_ignore_newline` but with BEG type 224 | # 225 | def join_consecutive! 226 | consecutive_groups = @document.select(&:ignore_newline_not_beg?).map do |code_line| 227 | take_while_including(code_line.index..) do |line| 228 | line.ignore_newline_not_beg? 229 | end 230 | end 231 | 232 | join_groups(consecutive_groups) 233 | self 234 | end 235 | 236 | # Join lines with a trailing slash 237 | # 238 | # source = <<~'EOM' 239 | # it "code can be split" \ 240 | # "across multiple lines" do 241 | # EOM 242 | # 243 | # lines = CleanDocument.new(source: source).join_consecutive!.lines 244 | # expect(lines[0].to_s).to eq(source) 245 | # expect(lines[1].to_s).to eq("") 246 | def join_trailing_slash! 247 | trailing_groups = @document.select(&:trailing_slash?).map do |code_line| 248 | take_while_including(code_line.index..) { |x| x.trailing_slash? } 249 | end 250 | join_groups(trailing_groups) 251 | self 252 | end 253 | 254 | # Helper method for joining "groups" of lines 255 | # 256 | # Input is expected to be type Array> 257 | # 258 | # The outer array holds the various "groups" while the 259 | # inner array holds code lines. 260 | # 261 | # All code lines are "joined" into the first line in 262 | # their group. 263 | # 264 | # To preserve document size, empty lines are placed 265 | # in the place of the lines that were "joined" 266 | def join_groups(groups) 267 | groups.each do |lines| 268 | line = lines.first 269 | 270 | # Handle the case of multiple groups in a row 271 | # if one is already replaced, move on 272 | next if @document[line.index].empty? 273 | 274 | # Join group into the first line 275 | @document[line.index] = CodeLine.new( 276 | lex: lines.map(&:lex).flatten, 277 | line: lines.join, 278 | index: line.index 279 | ) 280 | 281 | # Hide the rest of the lines 282 | lines[1..].each do |line| 283 | # The above lines already have newlines in them, if add more 284 | # then there will be double newline, use an empty line instead 285 | @document[line.index] = CodeLine.new(line: "", index: line.index, lex: []) 286 | end 287 | end 288 | self 289 | end 290 | 291 | # Helper method for grabbing elements from document 292 | # 293 | # Like `take_while` except when it stops 294 | # iterating, it also returns the line 295 | # that caused it to stop 296 | def take_while_including(range = 0..) 297 | take_next_and_stop = false 298 | @document[range].take_while do |line| 299 | next if take_next_and_stop 300 | 301 | take_next_and_stop = !(yield line) 302 | true 303 | end 304 | end 305 | end 306 | end 307 | --------------------------------------------------------------------------------