├── test ├── executable │ ├── source.py │ ├── source.rb │ └── source_with_comments.rb ├── lib │ ├── README │ ├── assert_warning.rb │ └── test │ │ └── unit │ │ ├── assertionfailederror.rb │ │ ├── collector.rb │ │ ├── util │ │ ├── procwrapper.rb │ │ ├── backtracefilter.rb │ │ └── observable.rb │ │ ├── ui │ │ ├── testrunnerutilities.rb │ │ ├── testrunnermediator.rb │ │ └── console │ │ │ └── testrunner.rb │ │ ├── failure.rb │ │ ├── error.rb │ │ ├── testsuite.rb │ │ ├── testresult.rb │ │ ├── collector │ │ └── dir.rb │ │ └── testcase.rb ├── unit │ ├── plugins │ │ ├── user_defined │ │ │ └── user_plugin.rb │ │ └── example.rb │ ├── plugins_with_default │ │ ├── default_plugin.rb │ │ └── example_without_register_for.rb │ ├── null.rb │ ├── text.rb │ ├── count.rb │ ├── suite.rb │ ├── json_encoder.rb │ ├── duo.rb │ ├── comment_filter.rb │ ├── filter.rb │ ├── lines_of_code.rb │ ├── word_list.rb │ ├── statistic.rb │ ├── plugin.rb │ ├── debug.rb │ ├── token_kind_filter.rb │ ├── tokens.rb │ ├── file_type.rb │ └── html.rb └── functional │ ├── suite.rb │ ├── for_redcloth.rb │ └── examples.rb ├── lib └── coderay │ ├── version.rb │ ├── styles │ ├── _map.rb │ ├── style.rb │ └── alpha.rb │ ├── scanners │ ├── xml.rb │ ├── text.rb │ ├── _map.rb │ ├── taskpaper.rb │ ├── debug.rb │ ├── raydebug.rb │ ├── erb.rb │ ├── ruby │ │ ├── string_state.rb │ │ └── patterns.rb │ ├── json.rb │ ├── yaml.rb │ ├── delphi.rb │ └── haml.rb │ ├── encoders │ ├── null.rb │ ├── _map.rb │ ├── div.rb │ ├── span.rb │ ├── page.rb │ ├── comment_filter.rb │ ├── count.rb │ ├── yaml.rb │ ├── text.rb │ ├── debug.rb │ ├── filter.rb │ ├── lines_of_code.rb │ ├── xml.rb │ ├── debug_lint.rb │ ├── lint.rb │ ├── html │ │ ├── css.rb │ │ ├── numbering.rb │ │ └── output.rb │ ├── json.rb │ ├── statistic.rb │ ├── token_kind_filter.rb │ └── terminal.rb │ ├── styles.rb │ ├── encoders.rb │ ├── scanners.rb │ ├── helpers │ ├── plugin.rb │ ├── word_list.rb │ └── file_type.rb │ ├── tokens_proxy.rb │ ├── duo.rb │ ├── for_redcloth.rb │ └── token_kinds.rb ├── .simplecov ├── rake_tasks ├── benchmark.rake ├── documentation.rake ├── statistic.rake ├── test.rake ├── generator.rake └── code_statistics.rb ├── .gitignore ├── .circleci └── config.yml ├── .codeclimate.yml ├── .rubocop.yml ├── Rakefile ├── spec ├── coderay_spec.rb └── spec_helper.rb ├── .travis.yml ├── Gemfile ├── README.markdown ├── MIT-LICENSE ├── coderay.gemspec ├── FOLDERS ├── bench └── bench.rb ├── CREDITS.textile └── README_INDEX.rdoc /test/executable/source.py: -------------------------------------------------------------------------------- 1 | class ClassName(): pass -------------------------------------------------------------------------------- /test/executable/source.rb: -------------------------------------------------------------------------------- 1 | class ClassName; end -------------------------------------------------------------------------------- /lib/coderay/version.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | VERSION = '1.1.3' 3 | end 4 | -------------------------------------------------------------------------------- /test/executable/source_with_comments.rb: -------------------------------------------------------------------------------- 1 | # a class 2 | class ClassName 3 | end 4 | -------------------------------------------------------------------------------- /.simplecov: -------------------------------------------------------------------------------- 1 | unless RUBY_VERSION[/^2.3/] 2 | SimpleCov.command_name $0 3 | SimpleCov.start 4 | end 5 | -------------------------------------------------------------------------------- /lib/coderay/styles/_map.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Styles 3 | 4 | default :alpha 5 | 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /test/lib/README: -------------------------------------------------------------------------------- 1 | Contents: 2 | - test/unit: We need the old Test::Unit for the scanner test suite to work with Ruby 1.9. 3 | -------------------------------------------------------------------------------- /rake_tasks/benchmark.rake: -------------------------------------------------------------------------------- 1 | desc 'Do a benchmark' 2 | task :benchmark do 3 | ruby 'bench/bench.rb ruby html' 4 | end 5 | 6 | task :bench => :benchmark 7 | -------------------------------------------------------------------------------- /test/unit/plugins/user_defined/user_plugin.rb: -------------------------------------------------------------------------------- 1 | class UserPlugin < PluginScannerTest::Plugins::Plugin 2 | 3 | register_for :user_plugin 4 | 5 | end 6 | -------------------------------------------------------------------------------- /test/unit/plugins/example.rb: -------------------------------------------------------------------------------- 1 | class Example < PluginScannerTest::Plugins::Plugin 2 | 3 | register_for :example 4 | title 'The Example' 5 | 6 | end 7 | -------------------------------------------------------------------------------- /test/unit/plugins_with_default/default_plugin.rb: -------------------------------------------------------------------------------- 1 | class DefaultPlugin < PluginScannerTest::PluginsWithDefault::Plugin 2 | 3 | register_for :default_plugin 4 | 5 | end 6 | -------------------------------------------------------------------------------- /test/unit/plugins_with_default/example_without_register_for.rb: -------------------------------------------------------------------------------- 1 | class ExampleWithoutRegisterFor < PluginScannerTest::PluginsWithDefault::Plugin 2 | 3 | register_for :wrong_id 4 | 5 | end 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | bench/example.* 3 | coverage 4 | doc 5 | Gemfile.lock 6 | old-stuff 7 | pkg 8 | spec/examples.txt 9 | spec/reports 10 | test/executable/source.rb.html 11 | test/executable/source.rb.json 12 | test/scanners 13 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | build: 3 | docker: 4 | - image: cimg/ruby:3.3.5 5 | environment: 6 | RAILS_ENV: test 7 | steps: 8 | - checkout 9 | - run: | 10 | bundle install 11 | - run: | 12 | bundle exec rake test 13 | -------------------------------------------------------------------------------- /test/unit/null.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class NullTest < Test::Unit::TestCase 5 | 6 | def test_null 7 | ruby = <<-RUBY 8 | puts "Hello world!" 9 | RUBY 10 | tokens = CodeRay.scan ruby, :ruby 11 | assert_equal '', tokens.encode(:null) 12 | end 13 | 14 | end -------------------------------------------------------------------------------- /test/unit/text.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class TextTest < Test::Unit::TestCase 5 | 6 | def test_count 7 | ruby = <<-RUBY 8 | puts "Hello world!" 9 | RUBY 10 | tokens = CodeRay.scan ruby, :ruby 11 | assert_equal ruby, tokens.encode(:text) 12 | end 13 | 14 | end -------------------------------------------------------------------------------- /lib/coderay/scanners/xml.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | load :html 5 | 6 | # Scanner for XML. 7 | # 8 | # Currently this is the same scanner as Scanners::HTML. 9 | class XML < HTML 10 | 11 | register_for :xml 12 | file_extension 'xml' 13 | 14 | end 15 | 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/coderay/encoders/null.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # = Null Encoder 5 | # 6 | # Does nothing and returns an empty string. 7 | class Null < Encoder 8 | 9 | register_for :null 10 | 11 | def text_token text, kind 12 | # do nothing 13 | end 14 | 15 | end 16 | 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /test/unit/count.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class CountTest < Test::Unit::TestCase 5 | 6 | def test_count 7 | tokens = CodeRay.scan <<-RUBY.strip, :ruby 8 | #!/usr/bin/env ruby 9 | # a minimal Ruby program 10 | puts "Hello world!" 11 | RUBY 12 | assert_equal 11, tokens.encode(:count) 13 | end 14 | 15 | end -------------------------------------------------------------------------------- /lib/coderay/styles.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # This module holds the Style class and its subclasses. 4 | # 5 | # See Plugin. 6 | module Styles 7 | 8 | extend PluginHost 9 | plugin_path File.dirname(__FILE__), 'styles' 10 | 11 | autoload :Style, CodeRay.coderay_path('styles', 'style') 12 | 13 | end 14 | 15 | end 16 | -------------------------------------------------------------------------------- /lib/coderay/styles/style.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | module Styles 4 | 5 | # Base class for styles. 6 | # 7 | # Styles are used by Encoders::HTML to colorize tokens. 8 | class Style 9 | extend Plugin 10 | plugin_host Styles 11 | 12 | DEFAULT_OPTIONS = { } # :nodoc: 13 | 14 | end 15 | 16 | end 17 | 18 | end 19 | -------------------------------------------------------------------------------- /test/lib/assert_warning.rb: -------------------------------------------------------------------------------- 1 | class Test::Unit::TestCase 2 | 3 | def assert_warning expected_warning 4 | require 'stringio' 5 | oldstderr = $stderr 6 | $stderr = StringIO.new 7 | yield 8 | $stderr.rewind 9 | given_warning = $stderr.read.chomp 10 | assert_equal expected_warning, given_warning 11 | ensure 12 | $stderr = oldstderr 13 | end 14 | 15 | end 16 | -------------------------------------------------------------------------------- /test/lib/test/unit/assertionfailederror.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2002 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | module Test 8 | module Unit 9 | 10 | # Thrown by Test::Unit::Assertions when an assertion fails. 11 | class AssertionFailedError < StandardError 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/coderay/encoders/_map.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | map \ 5 | :loc => :lines_of_code, 6 | :plain => :text, 7 | :plaintext => :text, 8 | :remove_comments => :comment_filter, 9 | :stats => :statistic, 10 | :term => :terminal, 11 | :tty => :terminal, 12 | :yml => :yaml 13 | 14 | # No default because Tokens#nonsense should raise NoMethodError. 15 | 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /test/unit/suite.rb: -------------------------------------------------------------------------------- 1 | require 'simplecov' if RUBY_VERSION >= '1.9' 2 | require 'test/unit' 3 | require 'rubygems' 4 | 5 | $VERBOSE = $CODERAY_DEBUG = true 6 | $:.unshift 'lib' 7 | 8 | mydir = File.dirname(__FILE__) 9 | suite = Dir[File.join(mydir, '*.rb')]. 10 | map { |tc| File.basename(tc).sub(/\.rb$/, '') } - %w'suite vhdl' 11 | 12 | puts "Running CodeRay unit tests: #{suite.join(', ')}" 13 | 14 | helpers = %w(file_type word_list tokens) 15 | for test_case in helpers + (suite - helpers) 16 | load File.join(mydir, test_case + '.rb') 17 | end 18 | -------------------------------------------------------------------------------- /lib/coderay/encoders/div.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | load :html 5 | 6 | # Wraps HTML output into a DIV element, using inline styles by default. 7 | # 8 | # See Encoders::HTML for available options. 9 | class Div < HTML 10 | 11 | FILE_EXTENSION = 'div.html' 12 | 13 | register_for :div 14 | 15 | DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \ 16 | :css => :style, 17 | :wrap => :div, 18 | :line_numbers => false 19 | 20 | end 21 | 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/functional/suite.rb: -------------------------------------------------------------------------------- 1 | require 'simplecov' if RUBY_VERSION >= '1.9' 2 | require 'test/unit' 3 | 4 | $VERBOSE = $CODERAY_DEBUG = true 5 | $:.unshift File.expand_path('../../../lib', __FILE__) 6 | require 'coderay' 7 | 8 | mydir = File.dirname(__FILE__) 9 | suite = Dir[File.join(mydir, '*.rb')]. 10 | map { |tc| File.basename(tc).sub(/\.rb$/, '') } - %w'suite for_redcloth' 11 | 12 | puts "Running basic CodeRay #{CodeRay::VERSION} tests: #{suite.join(', ')}" 13 | 14 | for test_case in suite 15 | load File.join(mydir, test_case + '.rb') 16 | end 17 | -------------------------------------------------------------------------------- /lib/coderay/encoders/span.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | load :html 5 | 6 | # Wraps HTML output into a SPAN element, using inline styles by default. 7 | # 8 | # See Encoders::HTML for available options. 9 | class Span < HTML 10 | 11 | FILE_EXTENSION = 'span.html' 12 | 13 | register_for :span 14 | 15 | DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \ 16 | :css => :style, 17 | :wrap => :span, 18 | :line_numbers => false 19 | 20 | end 21 | 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | checks: 3 | argument-count: 4 | enabled: false 5 | complex-logic: 6 | enabled: false 7 | file-lines: 8 | enabled: false 9 | identical-code: 10 | enabled: false 11 | method-complexity: 12 | enabled: false 13 | method-count: 14 | enabled: false 15 | method-lines: 16 | enabled: false 17 | nested-control-flow: 18 | enabled: false 19 | return-statements: 20 | enabled: false 21 | similar-code: 22 | enabled: false 23 | plugins: 24 | rubocop: 25 | enabled: true 26 | channel: rubocop-0-76 27 | -------------------------------------------------------------------------------- /lib/coderay/encoders/page.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | load :html 5 | 6 | # Wraps the output into a HTML page, using CSS classes and 7 | # line numbers in the table format by default. 8 | # 9 | # See Encoders::HTML for available options. 10 | class Page < HTML 11 | 12 | FILE_EXTENSION = 'html' 13 | 14 | register_for :page 15 | 16 | DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \ 17 | :css => :class, 18 | :wrap => :page, 19 | :line_numbers => :table 20 | 21 | end 22 | 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/coderay/scanners/text.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | # Scanner for plain text. 5 | # 6 | # Yields just one token of the kind :plain. 7 | # 8 | # Alias: +plaintext+, +plain+ 9 | class Text < Scanner 10 | 11 | register_for :text 12 | title 'Plain text' 13 | 14 | KINDS_NOT_LOC = [:plain] # :nodoc: 15 | 16 | protected 17 | 18 | def scan_tokens encoder, options 19 | encoder.text_token string, :plain 20 | encoder 21 | end 22 | 23 | end 24 | 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/coderay/encoders.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # This module holds the Encoder class and its subclasses. 4 | # For example, the HTML encoder is named CodeRay::Encoders::HTML 5 | # can be found in coderay/encoders/html. 6 | # 7 | # Encoders also provides methods and constants for the register 8 | # mechanism and the [] method that returns the Encoder class 9 | # belonging to the given format. 10 | module Encoders 11 | 12 | extend PluginHost 13 | plugin_path File.dirname(__FILE__), 'encoders' 14 | 15 | autoload :Encoder, CodeRay.coderay_path('encoders', 'encoder') 16 | 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/coderay/scanners/_map.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | map \ 5 | :'c++' => :cpp, 6 | :cplusplus => :cpp, 7 | :ecmascript => :java_script, 8 | :ecma_script => :java_script, 9 | :rhtml => :erb, 10 | :eruby => :erb, 11 | :irb => :ruby, 12 | :javascript => :java_script, 13 | :js => :java_script, 14 | :pascal => :delphi, 15 | :patch => :diff, 16 | :plain => :text, 17 | :plaintext => :text, 18 | :xhtml => :html, 19 | :yml => :yaml 20 | 21 | default :text 22 | 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/coderay/encoders/comment_filter.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | load :token_kind_filter 5 | 6 | # A simple Filter that removes all tokens of the :comment kind. 7 | # 8 | # Alias: +remove_comments+ 9 | # 10 | # Usage: 11 | # CodeRay.scan('print # foo', :ruby).comment_filter.text 12 | # #-> "print " 13 | # 14 | # See also: TokenKindFilter, LinesOfCode 15 | class CommentFilter < TokenKindFilter 16 | 17 | register_for :comment_filter 18 | 19 | DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \ 20 | :exclude => [:comment, :docstring] 21 | 22 | end 23 | 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /rake_tasks/documentation.rake: -------------------------------------------------------------------------------- 1 | begin 2 | if RUBY_VERSION >= '1.8.7' 3 | gem 'rdoc' if defined? gem 4 | require 'rdoc/task' 5 | else 6 | require 'rake/rdoctask' 7 | end 8 | rescue LoadError 9 | warn 'Please gem install rdoc.' 10 | end 11 | 12 | desc 'Generate documentation for CodeRay' 13 | Rake::RDocTask.new :doc do |rd| 14 | rd.main = 'lib/README' 15 | rd.title = 'CodeRay Documentation' 16 | 17 | rd.options << '--line-numbers' << '--tab-width' << '2' 18 | 19 | rd.main = 'README_INDEX.rdoc' 20 | rd.rdoc_files.add 'README_INDEX.rdoc' 21 | rd.rdoc_files.add Dir['lib'] 22 | rd.rdoc_dir = 'doc' 23 | end if defined? Rake::RDocTask 24 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_from: .rubocop_todo.yml 2 | 3 | AllCops: 4 | TargetRubyVersion: 2.3 5 | DisplayStyleGuide: true 6 | Exclude: 7 | - 'test/scanners/**/*' 8 | - 'bench/example.ruby' 9 | - 'old-stuff/**/*' 10 | - 'test/lib/**/*' 11 | 12 | Gemspec/RequiredRubyVersion: 13 | Enabled: false 14 | 15 | Gemspec/DuplicatedAssignment: 16 | Enabled: false 17 | 18 | Layout/AccessModifierIndentation: 19 | Enabled: false 20 | 21 | Layout/AlignArguments: 22 | Enabled: false 23 | 24 | Layout/AlignArray: 25 | Enabled: false 26 | 27 | Layout/AlignHash: 28 | Enabled: false 29 | 30 | Layout/SpaceInsideBlockBraces: 31 | EnforcedStyle: space 32 | EnforcedStyleForEmptyBraces: space 33 | -------------------------------------------------------------------------------- /lib/coderay/encoders/count.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # Returns the number of tokens. 5 | # 6 | # Text and block tokens are counted. 7 | class Count < Encoder 8 | 9 | register_for :count 10 | 11 | protected 12 | 13 | def setup options 14 | super 15 | 16 | @count = 0 17 | end 18 | 19 | def finish options 20 | output @count 21 | end 22 | 23 | public 24 | 25 | def text_token text, kind 26 | @count += 1 27 | end 28 | 29 | def begin_group kind 30 | @count += 1 31 | end 32 | alias end_group begin_group 33 | alias begin_line begin_group 34 | alias end_line begin_group 35 | 36 | end 37 | 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/coderay/scanners.rb: -------------------------------------------------------------------------------- 1 | require 'strscan' 2 | 3 | module CodeRay 4 | 5 | autoload :WordList, coderay_path('helpers', 'word_list') 6 | 7 | # = Scanners 8 | # 9 | # This module holds the Scanner class and its subclasses. 10 | # For example, the Ruby scanner is named CodeRay::Scanners::Ruby 11 | # can be found in coderay/scanners/ruby. 12 | # 13 | # Scanner also provides methods and constants for the register 14 | # mechanism and the [] method that returns the Scanner class 15 | # belonging to the given lang. 16 | # 17 | # See PluginHost. 18 | module Scanners 19 | 20 | extend PluginHost 21 | plugin_path File.dirname(__FILE__), 'scanners' 22 | 23 | autoload :Scanner, CodeRay.coderay_path('scanners', 'scanner') 24 | 25 | end 26 | 27 | end 28 | -------------------------------------------------------------------------------- /rake_tasks/statistic.rake: -------------------------------------------------------------------------------- 1 | desc 'Report code statistics (LOC) from the application' 2 | task :stats do 3 | require './rake_tasks/code_statistics' 4 | CodeStatistics.new( 5 | ['Main', 'lib', /coderay.rb$/], 6 | ['CodeRay', 'lib/coderay/'], 7 | [' Scanners', 'lib/coderay/scanners/**'], 8 | [' Encoders', 'lib/coderay/encoders/**'], 9 | [' Helpers', 'lib/coderay/helpers/**'], 10 | [' Styles', 'lib/coderay/styles/**'], 11 | ['Executable', 'bin', /coderay$/], 12 | ['Executable Tests', 'test/executable/**'], 13 | ['Functional Tests', 'test/functional/**'], 14 | ['Scanner Tests', 'test/scanners/**', /suite\.rb$/], 15 | ['Unit Tests', 'test/unit/**'], 16 | # [' Test Data', 'test/scanners/**', /\.in\./, false], 17 | ['Demos', 'sample/**'] 18 | ).print 19 | end 20 | -------------------------------------------------------------------------------- /lib/coderay/encoders/yaml.rb: -------------------------------------------------------------------------------- 1 | autoload :YAML, 'yaml' 2 | 3 | module CodeRay 4 | module Encoders 5 | 6 | # = YAML Encoder 7 | # 8 | # Slow. 9 | class YAML < Encoder 10 | 11 | register_for :yaml 12 | 13 | FILE_EXTENSION = 'yaml' 14 | 15 | protected 16 | def setup options 17 | super 18 | 19 | @data = [] 20 | end 21 | 22 | def finish options 23 | output ::YAML.dump(@data) 24 | end 25 | 26 | public 27 | def text_token text, kind 28 | @data << [text, kind] 29 | end 30 | 31 | def begin_group kind 32 | @data << [:begin_group, kind] 33 | end 34 | 35 | def end_group kind 36 | @data << [:end_group, kind] 37 | end 38 | 39 | def begin_line kind 40 | @data << [:begin_line, kind] 41 | end 42 | 43 | def end_line kind 44 | @data << [:end_line, kind] 45 | end 46 | 47 | end 48 | 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/coderay/encoders/text.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # Concats the tokens into a single string, resulting in the original 5 | # code string if no tokens were removed. 6 | # 7 | # Alias: +plain+, +plaintext+ 8 | # 9 | # == Options 10 | # 11 | # === :separator 12 | # A separator string to join the tokens. 13 | # 14 | # Default: empty String 15 | class Text < Encoder 16 | 17 | register_for :text 18 | 19 | FILE_EXTENSION = 'txt' 20 | 21 | DEFAULT_OPTIONS = { 22 | :separator => nil 23 | } 24 | 25 | def text_token text, kind 26 | super 27 | 28 | if @first 29 | @first = false 30 | else 31 | @out << @sep 32 | end if @sep 33 | end 34 | 35 | protected 36 | def setup options 37 | super 38 | 39 | @first = true 40 | @sep = options[:separator] 41 | end 42 | 43 | end 44 | 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | 3 | $:.unshift File.dirname(__FILE__) unless $:.include? '.' 4 | 5 | ROOT = '.' 6 | LIB_ROOT = File.join ROOT, 'lib' 7 | 8 | task :default => :test 9 | 10 | if File.directory? 'rake_tasks' 11 | 12 | # load rake tasks from subfolder 13 | for task_file in Dir['rake_tasks/*.rake'].sort 14 | load task_file 15 | end 16 | 17 | else 18 | 19 | # fallback tasks when rake_tasks folder is not present (eg. in the distribution package) 20 | desc 'Run CodeRay tests (basic)' 21 | task :test do 22 | ruby './test/functional/suite.rb' 23 | ruby './test/functional/for_redcloth.rb' 24 | end 25 | 26 | gem 'rdoc' if defined? gem 27 | require 'rdoc/task' 28 | desc 'Generate documentation for CodeRay' 29 | Rake::RDocTask.new :doc do |rd| 30 | rd.title = 'CodeRay Documentation' 31 | rd.main = 'README_INDEX.rdoc' 32 | rd.rdoc_files.add Dir['lib'] 33 | rd.rdoc_files.add rd.main 34 | rd.rdoc_dir = 'doc' 35 | end 36 | 37 | end 38 | -------------------------------------------------------------------------------- /test/unit/json_encoder.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class JSONEncoderTest < Test::Unit::TestCase 5 | 6 | def test_json_output 7 | old_load_paths = $:.dup 8 | begin 9 | $:.delete '.' 10 | $:.delete File.dirname(__FILE__) 11 | json = CodeRay.scan('puts "Hello world!"', :ruby).json 12 | assert_equal [ 13 | { "type" => "text", "text" => "puts", "kind" => "ident" }, 14 | { "type" => "text", "text" => " ", "kind" => "space" }, 15 | { "type" => "block", "action" => "open", "kind" => "string" }, 16 | { "type" => "text", "text" => "\"", "kind" => "delimiter" }, 17 | { "type" => "text", "text" => "Hello world!", "kind" => "content" }, 18 | { "type" => "text", "text" => "\"", "kind" => "delimiter" }, 19 | { "type" => "block", "action" => "close", "kind" => "string" }, 20 | ], JSON.load(json) 21 | ensure 22 | for path in old_load_paths - $: 23 | $: << path 24 | end 25 | end 26 | end 27 | 28 | end -------------------------------------------------------------------------------- /spec/coderay_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../spec_helper', __FILE__) 2 | 3 | RSpec.describe CodeRay do 4 | describe '::VERSION' do 5 | it "returns the Gem's version" do 6 | expect(CodeRay::VERSION).to match(/\A\d\.\d\.\d?\z/) 7 | end 8 | end 9 | 10 | describe '.coderay_path' do 11 | it 'returns an absolute file path to the given code file' do 12 | base = File.expand_path('../..', __FILE__) 13 | expect(CodeRay.coderay_path('file')).to eq("#{base}/lib/coderay/file") 14 | end 15 | end 16 | 17 | describe '.scan' do 18 | let(:code) { 'puts "Hello, World!"' } 19 | let(:tokens) do 20 | [ 21 | ['puts', :ident], 22 | [' ', :space], 23 | [:begin_group, :string], 24 | ['"', :delimiter], 25 | ['Hello, World!', :content], 26 | ['"', :delimiter], 27 | [:end_group, :string] 28 | ].flatten 29 | end 30 | 31 | it 'returns tokens' do 32 | expect(CodeRay.scan(code, :ruby).tokens).to eq(tokens) 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | env: 2 | global: 3 | - "JRUBY_OPTS=-Xcext.enabled=true" 4 | - "CC_TEST_REPORTER_ID=faa393209ff0a104cf37511a9a03510bcee37951971b1ca4ffc2af217851d47e" 5 | language: ruby 6 | os: linux 7 | rvm: 8 | - 1.8.7 9 | - ree 10 | - 1.9.3 11 | - 2.0 12 | - 2.1 13 | - 2.2 14 | - 2.3 15 | - 2.4 16 | - 2.5 17 | - 2.6 18 | - 2.7 19 | - 3.0 20 | - 3.1 21 | - 3.2 22 | - ruby-head 23 | - jruby 24 | jobs: 25 | allow_failures: 26 | - rvm: 1.8.7 27 | - rvm: ree 28 | - rvm: ruby-head 29 | - rvm: jruby 30 | branches: 31 | only: 32 | - master 33 | before_script: 34 | - if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT 35 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 36 | - chmod +x ./cc-test-reporter 37 | - ./cc-test-reporter before-build 38 | script: "rake test" # test:scanners" 39 | after_script: 40 | - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT 41 | -------------------------------------------------------------------------------- /test/unit/duo.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'yaml' 3 | require 'coderay' 4 | 5 | class DuoTest < Test::Unit::TestCase 6 | 7 | def test_two_arguments 8 | duo = CodeRay::Duo[:ruby, :html] 9 | assert_kind_of CodeRay::Scanners[:ruby], duo.scanner 10 | assert_kind_of CodeRay::Encoders[:html], duo.encoder 11 | end 12 | 13 | def test_two_hash 14 | duo = CodeRay::Duo[:ruby => :html] 15 | assert_kind_of CodeRay::Scanners[:ruby], duo.scanner 16 | assert_kind_of CodeRay::Encoders[:html], duo.encoder 17 | end 18 | 19 | def test_call 20 | duo = CodeRay::Duo[:python => :yml] 21 | yaml = [["def", :keyword], 22 | [" ", :space], 23 | ["test", :method], 24 | [":", :operator], 25 | [" ", :space], 26 | [:begin_group, :string], 27 | ["\"", :delimiter], 28 | ["pass", :content], 29 | ["\"", :delimiter], 30 | [:end_group, :string]] 31 | 32 | assert_equal yaml, YAML.load(duo.call('def test: "pass"')) 33 | end 34 | 35 | end 36 | -------------------------------------------------------------------------------- /test/lib/test/unit/collector.rb: -------------------------------------------------------------------------------- 1 | module Test 2 | module Unit 3 | module Collector 4 | def initialize 5 | @filters = [] 6 | end 7 | 8 | def filter=(filters) 9 | @filters = case(filters) 10 | when Proc 11 | [filters] 12 | when Array 13 | filters 14 | end 15 | end 16 | 17 | def add_suite(destination, suite) 18 | to_delete = suite.tests.find_all{|t| !include?(t)} 19 | to_delete.each{|t| suite.delete(t)} 20 | destination << suite unless(suite.size == 0) 21 | end 22 | 23 | def include?(test) 24 | return true if(@filters.empty?) 25 | @filters.each do |filter| 26 | result = filter[test] 27 | if(result.nil?) 28 | next 29 | elsif(!result) 30 | return false 31 | else 32 | return true 33 | end 34 | end 35 | true 36 | end 37 | 38 | def sort(suites) 39 | suites.sort_by{|s| s.name} 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in coderay.gemspec 4 | gemspec 5 | 6 | # Add dependencies to develop your gem here. 7 | # Include everything needed to run rake, tests, features, etc. 8 | group :development do 9 | gem 'bundler' 10 | gem 'json', '~> 1.8' if RUBY_VERSION < '2.0' 11 | gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' 12 | gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' 13 | gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' 14 | gem 'rspec', '~> 3.9.0' 15 | gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' 16 | gem 'simplecov', RUBY_VERSION < '2.7' ? '~> 0.17.1' : '>= 0.18.5' 17 | gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' 18 | gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' 19 | gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' 20 | end 21 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # CodeRay 2 | 3 | [![CircleCI](https://dl.circleci.com/status-badge/img/gh/rubychan/coderay/tree/master.svg?style=svg&circle-token=CCIPRJ_BbFff6nMhNtPdrCBNMDxNq_be00bbb00849a29d8d8d2e28e7b84cbf76a9ee5c)](https://dl.circleci.com/status-badge/redirect/gh/rubychan/coderay/tree/master) [![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) 4 | 5 | ## About 6 | 7 | CodeRay is a Ruby library for syntax highlighting. 8 | 9 | You put your code in, and you get it back colored; Keywords, strings, floats, comments - all in different colors. And with line numbers. 10 | 11 | ## Installation 12 | 13 | `gem install coderay` 14 | 15 | ### Dependencies 16 | 17 | CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0+. It also runs on JRuby. 18 | 19 | ## Example Usage 20 | 21 | ```ruby 22 | require 'coderay' 23 | 24 | html = CodeRay.scan("puts 'Hello, world!'", :ruby).div(:line_numbers => :table) 25 | ```` 26 | 27 | ## Documentation 28 | 29 | See [rubydoc](http://rubydoc.info/gems/coderay). 30 | -------------------------------------------------------------------------------- /lib/coderay/encoders/debug.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # = Debug Encoder 5 | # 6 | # Fast encoder producing simple debug output. 7 | # 8 | # It is readable and diff-able and is used for testing. 9 | # 10 | # You cannot fully restore the tokens information from the 11 | # output, because consecutive :space tokens are merged. 12 | # 13 | # See also: Scanners::Debug 14 | class Debug < Encoder 15 | 16 | register_for :debug 17 | 18 | FILE_EXTENSION = 'raydebug' 19 | 20 | def text_token text, kind 21 | if kind == :space 22 | @out << text 23 | else 24 | text = text.gsub('\\', '\\\\\\\\') if text.index('\\') 25 | text = text.gsub(')', '\\\\)') if text.index(')') 26 | @out << "#{kind}(#{text})" 27 | end 28 | end 29 | 30 | def begin_group kind 31 | @out << "#{kind}<" 32 | end 33 | 34 | def end_group kind 35 | @out << '>' 36 | end 37 | 38 | def begin_line kind 39 | @out << "#{kind}[" 40 | end 41 | 42 | def end_line kind 43 | @out << ']' 44 | end 45 | 46 | end 47 | 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /lib/coderay/scanners/taskpaper.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | class Taskpaper < Scanner 5 | 6 | register_for :taskpaper 7 | file_extension 'taskpaper' 8 | 9 | protected 10 | 11 | def scan_tokens encoder, options 12 | until eos? 13 | if match = scan(/\S.*:.*$/) # project 14 | encoder.text_token(match, :namespace) 15 | elsif match = scan(/-.+@done.*/) # completed task 16 | encoder.text_token(match, :done) 17 | elsif match = scan(/-(?:[^@\n]+|@(?!due))*/) # task 18 | encoder.text_token(match, :plain) 19 | elsif match = scan(/@due.*/) # comment 20 | encoder.text_token(match, :important) 21 | elsif match = scan(/.+/) # comment 22 | encoder.text_token(match, :comment) 23 | elsif match = scan(/\s+/) # space 24 | encoder.text_token(match, :space) 25 | else # other 26 | encoder.text_token getch, :error 27 | end 28 | end 29 | 30 | encoder 31 | end 32 | 33 | end 34 | 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2005 Kornelius Kalnbach (@murphy_karasu) 2 | 3 | http://coderay.rubychan.de/ 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /coderay.gemspec: -------------------------------------------------------------------------------- 1 | $:.push File.expand_path("../lib", __FILE__) 2 | 3 | require 'coderay/version' 4 | 5 | Gem::Specification.new do |s| 6 | s.name = 'coderay' 7 | 8 | if ENV['RELEASE'] 9 | s.version = CodeRay::VERSION 10 | else 11 | s.version = "#{CodeRay::VERSION}.rc#{ENV['RC'] || 1}" 12 | end 13 | 14 | s.authors = ['Kornelius Kalnbach'] 15 | s.email = ['murphy@rubychan.de'] 16 | s.homepage = 'http://coderay.rubychan.de' 17 | s.summary = 'Fast syntax highlighting for selected languages.' 18 | s.description = 'Fast and easy syntax highlighting for selected languages, written in Ruby. Comes with RedCloth integration and LOC counter.' 19 | 20 | s.license = 'MIT' 21 | 22 | s.platform = Gem::Platform::RUBY 23 | s.required_ruby_version = '>= 1.8.6' 24 | 25 | readme_file = 'README_INDEX.rdoc' 26 | 27 | s.files = `git ls-files -- lib/* #{readme_file} MIT-LICENSE`.split("\n") 28 | s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } 29 | s.require_paths = ['lib'] 30 | 31 | s.rdoc_options = '-Nw2', "-m#{readme_file}", '-t CodeRay Documentation' 32 | s.extra_rdoc_files = readme_file 33 | end 34 | -------------------------------------------------------------------------------- /test/unit/comment_filter.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class CommentFilterTest < Test::Unit::TestCase 5 | 6 | def test_filtering_comments 7 | tokens = CodeRay.scan <<-RUBY, :ruby 8 | #!/usr/bin/env ruby 9 | # a minimal Ruby program 10 | puts "Hello world!" 11 | RUBY 12 | assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text 13 | #!/usr/bin/env ruby 14 | 15 | puts "Hello world!" 16 | RUBY_FILTERED 17 | end 18 | 19 | def test_filtering_docstrings 20 | tokens = CodeRay.scan <<-PYTHON, :python 21 | ''' 22 | Assuming this is file mymodule.py then this string, being the 23 | first statement in the file will become the mymodule modules 24 | docstring when the file is imported 25 | ''' 26 | 27 | class Myclass(): 28 | """ 29 | The class's docstring 30 | """ 31 | 32 | def mymethod(self): 33 | '''The method's docstring''' 34 | 35 | def myfunction(): 36 | """The function's docstring""" 37 | PYTHON 38 | assert_equal <<-PYTHON_FILTERED.chomp, tokens.comment_filter.text 39 | 40 | 41 | class Myclass(): 42 | 43 | 44 | def mymethod(self): 45 | 46 | 47 | def myfunction(): 48 | 49 | 50 | PYTHON_FILTERED 51 | end 52 | 53 | end -------------------------------------------------------------------------------- /FOLDERS: -------------------------------------------------------------------------------- 1 | = CodeRay - folder structure 2 | 3 | == bench - Benchmarking system 4 | 5 | All benchmarking stuff goes here. 6 | 7 | Test inputs are stored in files named example.. 8 | Test outputs go to bench/test.. 9 | 10 | Run bench/bench.rb to get a usage description. 11 | 12 | Run rake bench to perform an example benchmark. 13 | 14 | 15 | == bin - Scripts 16 | 17 | Executional files for CodeRay. 18 | 19 | coderay:: The CodeRay executable. 20 | 21 | == demo - Demos and functional tests 22 | 23 | Demonstrational scripts to show of CodeRay's features. 24 | 25 | Run them as functional tests with rake test:demos. 26 | 27 | 28 | == etc - Lots of stuff 29 | 30 | Some additional files for CodeRay, mainly graphics and Vim scripts. 31 | 32 | 33 | == lib - CodeRay library code 34 | 35 | This is the base directory for the CodeRay library. 36 | 37 | 38 | == rake_helpers - Rake helper libraries 39 | 40 | Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. 41 | 42 | 43 | == test - Tests 44 | 45 | In the subfolder scanners/ are the scanners tests. 46 | Each language has its own subfolder and sub-suite. 47 | 48 | Run with rake test. 49 | -------------------------------------------------------------------------------- /test/unit/filter.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class FilterTest < Test::Unit::TestCase 5 | 6 | def test_creation 7 | filter = nil 8 | assert_nothing_raised do 9 | filter = CodeRay.encoder :filter 10 | end 11 | assert CodeRay::Encoders::Filter < CodeRay::Encoders::Encoder 12 | assert_kind_of CodeRay::Encoders::Encoder, filter 13 | end 14 | 15 | def test_filtering_text_tokens 16 | tokens = CodeRay::Tokens.new 17 | 10.times do |i| 18 | tokens.text_token i.to_s, :index 19 | end 20 | assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens) 21 | assert_equal CodeRay::Tokens, tokens.filter.class 22 | assert_equal tokens, tokens.filter 23 | end 24 | 25 | def test_filtering_block_tokens 26 | tokens = CodeRay::Tokens.new 27 | 10.times do |i| 28 | tokens.begin_group :index 29 | tokens.text_token i.to_s, :content 30 | tokens.end_group :index 31 | tokens.begin_line :index 32 | tokens.text_token i.to_s, :content 33 | tokens.end_line :index 34 | end 35 | assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens) 36 | assert_equal CodeRay::Tokens, tokens.filter.class 37 | assert_equal tokens, tokens.filter 38 | end 39 | 40 | end 41 | -------------------------------------------------------------------------------- /lib/coderay/encoders/filter.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # A Filter encoder has another Tokens instance as output. 5 | # It can be subclass to select, remove, or modify tokens in the stream. 6 | # 7 | # Subclasses of Filter are called "Filters" and can be chained. 8 | # 9 | # == Options 10 | # 11 | # === :tokens 12 | # 13 | # The Tokens object which will receive the output. 14 | # 15 | # Default: Tokens.new 16 | # 17 | # See also: TokenKindFilter 18 | class Filter < Encoder 19 | 20 | register_for :filter 21 | 22 | protected 23 | def setup options 24 | super 25 | 26 | @tokens = options[:tokens] || Tokens.new 27 | end 28 | 29 | def finish options 30 | output @tokens 31 | end 32 | 33 | public 34 | 35 | def text_token text, kind # :nodoc: 36 | @tokens.text_token text, kind 37 | end 38 | 39 | def begin_group kind # :nodoc: 40 | @tokens.begin_group kind 41 | end 42 | 43 | def begin_line kind # :nodoc: 44 | @tokens.begin_line kind 45 | end 46 | 47 | def end_group kind # :nodoc: 48 | @tokens.end_group kind 49 | end 50 | 51 | def end_line kind # :nodoc: 52 | @tokens.end_line kind 53 | end 54 | 55 | end 56 | 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /test/lib/test/unit/util/procwrapper.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2002 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | module Test 8 | module Unit 9 | module Util 10 | 11 | # Allows the storage of a Proc passed through '&' in a 12 | # hash. 13 | # 14 | # Note: this may be inefficient, since the hash being 15 | # used is not necessarily very good. In Observable, 16 | # efficiency is not too important, since the hash is 17 | # only accessed when adding and removing listeners, 18 | # not when notifying. 19 | 20 | class ProcWrapper 21 | 22 | # Creates a new wrapper for a_proc. 23 | def initialize(a_proc) 24 | @a_proc = a_proc 25 | @hash = a_proc.inspect.sub(/^(#<#{a_proc.class}:)/){''}.sub(/(>)$/){''}.hex 26 | end 27 | 28 | def hash 29 | return @hash 30 | end 31 | 32 | def ==(other) 33 | case(other) 34 | when ProcWrapper 35 | return @a_proc == other.to_proc 36 | else 37 | return super 38 | end 39 | end 40 | alias :eql? :== 41 | 42 | def to_proc 43 | return @a_proc 44 | end 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/coderay/encoders/lines_of_code.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # Counts the LoC (Lines of Code). Returns an Integer >= 0. 5 | # 6 | # Alias: +loc+ 7 | # 8 | # Everything that is not comment, markup, doctype/shebang, or an empty line, 9 | # is considered to be code. 10 | # 11 | # For example, 12 | # * HTML files not containing JavaScript have 0 LoC 13 | # * in a Java class without comments, LoC is the number of non-empty lines 14 | # 15 | # A Scanner class should define the token kinds that are not code in the 16 | # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype]. 17 | class LinesOfCode < TokenKindFilter 18 | 19 | register_for :lines_of_code 20 | 21 | NON_EMPTY_LINE = /^\s*\S.*$/ 22 | 23 | protected 24 | 25 | def setup options 26 | if scanner 27 | kinds_not_loc = scanner.class::KINDS_NOT_LOC 28 | else 29 | warn "Tokens have no associated scanner, counting all nonempty lines." if $VERBOSE 30 | kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC 31 | end 32 | 33 | options[:exclude] = kinds_not_loc 34 | 35 | super options 36 | end 37 | 38 | def finish options 39 | output @tokens.text.scan(NON_EMPTY_LINE).size 40 | end 41 | 42 | end 43 | 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /test/lib/test/unit/ui/testrunnerutilities.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2002 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | module Test 8 | module Unit 9 | module UI 10 | 11 | SILENT = 0 12 | PROGRESS_ONLY = 1 13 | NORMAL = 2 14 | VERBOSE = 3 15 | 16 | # Provides some utilities common to most, if not all, 17 | # TestRunners. 18 | # 19 | #-- 20 | # 21 | # Perhaps there ought to be a TestRunner superclass? There 22 | # seems to be a decent amount of shared code between test 23 | # runners. 24 | 25 | module TestRunnerUtilities 26 | 27 | # Creates a new TestRunner and runs the suite. 28 | def run(suite, output_level=NORMAL) 29 | return new(suite, output_level).start 30 | end 31 | 32 | # Takes care of the ARGV parsing and suite 33 | # determination necessary for running one of the 34 | # TestRunners from the command line. 35 | def start_command_line_test 36 | if ARGV.empty? 37 | puts "You should supply the name of a test suite file to the runner" 38 | exit 39 | end 40 | require ARGV[0].gsub(/.+::/, '') 41 | new(eval(ARGV[0])).start 42 | end 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/coderay/helpers/plugin.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # = Plugin 4 | # 5 | # Plugins have to include this module. 6 | # 7 | # IMPORTANT: Use extend for this module. 8 | # 9 | # See CodeRay::PluginHost for examples. 10 | module Plugin 11 | 12 | attr_reader :plugin_id 13 | 14 | # Register this class for the given +id+. 15 | # 16 | # Example: 17 | # class MyPlugin < PluginHost::BaseClass 18 | # register_for :my_id 19 | # ... 20 | # end 21 | # 22 | # See PluginHost.register. 23 | def register_for id 24 | @plugin_id = id 25 | plugin_host.register self, id 26 | end 27 | 28 | # Returns the title of the plugin, or sets it to the 29 | # optional argument +title+. 30 | def title title = nil 31 | if title 32 | @title = title.to_s 33 | else 34 | @title ||= name[/([^:]+)$/, 1] 35 | end 36 | end 37 | 38 | # The PluginHost for this Plugin class. 39 | def plugin_host host = nil 40 | if host.is_a? PluginHost 41 | const_set :PLUGIN_HOST, host 42 | end 43 | self::PLUGIN_HOST 44 | end 45 | 46 | def aliases 47 | plugin_host.plugin_hash.inject [] do |aliases, (key, _)| 48 | aliases << key if plugin_host[key] == self 49 | aliases 50 | end 51 | end 52 | 53 | end 54 | 55 | end 56 | -------------------------------------------------------------------------------- /test/lib/test/unit/util/backtracefilter.rb: -------------------------------------------------------------------------------- 1 | module Test 2 | module Unit 3 | module Util 4 | module BacktraceFilter 5 | TESTUNIT_FILE_SEPARATORS = %r{[\\/:]} 6 | TESTUNIT_PREFIX = __FILE__.split(TESTUNIT_FILE_SEPARATORS)[0..-3] 7 | TESTUNIT_RB_FILE = /\.rb\Z/ 8 | 9 | def filter_backtrace(backtrace, prefix=nil) 10 | return ["No backtrace"] unless(backtrace) 11 | split_p = if(prefix) 12 | prefix.split(TESTUNIT_FILE_SEPARATORS) 13 | else 14 | TESTUNIT_PREFIX 15 | end 16 | match = proc do |e| 17 | split_e = e.split(TESTUNIT_FILE_SEPARATORS)[0, split_p.size] 18 | next false unless(split_e[0..-2] == split_p[0..-2]) 19 | split_e[-1].sub(TESTUNIT_RB_FILE, '') == split_p[-1] 20 | end 21 | return backtrace unless(backtrace.detect(&match)) 22 | found_prefix = false 23 | new_backtrace = backtrace.reverse.reject do |e| 24 | if(match[e]) 25 | found_prefix = true 26 | true 27 | elsif(found_prefix) 28 | false 29 | else 30 | true 31 | end 32 | end.reverse 33 | new_backtrace = (new_backtrace.empty? ? backtrace : new_backtrace) 34 | new_backtrace = new_backtrace.reject(&match) 35 | new_backtrace.empty? ? backtrace : new_backtrace 36 | end 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /test/lib/test/unit/failure.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2002 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | module Test 8 | module Unit 9 | 10 | # Encapsulates a test failure. Created by Test::Unit::TestCase 11 | # when an assertion fails. 12 | class Failure 13 | attr_reader :test_name, :location, :message 14 | 15 | SINGLE_CHARACTER = 'F' 16 | 17 | # Creates a new Failure with the given location and 18 | # message. 19 | def initialize(test_name, location, message) 20 | @test_name = test_name 21 | @location = location 22 | @message = message 23 | end 24 | 25 | # Returns a single character representation of a failure. 26 | def single_character_display 27 | SINGLE_CHARACTER 28 | end 29 | 30 | # Returns a brief version of the error description. 31 | def short_display 32 | "#@test_name: #{@message.split("\n")[0]}" 33 | end 34 | 35 | # Returns a verbose version of the error description. 36 | def long_display 37 | location_display = if(location.size == 1) 38 | location[0].sub(/\A(.+:\d+).*/, ' [\\1]') 39 | else 40 | "\n [#{location.join("\n ")}]" 41 | end 42 | "Failure:\n#@test_name#{location_display}:\n#@message" 43 | end 44 | 45 | # Overridden to return long_display. 46 | def to_s 47 | long_display 48 | end 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/unit/lines_of_code.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | $VERBOSE = true 4 | 5 | require File.expand_path('../../lib/assert_warning', __FILE__) 6 | 7 | class LinesOfCodeTest < Test::Unit::TestCase 8 | 9 | def test_creation 10 | assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder 11 | filter = nil 12 | assert_nothing_raised do 13 | filter = CodeRay.encoder :loc 14 | end 15 | assert_kind_of CodeRay::Encoders::LinesOfCode, filter 16 | assert_nothing_raised do 17 | filter = CodeRay.encoder :lines_of_code 18 | end 19 | assert_kind_of CodeRay::Encoders::LinesOfCode, filter 20 | end 21 | 22 | def test_lines_of_code 23 | tokens = CodeRay.scan <<-RUBY, :ruby 24 | #!/usr/bin/env ruby 25 | 26 | # a minimal Ruby program 27 | puts "Hello world!" 28 | RUBY 29 | assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens) 30 | assert_equal 1, tokens.lines_of_code 31 | assert_equal 1, tokens.loc 32 | end 33 | 34 | class ScannerMockup 35 | KINDS_NOT_LOC = [:space] 36 | end 37 | 38 | def test_filtering_block_tokens 39 | tokens = CodeRay::Tokens.new 40 | tokens.concat ["Hello\n", :world] 41 | tokens.concat ["\n", :space] 42 | tokens.concat ["Hello\n", :comment] 43 | 44 | assert_warning 'Tokens have no associated scanner, counting all nonempty lines.' do 45 | assert_equal 1, tokens.lines_of_code 46 | end 47 | 48 | tokens.scanner = ScannerMockup.new 49 | assert_equal 2, tokens.lines_of_code 50 | end 51 | 52 | end -------------------------------------------------------------------------------- /test/lib/test/unit/error.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2002 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | require 'test/unit/util/backtracefilter' 8 | 9 | module Test 10 | module Unit 11 | 12 | # Encapsulates an error in a test. Created by 13 | # Test::Unit::TestCase when it rescues an exception thrown 14 | # during the processing of a test. 15 | class Error 16 | include Util::BacktraceFilter 17 | 18 | attr_reader(:test_name, :exception) 19 | 20 | SINGLE_CHARACTER = 'E' 21 | 22 | # Creates a new Error with the given test_name and 23 | # exception. 24 | def initialize(test_name, exception) 25 | @test_name = test_name 26 | @exception = exception 27 | end 28 | 29 | # Returns a single character representation of an error. 30 | def single_character_display 31 | SINGLE_CHARACTER 32 | end 33 | 34 | # Returns the message associated with the error. 35 | def message 36 | "#{@exception.class.name}: #{@exception.message}" 37 | end 38 | 39 | # Returns a brief version of the error description. 40 | def short_display 41 | "#@test_name: #{message.split("\n")[0]}" 42 | end 43 | 44 | # Returns a verbose version of the error description. 45 | def long_display 46 | backtrace = filter_backtrace(@exception.backtrace).join("\n ") 47 | "Error:\n#@test_name:\n#{message}\n #{backtrace}" 48 | end 49 | 50 | # Overridden to return long_display. 51 | def to_s 52 | long_display 53 | end 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /bench/bench.rb: -------------------------------------------------------------------------------- 1 | require 'benchmark' 2 | $: << File.expand_path('../../lib', __FILE__) 3 | require 'coderay' 4 | 5 | if ARGV.include? '-h' 6 | puts DATA.read 7 | exit 8 | end 9 | 10 | lang = ARGV.fetch(0, 'ruby') 11 | data = nil 12 | File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read } 13 | raise 'Example file is empty.' if data.empty? 14 | 15 | format = ARGV.fetch(1, 'html').downcase 16 | encoder = CodeRay.encoder(format) 17 | 18 | size = ARGV.fetch(2, 3000).to_i * 1000 19 | unless size.zero? 20 | data += data until data.size >= size 21 | data = data[0, size] 22 | end 23 | size = data.size 24 | puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round] 25 | 26 | n = ARGV.fetch(3, 10).to_s[/\d+/].to_i 27 | require 'profile' if ARGV.include? '-p' 28 | times = [] 29 | n.times do |i| 30 | time = Benchmark.realtime { encoder.encode(data, lang) } 31 | puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0] 32 | times << time 33 | end 34 | 35 | times_sum = times.inject(0) { |time, sum| sum + time } 36 | puts 'Average time: %5.2f s, %4.0f kB/s' % [times_sum / times.size, (size * n) / times_sum / 1000.0] 37 | puts 'Best time: %5.2f s, %4.0f kB/s' % [times.min, size / times.min / 1000.0] 38 | 39 | __END__ 40 | Usage: 41 | ruby bench.rb [lang] [format] [size in kB] [number of runs] 42 | 43 | - lang defaults to ruby. 44 | - format defaults to html. 45 | - size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input. 46 | - number of runs defaults to 5. 47 | 48 | -h prints this help 49 | -p generates a profile (slow, use with SIZE = 1) 50 | -w waits after the benchmark (for debugging memory usw) 51 | -------------------------------------------------------------------------------- /lib/coderay/encoders/xml.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # = XML Encoder 5 | # 6 | # Uses REXML. Very slow. 7 | class XML < Encoder 8 | 9 | register_for :xml 10 | 11 | FILE_EXTENSION = 'xml' 12 | 13 | autoload :REXML, 'rexml/document' 14 | 15 | DEFAULT_OPTIONS = { 16 | :tab_width => 8, 17 | :pretty => -1, 18 | :transitive => false, 19 | } 20 | 21 | protected 22 | def setup options 23 | super 24 | 25 | @doc = REXML::Document.new 26 | @doc << REXML::XMLDecl.new 27 | @tab_width = options[:tab_width] 28 | @root = @node = @doc.add_element('coderay-tokens') 29 | end 30 | 31 | def finish options 32 | @doc.write @out, options[:pretty], options[:transitive], true 33 | 34 | super 35 | end 36 | 37 | public 38 | def text_token text, kind 39 | if kind == :space 40 | token = @node 41 | else 42 | token = @node.add_element kind.to_s 43 | end 44 | text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl| 45 | case 46 | when space 47 | token << REXML::Text.new(space, true) 48 | when tab 49 | token << REXML::Text.new(tab, true) 50 | when nl 51 | token << REXML::Text.new(nl, true) 52 | else 53 | token << REXML::Text.new($&) 54 | end 55 | end 56 | end 57 | 58 | def begin_group kind 59 | @node = @node.add_element kind.to_s 60 | end 61 | 62 | def end_group kind 63 | if @node == @root 64 | raise 'no token to close!' 65 | end 66 | @node = @node.parent 67 | end 68 | 69 | end 70 | 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/coderay/encoders/debug_lint.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | load :lint 5 | 6 | # = Debug Lint Encoder 7 | # 8 | # Debug encoder with additional checks for: 9 | # 10 | # - empty tokens 11 | # - incorrect nesting 12 | # 13 | # It will raise an InvalidTokenStream exception when any of the above occurs. 14 | # 15 | # See also: Encoders::Debug 16 | class DebugLint < Debug 17 | 18 | register_for :debug_lint 19 | 20 | def text_token text, kind 21 | raise Lint::EmptyToken, 'empty token for %p' % [kind] if text.empty? 22 | raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind 23 | super 24 | end 25 | 26 | def begin_group kind 27 | @opened << kind 28 | super 29 | end 30 | 31 | def end_group kind 32 | raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind 33 | @opened.pop 34 | super 35 | end 36 | 37 | def begin_line kind 38 | @opened << kind 39 | super 40 | end 41 | 42 | def end_line kind 43 | raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind 44 | @opened.pop 45 | super 46 | end 47 | 48 | protected 49 | 50 | def setup options 51 | super 52 | @opened = [] 53 | end 54 | 55 | def finish options 56 | raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? 57 | super 58 | end 59 | 60 | end 61 | 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/coderay/tokens_proxy.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # The result of a scan operation is a TokensProxy, but should act like Tokens. 4 | # 5 | # This proxy makes it possible to use the classic CodeRay.scan.encode API 6 | # while still providing the benefits of direct streaming. 7 | class TokensProxy 8 | 9 | attr_accessor :input, :lang, :options, :block 10 | 11 | # Create a new TokensProxy with the arguments of CodeRay.scan. 12 | def initialize input, lang, options = {}, block = nil 13 | @input = input 14 | @lang = lang 15 | @options = options 16 | @block = block 17 | end 18 | 19 | # Call CodeRay.encode if +encoder+ is a Symbol; 20 | # otherwise, convert the receiver to tokens and call encoder.encode_tokens. 21 | def encode encoder, options = {} 22 | if encoder.respond_to? :to_sym 23 | CodeRay.encode(input, lang, encoder, options) 24 | else 25 | encoder.encode_tokens tokens, options 26 | end 27 | end 28 | 29 | # Tries to call encode; 30 | # delegates to tokens otherwise. 31 | def method_missing method, *args, &blk 32 | encode method.to_sym, *args 33 | rescue PluginHost::PluginNotFound 34 | tokens.send(method, *args, &blk) 35 | end 36 | 37 | # The (cached) result of the tokenized input; a Tokens instance. 38 | def tokens 39 | @tokens ||= scanner.tokenize(input) 40 | end 41 | 42 | # A (cached) scanner instance to use for the scan task. 43 | def scanner 44 | @scanner ||= CodeRay.scanner(lang, options, &block) 45 | end 46 | 47 | # Overwrite Struct#each. 48 | def each *args, &blk 49 | tokens.each(*args, &blk) 50 | self 51 | end 52 | 53 | end 54 | 55 | end 56 | -------------------------------------------------------------------------------- /lib/coderay/encoders/lint.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # = Lint Encoder 5 | # 6 | # Checks for: 7 | # 8 | # - empty tokens 9 | # - incorrect nesting 10 | # 11 | # It will raise an InvalidTokenStream exception when any of the above occurs. 12 | # 13 | # See also: Encoders::DebugLint 14 | class Lint < Debug 15 | 16 | register_for :lint 17 | 18 | InvalidTokenStream = Class.new StandardError 19 | EmptyToken = Class.new InvalidTokenStream 20 | UnknownTokenKind = Class.new InvalidTokenStream 21 | IncorrectTokenGroupNesting = Class.new InvalidTokenStream 22 | 23 | def text_token text, kind 24 | raise EmptyToken, 'empty token for %p' % [kind] if text.empty? 25 | raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind 26 | end 27 | 28 | def begin_group kind 29 | @opened << kind 30 | end 31 | 32 | def end_group kind 33 | raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind 34 | @opened.pop 35 | end 36 | 37 | def begin_line kind 38 | @opened << kind 39 | end 40 | 41 | def end_line kind 42 | raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind 43 | @opened.pop 44 | end 45 | 46 | protected 47 | 48 | def setup options 49 | @opened = [] 50 | end 51 | 52 | def finish options 53 | raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? 54 | end 55 | 56 | end 57 | 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/coderay/encoders/html/css.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | class HTML 5 | class CSS # :nodoc: 6 | 7 | attr :stylesheet 8 | 9 | def CSS.load_stylesheet style = nil 10 | CodeRay::Styles[style] 11 | end 12 | 13 | def initialize style = :default 14 | @styles = Hash.new 15 | style = CSS.load_stylesheet style 16 | @stylesheet = [ 17 | style::CSS_MAIN_STYLES, 18 | style::TOKEN_COLORS.gsub(/^(?!$)/, '.CodeRay ') 19 | ].join("\n") 20 | parse style::TOKEN_COLORS 21 | end 22 | 23 | def get_style_for_css_classes css_classes 24 | cl = @styles[css_classes.first] 25 | return '' unless cl 26 | style = '' 27 | 1.upto css_classes.size do |offset| 28 | break if style = cl[css_classes[offset .. -1]] 29 | end 30 | # warn 'Style not found: %p' % [styles] if style.empty? 31 | return style 32 | end 33 | 34 | private 35 | 36 | CSS_CLASS_PATTERN = / 37 | ( # $1 = selectors 38 | (?: 39 | (?: \s* \. [-\w]+ )+ 40 | \s* ,? 41 | )+ 42 | ) 43 | \s* \{ \s* 44 | ( [^\}]+ )? # $2 = style 45 | \s* \} \s* 46 | | 47 | ( [^\n]+ ) # $3 = error 48 | /mx 49 | def parse stylesheet 50 | stylesheet.scan CSS_CLASS_PATTERN do |selectors, style, error| 51 | raise "CSS parse error: '#{error.inspect}' not recognized" if error 52 | for selector in selectors.split(',') 53 | classes = selector.scan(/[-\w]+/) 54 | cl = classes.pop 55 | @styles[cl] ||= Hash.new 56 | @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') 57 | end 58 | end 59 | end 60 | 61 | end 62 | end 63 | 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /test/unit/word_list.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay/helpers/word_list' 3 | 4 | class WordListTest < Test::Unit::TestCase 5 | 6 | include CodeRay 7 | 8 | # define word arrays 9 | RESERVED_WORDS = %w[ 10 | asm break case continue default do else 11 | ... 12 | ] 13 | 14 | PREDEFINED_TYPES = %w[ 15 | int long short char void 16 | ... 17 | ] 18 | 19 | PREDEFINED_CONSTANTS = %w[ 20 | EOF NULL ... 21 | ] 22 | 23 | # make a WordList 24 | IDENT_KIND = WordList.new(:ident). 25 | add(RESERVED_WORDS, :reserved). 26 | add(PREDEFINED_TYPES, :predefined_type). 27 | add(PREDEFINED_CONSTANTS, :predefined_constant) 28 | 29 | def test_word_list_example 30 | assert_equal :predefined_type, IDENT_KIND['void'] 31 | # assert_equal :predefined_constant, IDENT_KIND['...'] # not specified 32 | end 33 | 34 | def test_word_list 35 | list = WordList.new(:ident).add(['foobar'], :reserved) 36 | assert_equal :reserved, list['foobar'] 37 | assert_equal :ident, list['FooBar'] 38 | assert_equal 1, list.size 39 | end 40 | 41 | def test_case_ignoring_word_list 42 | list = WordList::CaseIgnoring.new(:ident).add(['foobar'], :reserved) 43 | assert_equal :ident, list['foo'] 44 | assert_equal :reserved, list['foobar'] 45 | assert_equal :reserved, list['FooBar'] 46 | assert_equal 1, list.size 47 | 48 | list = WordList::CaseIgnoring.new(:ident).add(['FooBar'], :reserved) 49 | assert_equal :ident, list['foo'] 50 | assert_equal :reserved, list['foobar'] 51 | assert_equal :reserved, list['FooBar'] 52 | assert_equal 1, list.size 53 | end 54 | 55 | def test_dup 56 | list = WordList.new(:ident).add(['foobar'], :reserved) 57 | assert_equal :reserved, list['foobar'] 58 | list2 = list.dup 59 | list2.add(%w[foobar], :keyword) 60 | assert_equal :keyword, list2['foobar'] 61 | assert_equal :reserved, list['foobar'] 62 | end 63 | 64 | end 65 | -------------------------------------------------------------------------------- /lib/coderay/helpers/word_list.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # = WordList 4 | # 5 | # A Hash subclass designed for mapping word lists to token types. 6 | # 7 | # A WordList is a Hash with some additional features. 8 | # It is intended to be used for keyword recognition. 9 | # 10 | # WordList is optimized to be used in Scanners, 11 | # typically to decide whether a given ident is a special token. 12 | # 13 | # For case insensitive words use WordList::CaseIgnoring. 14 | # 15 | # Example: 16 | # 17 | # # define word arrays 18 | # RESERVED_WORDS = %w[ 19 | # asm break case continue default do else 20 | # ] 21 | # 22 | # PREDEFINED_TYPES = %w[ 23 | # int long short char void 24 | # ] 25 | # 26 | # # make a WordList 27 | # IDENT_KIND = WordList.new(:ident). 28 | # add(RESERVED_WORDS, :reserved). 29 | # add(PREDEFINED_TYPES, :predefined_type) 30 | # 31 | # ... 32 | # 33 | # def scan_tokens tokens, options 34 | # ... 35 | # 36 | # elsif scan(/[A-Za-z_][A-Za-z_0-9]*/) 37 | # # use it 38 | # kind = IDENT_KIND[match] 39 | # ... 40 | class WordList < Hash 41 | 42 | # Create a new WordList with +default+ as default value. 43 | def initialize default = false 44 | super default 45 | end 46 | 47 | # Add words to the list and associate them with +value+. 48 | # 49 | # Returns +self+, so you can concat add calls. 50 | def add words, value = true 51 | words.each { |word| self[word] = value } 52 | self 53 | end 54 | 55 | end 56 | 57 | 58 | # A CaseIgnoring WordList is like a WordList, only that 59 | # keys are compared case-insensitively (normalizing keys using +downcase+). 60 | class WordList::CaseIgnoring < WordList 61 | 62 | def [] key 63 | super key.downcase 64 | end 65 | 66 | def []= key, value 67 | super key.downcase, value 68 | end 69 | 70 | end 71 | 72 | end 73 | -------------------------------------------------------------------------------- /test/unit/statistic.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class StatisticEncoderTest < Test::Unit::TestCase 5 | 6 | def test_creation 7 | assert CodeRay::Encoders::Statistic < CodeRay::Encoders::Encoder 8 | stats = nil 9 | assert_nothing_raised do 10 | stats = CodeRay.encoder :statistic 11 | end 12 | assert_kind_of CodeRay::Encoders::Encoder, stats 13 | end 14 | 15 | TEST_INPUT = CodeRay::Tokens[ 16 | ['10', :integer], 17 | ['(\\)', :operator], 18 | [:begin_group, :string], 19 | ['test', :content], 20 | [:end_group, :string], 21 | [:begin_line, :test], 22 | ["\n", :space], 23 | ["\n \t", :space], 24 | [" \n", :space], 25 | ["[]", :method], 26 | [:end_line, :test], 27 | ] 28 | TEST_INPUT.flatten! 29 | TEST_OUTPUT = <<-'DEBUG' 30 | 31 | Code Statistics 32 | 33 | Tokens 11 34 | Non-Whitespace 4 35 | Bytes Total 20 36 | 37 | Token Types (7): 38 | type count ratio size (average) 39 | ------------------------------------------------------------- 40 | TOTAL 11 100.00 % 1.8 41 | space 3 27.27 % 3.0 42 | string 2 18.18 % 0.0 43 | test 2 18.18 % 0.0 44 | :begin_group 1 9.09 % 0.0 45 | :begin_line 1 9.09 % 0.0 46 | :end_group 1 9.09 % 0.0 47 | :end_line 1 9.09 % 0.0 48 | content 1 9.09 % 4.0 49 | integer 1 9.09 % 2.0 50 | method 1 9.09 % 2.0 51 | operator 1 9.09 % 3.0 52 | 53 | DEBUG 54 | 55 | def test_filtering_text_tokens 56 | assert_equal TEST_OUTPUT, CodeRay::Encoders::Statistic.new.encode_tokens(TEST_INPUT) 57 | assert_equal TEST_OUTPUT, TEST_INPUT.statistic 58 | end 59 | 60 | end 61 | -------------------------------------------------------------------------------- /test/unit/plugin.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'pathname' 3 | 4 | $:.unshift File.expand_path('../../../lib', __FILE__) 5 | require 'coderay' 6 | 7 | class PluginScannerTest < Test::Unit::TestCase 8 | 9 | module Plugins 10 | extend CodeRay::PluginHost 11 | plugin_path File.dirname(__FILE__), 'plugins' 12 | class Plugin 13 | extend CodeRay::Plugin 14 | plugin_host Plugins 15 | end 16 | end 17 | 18 | module PluginsWithDefault 19 | extend CodeRay::PluginHost 20 | plugin_path File.dirname(__FILE__), 'plugins_with_default' 21 | class Plugin 22 | extend CodeRay::Plugin 23 | plugin_host PluginsWithDefault 24 | end 25 | default :default_plugin 26 | end 27 | 28 | def test_load 29 | require Pathname.new(__FILE__).realpath.dirname + 'plugins' + 'user_defined' + 'user_plugin' 30 | assert_equal 'UserPlugin', Plugins.load(:user_plugin).name 31 | end 32 | 33 | def test_load_all 34 | assert_instance_of Symbol, Plugins.load_all.first 35 | assert_operator Plugins.all_plugins.first, :<, Plugins::Plugin 36 | assert_equal 'The Example', Plugins.all_plugins.map { |plugin| plugin.title }.sort.first 37 | end 38 | 39 | def test_default 40 | assert_nothing_raised do 41 | assert_operator PluginsWithDefault[:gargamel], :<, PluginsWithDefault::Plugin 42 | end 43 | assert_equal PluginsWithDefault::Default, PluginsWithDefault.default 44 | end 45 | 46 | def test_plugin_not_found 47 | assert_raise CodeRay::PluginHost::PluginNotFound do 48 | Plugins[:thestral] 49 | end 50 | assert_raise ArgumentError do 51 | Plugins[14] 52 | end 53 | assert_raise ArgumentError do 54 | Plugins['test/test'] 55 | end 56 | assert_raise CodeRay::PluginHost::PluginNotFound do 57 | PluginsWithDefault[:example_without_register_for] 58 | end 59 | end 60 | 61 | def test_autoload_constants 62 | assert_operator Plugins::Example, :<, Plugins::Plugin 63 | end 64 | 65 | def test_title 66 | assert_equal 'The Example', Plugins::Example.title 67 | end 68 | 69 | end 70 | -------------------------------------------------------------------------------- /lib/coderay/scanners/debug.rb: -------------------------------------------------------------------------------- 1 | require 'set' 2 | 3 | module CodeRay 4 | module Scanners 5 | 6 | # = Debug Scanner 7 | # 8 | # Interprets the output of the Encoders::Debug encoder (basically the inverse function). 9 | class Debug < Scanner 10 | 11 | register_for :debug 12 | title 'CodeRay Token Dump Import' 13 | 14 | protected 15 | 16 | def setup 17 | super 18 | @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set 19 | end 20 | 21 | def scan_tokens encoder, options 22 | 23 | opened_tokens = [] 24 | 25 | until eos? 26 | 27 | if match = scan(/\s+/) 28 | encoder.text_token match, :space 29 | 30 | elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x) 31 | if @known_token_kinds.include? self[1] 32 | encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym 33 | else 34 | encoder.text_token matched, :unknown 35 | end 36 | 37 | elsif match = scan(/ (\w+) ([<\[]) /x) 38 | if @known_token_kinds.include? self[1] 39 | kind = self[1].to_sym 40 | else 41 | kind = :unknown 42 | end 43 | 44 | opened_tokens << kind 45 | case self[2] 46 | when '<' 47 | encoder.begin_group kind 48 | when '[' 49 | encoder.begin_line kind 50 | else 51 | raise 'CodeRay bug: This case should not be reached.' 52 | end 53 | 54 | elsif !opened_tokens.empty? && match = scan(/ > /x) 55 | encoder.end_group opened_tokens.pop 56 | 57 | elsif !opened_tokens.empty? && match = scan(/ \] /x) 58 | encoder.end_line opened_tokens.pop 59 | 60 | else 61 | encoder.text_token getch, :space 62 | 63 | end 64 | 65 | end 66 | 67 | encoder.end_group opened_tokens.pop until opened_tokens.empty? 68 | 69 | encoder 70 | end 71 | 72 | end 73 | 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/coderay/scanners/raydebug.rb: -------------------------------------------------------------------------------- 1 | require 'set' 2 | 3 | module CodeRay 4 | module Scanners 5 | 6 | # = Raydebug Scanner 7 | # 8 | # Highlights the output of the Encoders::Debug encoder. 9 | class Raydebug < Scanner 10 | 11 | register_for :raydebug 12 | file_extension 'raydebug' 13 | title 'CodeRay Token Dump' 14 | 15 | protected 16 | 17 | def setup 18 | super 19 | @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set 20 | end 21 | 22 | def scan_tokens encoder, options 23 | 24 | opened_tokens = [] 25 | 26 | until eos? 27 | 28 | if match = scan(/\s+/) 29 | encoder.text_token match, :space 30 | 31 | elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) /x) 32 | kind = self[1] 33 | encoder.text_token kind, :class 34 | encoder.text_token '(', :operator 35 | match = self[2] 36 | unless match.empty? 37 | if @known_token_kinds.include? kind 38 | encoder.text_token match, kind.to_sym 39 | else 40 | encoder.text_token match, :plain 41 | end 42 | end 43 | encoder.text_token match, :operator if match = scan(/\)/) 44 | 45 | elsif match = scan(/ (\w+) ([<\[]) /x) 46 | encoder.text_token self[1], :class 47 | if @known_token_kinds.include? self[1] 48 | kind = self[1].to_sym 49 | else 50 | kind = :unknown 51 | end 52 | opened_tokens << kind 53 | encoder.begin_group kind 54 | encoder.text_token self[2], :operator 55 | 56 | elsif !opened_tokens.empty? && match = scan(/ [>\]] /x) 57 | encoder.text_token match, :operator 58 | encoder.end_group opened_tokens.pop 59 | 60 | else 61 | encoder.text_token getch, :space 62 | 63 | end 64 | 65 | end 66 | 67 | encoder.end_group opened_tokens.pop until opened_tokens.empty? 68 | 69 | encoder 70 | end 71 | 72 | end 73 | 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/coderay/scanners/erb.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | load :html 5 | load :ruby 6 | 7 | # Scanner for HTML ERB templates. 8 | class ERB < Scanner 9 | 10 | register_for :erb 11 | title 'HTML ERB Template' 12 | 13 | KINDS_NOT_LOC = HTML::KINDS_NOT_LOC 14 | 15 | ERB_RUBY_BLOCK = / 16 | (<%(?!%)[-=\#]?) 17 | ((?> 18 | [^\-%]* # normal* 19 | (?> # special 20 | (?: %(?!>) | -(?!%>) ) 21 | [^\-%]* # normal* 22 | )* 23 | )) 24 | ((?: -?%> )?) 25 | /x # :nodoc: 26 | 27 | START_OF_ERB = / 28 | <%(?!%) 29 | /x # :nodoc: 30 | 31 | protected 32 | 33 | def setup 34 | @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true 35 | @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true 36 | end 37 | 38 | def reset_instance 39 | super 40 | @html_scanner.reset 41 | end 42 | 43 | def scan_tokens encoder, options 44 | 45 | until eos? 46 | 47 | if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_rest) and not match.empty? 48 | @html_scanner.tokenize match, :tokens => encoder 49 | 50 | elsif match = scan(/#{ERB_RUBY_BLOCK}/o) 51 | start_tag = self[1] 52 | code = self[2] 53 | end_tag = self[3] 54 | 55 | encoder.begin_group :inline 56 | encoder.text_token start_tag, :inline_delimiter 57 | 58 | if start_tag == '<%#' 59 | encoder.text_token code, :comment 60 | else 61 | @ruby_scanner.tokenize code, :tokens => encoder 62 | end unless code.empty? 63 | 64 | encoder.text_token end_tag, :inline_delimiter unless end_tag.empty? 65 | encoder.end_group :inline 66 | 67 | else 68 | raise_inspect 'else-case reached!', encoder 69 | 70 | end 71 | 72 | end 73 | 74 | encoder 75 | 76 | end 77 | 78 | end 79 | 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/coderay/encoders/json.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # A simple JSON Encoder. 5 | # 6 | # Example: 7 | # CodeRay.scan('puts "Hello world!"', :ruby).json 8 | # yields 9 | # [ 10 | # {"type"=>"text", "text"=>"puts", "kind"=>"ident"}, 11 | # {"type"=>"text", "text"=>" ", "kind"=>"space"}, 12 | # {"type"=>"block", "action"=>"open", "kind"=>"string"}, 13 | # {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"}, 14 | # {"type"=>"text", "text"=>"Hello world!", "kind"=>"content"}, 15 | # {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"}, 16 | # {"type"=>"block", "action"=>"close", "kind"=>"string"}, 17 | # ] 18 | class JSON < Encoder 19 | 20 | begin 21 | require 'json' 22 | rescue LoadError 23 | begin 24 | require 'rubygems' unless defined? Gem 25 | gem 'json' 26 | require 'json' 27 | rescue LoadError 28 | $stderr.puts "The JSON encoder needs the JSON library.\n" \ 29 | "Please gem install json." 30 | raise 31 | end 32 | end 33 | 34 | register_for :json 35 | FILE_EXTENSION = 'json' 36 | 37 | protected 38 | def setup options 39 | super 40 | 41 | @first = true 42 | @out << '[' 43 | end 44 | 45 | def finish options 46 | @out << ']' 47 | end 48 | 49 | def append data 50 | if @first 51 | @first = false 52 | else 53 | @out << ',' 54 | end 55 | 56 | @out << data.to_json 57 | end 58 | 59 | public 60 | def text_token text, kind 61 | append :type => 'text', :text => text, :kind => kind 62 | end 63 | 64 | def begin_group kind 65 | append :type => 'block', :action => 'open', :kind => kind 66 | end 67 | 68 | def end_group kind 69 | append :type => 'block', :action => 'close', :kind => kind 70 | end 71 | 72 | def begin_line kind 73 | append :type => 'block', :action => 'begin_line', :kind => kind 74 | end 75 | 76 | def end_line kind 77 | append :type => 'block', :action => 'end_line', :kind => kind 78 | end 79 | 80 | end 81 | 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /test/unit/debug.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class DebugEncoderTest < Test::Unit::TestCase 5 | 6 | def test_creation 7 | debug = nil 8 | assert_nothing_raised do 9 | debug = CodeRay.encoder :debug 10 | end 11 | assert CodeRay::Encoders::Debug < CodeRay::Encoders::Encoder 12 | assert_kind_of CodeRay::Encoders::Encoder, debug 13 | end 14 | 15 | TEST_INPUT = CodeRay::Tokens[ 16 | ['10', :integer], 17 | ['(\\)', :operator], 18 | [:begin_group, :string], 19 | ['test', :content], 20 | [:end_group, :string], 21 | [:begin_line, :head], 22 | ["\n", :space], 23 | ["\n \t", :space], 24 | [" \n", :space], 25 | ["[]", :method], 26 | [:end_line, :head], 27 | ] 28 | TEST_INPUT.flatten! 29 | TEST_OUTPUT = <<-'DEBUG'.chomp 30 | integer(10)operator((\\\))stringhead[ 31 | 32 | 33 | method([])] 34 | DEBUG 35 | 36 | def test_filtering_text_tokens 37 | assert_equal TEST_OUTPUT, CodeRay::Encoders::Debug.new.encode_tokens(TEST_INPUT) 38 | assert_equal TEST_OUTPUT, TEST_INPUT.debug 39 | end 40 | 41 | end 42 | 43 | class DebugScannerTest < Test::Unit::TestCase 44 | 45 | def test_creation 46 | assert CodeRay::Scanners::Debug < CodeRay::Scanners::Scanner 47 | debug = nil 48 | assert_nothing_raised do 49 | debug = CodeRay.scanner :debug 50 | end 51 | assert_kind_of CodeRay::Scanners::Scanner, debug 52 | end 53 | 54 | TEST_INPUT = <<-'DEBUG'.chomp 55 | integer(10)operator((\\\))stringtest[ 56 | 57 | 58 | method([])] 59 | DEBUG 60 | TEST_OUTPUT = CodeRay::Tokens[ 61 | ['10', :integer], 62 | ['(\\)', :operator], 63 | [:begin_group, :string], 64 | ['test', :content], 65 | [:end_group, :string], 66 | [:begin_line, :unknown], 67 | ["\n\n \t \n", :space], 68 | ["[]", :method], 69 | [:end_line, :unknown], 70 | ].flatten 71 | 72 | def test_filtering_text_tokens 73 | assert_equal TEST_OUTPUT, CodeRay::Scanners::Debug.new.tokenize(TEST_INPUT) 74 | assert_kind_of CodeRay::TokensProxy, CodeRay.scan(TEST_INPUT, :debug) 75 | assert_equal TEST_OUTPUT, CodeRay.scan(TEST_INPUT, :debug).tokens 76 | end 77 | 78 | end 79 | -------------------------------------------------------------------------------- /test/lib/test/unit/testsuite.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2003 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | module Test 8 | module Unit 9 | 10 | # A collection of tests which can be #run. 11 | # 12 | # Note: It is easy to confuse a TestSuite instance with 13 | # something that has a static suite method; I know because _I_ 14 | # have trouble keeping them straight. Think of something that 15 | # has a suite method as simply providing a way to get a 16 | # meaningful TestSuite instance. 17 | class TestSuite 18 | attr_reader :name, :tests 19 | 20 | STARTED = name + "::STARTED" 21 | FINISHED = name + "::FINISHED" 22 | 23 | # Creates a new TestSuite with the given name. 24 | def initialize(name="Unnamed TestSuite") 25 | @name = name 26 | @tests = [] 27 | end 28 | 29 | # Runs the tests and/or suites contained in this 30 | # TestSuite. 31 | def run(result, &progress_block) 32 | yield(STARTED, name) 33 | @tests.each do |test| 34 | test.run(result, &progress_block) 35 | end 36 | yield(FINISHED, name) 37 | end 38 | 39 | # Adds the test to the suite. 40 | def <<(test) 41 | @tests << test 42 | self 43 | end 44 | 45 | def delete(test) 46 | @tests.delete(test) 47 | end 48 | 49 | # Retuns the rolled up number of tests in this suite; 50 | # i.e. if the suite contains other suites, it counts the 51 | # tests within those suites, not the suites themselves. 52 | def size 53 | total_size = 0 54 | @tests.each { |test| total_size += test.size } 55 | total_size 56 | end 57 | 58 | def empty? 59 | tests.empty? 60 | end 61 | 62 | # Overridden to return the name given the suite at 63 | # creation. 64 | def to_s 65 | @name 66 | end 67 | 68 | # It's handy to be able to compare TestSuite instances. 69 | def ==(other) 70 | return false unless(other.kind_of?(self.class)) 71 | return false unless(@name == other.name) 72 | @tests == other.tests 73 | end 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /test/unit/token_kind_filter.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class TokenKindFilterTest < Test::Unit::TestCase 5 | 6 | def test_creation 7 | assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Encoder 8 | assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Filter 9 | filter = nil 10 | assert_nothing_raised do 11 | filter = CodeRay.encoder :token_kind_filter 12 | end 13 | assert_instance_of CodeRay::Encoders::TokenKindFilter, filter 14 | end 15 | 16 | def test_filtering_text_tokens 17 | tokens = CodeRay::Tokens.new 18 | for i in 1..10 19 | tokens.text_token i.to_s, :index 20 | tokens.text_token ' ', :space if i < 10 21 | end 22 | assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).count 23 | assert_equal 10, tokens.token_kind_filter(:exclude => :space).count 24 | assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).count 25 | assert_equal 9, tokens.token_kind_filter(:include => :space).count 26 | assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).count 27 | assert_equal 0, tokens.token_kind_filter(:exclude => :all).count 28 | end 29 | 30 | def test_filtering_block_tokens 31 | tokens = CodeRay::Tokens.new 32 | 10.times do |i| 33 | tokens.begin_group :index 34 | tokens.text_token i.to_s, :content 35 | tokens.end_group :index 36 | tokens.begin_group :naught if i == 5 37 | tokens.end_group :naught if i == 7 38 | tokens.begin_line :blubb 39 | tokens.text_token i.to_s, :content 40 | tokens.end_line :blubb 41 | end 42 | assert_equal 16, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).count 43 | assert_equal 16, tokens.token_kind_filter(:include => :blubb).count 44 | assert_equal 24, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => [:blubb, :content]).count 45 | assert_equal 24, tokens.token_kind_filter(:include => [:blubb, :content]).count 46 | assert_equal 32, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).count 47 | assert_equal 32, tokens.token_kind_filter(:exclude => :index).count 48 | end 49 | 50 | end 51 | -------------------------------------------------------------------------------- /test/unit/tokens.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class TokensTest < Test::Unit::TestCase 5 | 6 | def test_creation 7 | assert CodeRay::Tokens < Array 8 | tokens = nil 9 | assert_nothing_raised do 10 | tokens = CodeRay::Tokens.new 11 | end 12 | assert_kind_of Array, tokens 13 | end 14 | 15 | def test_adding_tokens 16 | tokens = make_tokens 17 | assert_equal tokens.size, 8 18 | assert_equal tokens.count, 4 19 | end 20 | 21 | def test_to_s 22 | assert_equal 'string()', make_tokens.to_s 23 | end 24 | 25 | def test_encode_with_nonsense 26 | assert_raise NoMethodError do 27 | make_tokens.nonsense 28 | end 29 | end 30 | 31 | def test_split_into_parts 32 | parts_4_3 = [ 33 | ["stri", :type], 34 | ["ng", :type, :begin_group, :operator, "(", :content, :end_group, :operator], 35 | [:begin_group, :operator, ")", :content, :end_group, :operator] 36 | ] 37 | assert_equal parts_4_3, make_tokens.split_into_parts(4, 3) 38 | assert_equal [make_tokens.to_a], make_tokens.split_into_parts 39 | 40 | parts_7_0_1 = [ 41 | ["string", :type, :begin_group, :operator, "(", :content, :end_group, :operator], 42 | [], 43 | [:begin_group, :operator, ")", :content, :end_group, :operator] 44 | ] 45 | assert_equal parts_7_0_1, make_tokens.split_into_parts(7, 0, 1) 46 | 47 | line = CodeRay::Tokens[:begin_line, :head, '...', :plain] 48 | line_parts = [ 49 | [:begin_line, :head, ".", :plain, :end_line, :head], 50 | [:begin_line, :head, "..", :plain] 51 | ] 52 | assert_equal line_parts, line.split_into_parts(1) 53 | 54 | assert_raise ArgumentError do 55 | CodeRay::Tokens[:bullshit, :input].split_into_parts 56 | end 57 | assert_raise ArgumentError do 58 | CodeRay::Tokens[42, 43].split_into_parts 59 | end 60 | end 61 | 62 | def test_encode 63 | assert_match(/\A\[\{(?:"type":"text"|"text":"string"|"kind":"type"|,){5}\},/, make_tokens.encode(:json)) 64 | end 65 | 66 | def make_tokens 67 | tokens = CodeRay::Tokens.new 68 | assert_nothing_raised do 69 | tokens.text_token 'string', :type 70 | tokens.begin_group :operator 71 | tokens.text_token '()', :content 72 | tokens.end_group :operator 73 | end 74 | tokens 75 | end 76 | 77 | end -------------------------------------------------------------------------------- /test/lib/test/unit/ui/testrunnermediator.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2002 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | require 'test/unit' 8 | require 'test/unit/util/observable' 9 | require 'test/unit/testresult' 10 | 11 | module Test 12 | module Unit 13 | module UI 14 | 15 | # Provides an interface to write any given UI against, 16 | # hopefully making it easy to write new UIs. 17 | class TestRunnerMediator 18 | RESET = name + "::RESET" 19 | STARTED = name + "::STARTED" 20 | FINISHED = name + "::FINISHED" 21 | 22 | include Util::Observable 23 | 24 | # Creates a new TestRunnerMediator initialized to run 25 | # the passed suite. 26 | def initialize(suite) 27 | @suite = suite 28 | end 29 | 30 | # Runs the suite the TestRunnerMediator was created 31 | # with. 32 | def run_suite 33 | Unit.run = true 34 | begin_time = Time.now 35 | notify_listeners(RESET, @suite.size) 36 | result = create_result 37 | notify_listeners(STARTED, result) 38 | result_listener = result.add_listener(TestResult::CHANGED) do |updated_result| 39 | notify_listeners(TestResult::CHANGED, updated_result) 40 | end 41 | 42 | fault_listener = result.add_listener(TestResult::FAULT) do |fault| 43 | notify_listeners(TestResult::FAULT, fault) 44 | end 45 | 46 | @suite.run(result) do |channel, value| 47 | notify_listeners(channel, value) 48 | end 49 | 50 | result.remove_listener(TestResult::FAULT, fault_listener) 51 | result.remove_listener(TestResult::CHANGED, result_listener) 52 | end_time = Time.now 53 | elapsed_time = end_time - begin_time 54 | notify_listeners(FINISHED, elapsed_time) #"Finished in #{elapsed_time} seconds.") 55 | return result 56 | end 57 | 58 | private 59 | # A factory method to create the result the mediator 60 | # should run with. Can be overridden by subclasses if 61 | # one wants to use a different result. 62 | def create_result 63 | return TestResult.new 64 | end 65 | end 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /test/lib/test/unit/testresult.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # Author:: Nathaniel Talbott. 3 | # Copyright:: Copyright (c) 2000-2002 Nathaniel Talbott. All rights reserved. 4 | # License:: Ruby license. 5 | 6 | require 'test/unit/util/observable' 7 | 8 | module Test 9 | module Unit 10 | 11 | # Collects Test::Unit::Failure and Test::Unit::Error so that 12 | # they can be displayed to the user. To this end, observers 13 | # can be added to it, allowing the dynamic updating of, say, a 14 | # UI. 15 | class TestResult 16 | include Util::Observable 17 | 18 | CHANGED = "CHANGED" 19 | FAULT = "FAULT" 20 | 21 | attr_reader(:run_count, :assertion_count) 22 | 23 | # Constructs a new, empty TestResult. 24 | def initialize 25 | @run_count, @assertion_count = 0, 0 26 | @failures, @errors = Array.new, Array.new 27 | end 28 | 29 | # Records a test run. 30 | def add_run 31 | @run_count += 1 32 | notify_listeners(CHANGED, self) 33 | end 34 | 35 | # Records a Test::Unit::Failure. 36 | def add_failure(failure) 37 | @failures << failure 38 | notify_listeners(FAULT, failure) 39 | notify_listeners(CHANGED, self) 40 | end 41 | 42 | # Records a Test::Unit::Error. 43 | def add_error(error) 44 | @errors << error 45 | notify_listeners(FAULT, error) 46 | notify_listeners(CHANGED, self) 47 | end 48 | 49 | # Records an individual assertion. 50 | def add_assertion 51 | @assertion_count += 1 52 | notify_listeners(CHANGED, self) 53 | end 54 | 55 | # Returns a string contain the recorded runs, assertions, 56 | # failures and errors in this TestResult. 57 | def to_s 58 | "#{run_count} tests, #{assertion_count} assertions, #{failure_count} failures, #{error_count} errors" 59 | end 60 | 61 | # Returns whether or not this TestResult represents 62 | # successful completion. 63 | def passed? 64 | return @failures.empty? && @errors.empty? 65 | end 66 | 67 | # Returns the number of failures this TestResult has 68 | # recorded. 69 | def failure_count 70 | return @failures.size 71 | end 72 | 73 | # Returns the number of errors this TestResult has 74 | # recorded. 75 | def error_count 76 | return @errors.size 77 | end 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /lib/coderay/encoders/statistic.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # Makes a statistic for the given tokens. 5 | # 6 | # Alias: +stats+ 7 | class Statistic < Encoder 8 | 9 | register_for :statistic 10 | 11 | attr_reader :type_stats, :real_token_count # :nodoc: 12 | 13 | TypeStats = Struct.new :count, :size # :nodoc: 14 | 15 | protected 16 | 17 | def setup options 18 | super 19 | 20 | @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 } 21 | @real_token_count = 0 22 | end 23 | 24 | STATS = <<-STATS # :nodoc: 25 | 26 | Code Statistics 27 | 28 | Tokens %8d 29 | Non-Whitespace %8d 30 | Bytes Total %8d 31 | 32 | Token Types (%d): 33 | type count ratio size (average) 34 | ------------------------------------------------------------- 35 | %s 36 | STATS 37 | 38 | TOKEN_TYPES_ROW = <<-TKR # :nodoc: 39 | %-20s %8d %6.2f %% %5.1f 40 | TKR 41 | 42 | def finish options 43 | all = @type_stats['TOTAL'] 44 | all_count, all_size = all.count, all.size 45 | @type_stats.each do |type, stat| 46 | stat.size /= stat.count.to_f 47 | end 48 | types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v| 49 | TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size] 50 | end.join 51 | @out << STATS % [ 52 | all_count, @real_token_count, all_size, 53 | @type_stats.delete_if { |k, v| k.is_a? String }.size, 54 | types_stats 55 | ] 56 | 57 | super 58 | end 59 | 60 | public 61 | 62 | def text_token text, kind 63 | @real_token_count += 1 unless kind == :space 64 | @type_stats[kind].count += 1 65 | @type_stats[kind].size += text.size 66 | @type_stats['TOTAL'].size += text.size 67 | @type_stats['TOTAL'].count += 1 68 | end 69 | 70 | def begin_group kind 71 | block_token ':begin_group', kind 72 | end 73 | 74 | def end_group kind 75 | block_token ':end_group', kind 76 | end 77 | 78 | def begin_line kind 79 | block_token ':begin_line', kind 80 | end 81 | 82 | def end_line kind 83 | block_token ':end_line', kind 84 | end 85 | 86 | def block_token action, kind 87 | @type_stats['TOTAL'].count += 1 88 | @type_stats[action].count += 1 89 | @type_stats[kind].count += 1 90 | end 91 | 92 | end 93 | 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /lib/coderay/duo.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # = Duo 4 | # 5 | # A Duo is a convenient way to use CodeRay. You just create a Duo, 6 | # giving it a lang (language of the input code) and a format (desired 7 | # output format), and call Duo#highlight with the code. 8 | # 9 | # Duo makes it easy to re-use both scanner and encoder for a repetitive 10 | # task. It also provides a very easy interface syntax: 11 | # 12 | # require 'coderay' 13 | # CodeRay::Duo[:python, :div].highlight 'import this' 14 | # 15 | # Until you want to do uncommon things with CodeRay, I recommend to use 16 | # this method, since it takes care of everything. 17 | class Duo 18 | 19 | attr_accessor :lang, :format, :options 20 | 21 | # Create a new Duo, holding a lang and a format to highlight code. 22 | # 23 | # simple: 24 | # CodeRay::Duo[:ruby, :html].highlight 'bla 42' 25 | # 26 | # with options: 27 | # CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??' 28 | # 29 | # alternative syntax without options: 30 | # CodeRay::Duo[:ruby => :statistic].encode 'class << self; end' 31 | # 32 | # alternative syntax with options: 33 | # CodeRay::Duo[{ :ruby => :statistic }, :do => :something].encode 'abc' 34 | # 35 | # The options are forwarded to scanner and encoder 36 | # (see CodeRay.get_scanner_options). 37 | def initialize lang = nil, format = nil, options = {} 38 | if format.nil? && lang.is_a?(Hash) && lang.size == 1 39 | @lang = lang.keys.first 40 | @format = lang[@lang] 41 | else 42 | @lang = lang 43 | @format = format 44 | end 45 | @options = options 46 | end 47 | 48 | class << self 49 | # To allow calls like Duo[:ruby, :html].highlight. 50 | alias [] new 51 | end 52 | 53 | # The scanner of the duo. Only created once. 54 | def scanner 55 | @scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options) 56 | end 57 | 58 | # The encoder of the duo. Only created once. 59 | def encoder 60 | @encoder ||= CodeRay.encoder @format, @options 61 | end 62 | 63 | # Tokenize and highlight the code using +scanner+ and +encoder+. 64 | def encode code, options = {} 65 | options = @options.merge options 66 | encoder.encode(code, @lang, options) 67 | end 68 | alias highlight encode 69 | 70 | # Allows to use Duo like a proc object: 71 | # 72 | # CodeRay::Duo[:python => :yaml].call(code) 73 | # 74 | # or, in Ruby 1.9 and later: 75 | # 76 | # CodeRay::Duo[:python => :yaml].(code) 77 | alias call encode 78 | 79 | end 80 | 81 | end 82 | -------------------------------------------------------------------------------- /rake_tasks/test.rake: -------------------------------------------------------------------------------- 1 | namespace :test do 2 | desc 'run functional tests' 3 | task :functional do 4 | ruby './test/functional/suite.rb' 5 | ruby './test/functional/for_redcloth.rb' unless (''.chop! rescue true) 6 | end 7 | 8 | desc 'run unit tests' 9 | task :units do 10 | ruby './test/unit/suite.rb' 11 | end 12 | 13 | scanner_suite = 'test/scanners/suite.rb' 14 | desc 'run all scanner tests' 15 | task :scanners => :update_scanner_suite do 16 | ruby scanner_suite 17 | end 18 | 19 | desc 'update scanner test suite from GitHub' 20 | task :update_scanner_suite do 21 | if File.exist? scanner_suite 22 | Dir.chdir File.dirname(scanner_suite) do 23 | if File.directory? '.git' 24 | puts 'Updating scanner test suite...' 25 | sh 'git pull' 26 | elsif File.directory? '.svn' 27 | raise <<-ERROR 28 | Found the deprecated Subversion scanner test suite in ./#{File.dirname(scanner_suite)}. 29 | Please rename or remove it and run again to use the GitHub repository: 30 | 31 | mv test/scanners test/scanners-old 32 | ERROR 33 | else 34 | raise 'No scanner test suite found.' 35 | end 36 | end 37 | else 38 | puts 'Downloading scanner test suite...' 39 | sh 'git clone https://github.com/rubychan/coderay-scanner-tests.git test/scanners/' 40 | end unless ENV['SKIP_UPDATE_SCANNER_SUITE'] 41 | end 42 | 43 | namespace :scanner do 44 | Dir['./test/scanners/*'].each do |scanner| 45 | next unless File.directory? scanner 46 | lang = File.basename(scanner) 47 | desc "run all scanner tests for #{lang}" 48 | task lang => :update_scanner_suite do 49 | ruby "./test/scanners/suite.rb #{lang}" 50 | end 51 | end 52 | end 53 | 54 | desc 'clean test output files' 55 | task :clean do 56 | for file in Dir['test/scanners/**/*.actual.*'] 57 | rm file 58 | end 59 | for file in Dir['test/scanners/**/*.debug.diff'] 60 | rm file 61 | end 62 | for file in Dir['test/scanners/**/*.debug.diff.html'] 63 | rm file 64 | end 65 | for file in Dir['test/scanners/**/*.expected.html'] 66 | rm file 67 | end 68 | end 69 | 70 | desc 'test the CodeRay executable' 71 | task :exe do 72 | if RUBY_VERSION >= '1.8.7' 73 | ruby './test/executable/suite.rb' 74 | else 75 | puts 76 | puts "Can't run executable tests because shoulda-context requires Ruby 1.8.7+." 77 | puts "Skipping." 78 | end 79 | end 80 | end 81 | 82 | if RUBY_VERSION >= '1.9' 83 | require 'rspec/core/rake_task' 84 | RSpec::Core::RakeTask.new(:spec) 85 | end 86 | 87 | task :test => %w(test:functional test:units test:exe spec) 88 | -------------------------------------------------------------------------------- /lib/coderay/scanners/ruby/string_state.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | module CodeRay 3 | module Scanners 4 | 5 | class Ruby 6 | 7 | class StringState < Struct.new :type, :interpreted, :delim, :heredoc, 8 | :opening_paren, :paren_depth, :pattern, :next_state # :nodoc: all 9 | 10 | CLOSING_PAREN = Hash[ *%w[ 11 | ( ) 12 | [ ] 13 | < > 14 | { } 15 | ] ].each { |k,v| k.freeze; v.freeze } # debug, if I try to change it with << 16 | 17 | STRING_PATTERN = Hash.new do |h, k| 18 | delim, interpreted = *k 19 | delim_pattern = Regexp.escape(delim) 20 | if closing_paren = CLOSING_PAREN[delim] 21 | delim_pattern << Regexp.escape(closing_paren) 22 | end 23 | delim_pattern << '\\\\' unless delim == '\\' 24 | 25 | # special_escapes = 26 | # case interpreted 27 | # when :regexp_symbols 28 | # '| [|?*+(){}\[\].^$]' 29 | # end 30 | 31 | if interpreted && delim != '#' 32 | / (?= [#{delim_pattern}] | \# [{$@] ) /mx 33 | else 34 | / (?= [#{delim_pattern}] ) /mx 35 | end.tap do |pattern| 36 | h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256 37 | end 38 | end 39 | 40 | def self.simple_key_pattern delim 41 | if delim == "'" 42 | / (?> (?: [^\\']+ | \\. )* ) ' : /mx 43 | else 44 | / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#\{[^\{\}]+\} | \#(?!\{) )* ) " : /mx 45 | end 46 | end 47 | 48 | def initialize kind, interpreted, delim, heredoc = false 49 | if heredoc 50 | pattern = heredoc_pattern delim, interpreted, heredoc == :indented 51 | delim = nil 52 | else 53 | pattern = STRING_PATTERN[ [delim, interpreted] ] 54 | if closing_paren = CLOSING_PAREN[delim] 55 | opening_paren = delim 56 | delim = closing_paren 57 | paren_depth = 1 58 | end 59 | end 60 | super kind, interpreted, delim, heredoc, opening_paren, paren_depth, pattern, :initial 61 | end 62 | 63 | def heredoc_pattern delim, interpreted, indented 64 | # delim = delim.dup # workaround for old Ruby 65 | delim_pattern = Regexp.escape(delim) 66 | delim_pattern = / (?:\A|\n) #{ '(?>[ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x 67 | if interpreted 68 | / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx # $1 set == end of heredoc 69 | else 70 | / (?= #{delim_pattern}() | \\ ) /mx 71 | end 72 | end 73 | 74 | end 75 | 76 | end 77 | 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/coderay/encoders/token_kind_filter.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | load :filter 5 | 6 | # A Filter that selects tokens based on their token kind. 7 | # 8 | # == Options 9 | # 10 | # === :exclude 11 | # 12 | # One or many symbols (in an Array) which shall be excluded. 13 | # 14 | # Default: [] 15 | # 16 | # === :include 17 | # 18 | # One or many symbols (in an array) which shall be included. 19 | # 20 | # Default: :all, which means all tokens are included. 21 | # 22 | # Exclusion wins over inclusion. 23 | # 24 | # See also: CommentFilter 25 | class TokenKindFilter < Filter 26 | 27 | register_for :token_kind_filter 28 | 29 | DEFAULT_OPTIONS = { 30 | :exclude => [], 31 | :include => :all 32 | } 33 | 34 | protected 35 | def setup options 36 | super 37 | 38 | @group_excluded = false 39 | @exclude = options[:exclude] 40 | @exclude = Array(@exclude) unless @exclude == :all 41 | @include = options[:include] 42 | @include = Array(@include) unless @include == :all 43 | end 44 | 45 | def include_text_token? text, kind 46 | include_group? kind 47 | end 48 | 49 | def include_group? kind 50 | (@include == :all || @include.include?(kind)) && 51 | !(@exclude == :all || @exclude.include?(kind)) 52 | end 53 | 54 | public 55 | 56 | # Add the token to the output stream if +kind+ matches the conditions. 57 | def text_token text, kind 58 | super if !@group_excluded && include_text_token?(text, kind) 59 | end 60 | 61 | # Add the token group to the output stream if +kind+ matches the 62 | # conditions. 63 | # 64 | # If it does not, all tokens inside the group are excluded from the 65 | # stream, even if their kinds match. 66 | def begin_group kind 67 | if @group_excluded 68 | @group_excluded += 1 69 | elsif include_group? kind 70 | super 71 | else 72 | @group_excluded = 1 73 | end 74 | end 75 | 76 | # See +begin_group+. 77 | def begin_line kind 78 | if @group_excluded 79 | @group_excluded += 1 80 | elsif include_group? kind 81 | super 82 | else 83 | @group_excluded = 1 84 | end 85 | end 86 | 87 | # Take care of re-enabling the delegation of tokens to the output stream 88 | # if an exluded group has ended. 89 | def end_group kind 90 | if @group_excluded 91 | @group_excluded -= 1 92 | @group_excluded = false if @group_excluded.zero? 93 | else 94 | super 95 | end 96 | end 97 | 98 | # See +end_group+. 99 | def end_line kind 100 | if @group_excluded 101 | @group_excluded -= 1 102 | @group_excluded = false if @group_excluded.zero? 103 | else 104 | super 105 | end 106 | end 107 | 108 | end 109 | 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /lib/coderay/scanners/json.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | # Scanner for JSON (JavaScript Object Notation). 5 | class JSON < Scanner 6 | 7 | register_for :json 8 | file_extension 'json' 9 | 10 | KINDS_NOT_LOC = [ 11 | :float, :char, :content, :delimiter, 12 | :error, :integer, :operator, :value, 13 | ] # :nodoc: 14 | 15 | ESCAPE = / [bfnrt\\"\/] /x # :nodoc: 16 | UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: 17 | KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /x 18 | 19 | protected 20 | 21 | def setup 22 | @state = :initial 23 | end 24 | 25 | # See http://json.org/ for a definition of the JSON lexic/grammar. 26 | def scan_tokens encoder, options 27 | state = options[:state] || @state 28 | 29 | if [:string, :key].include? state 30 | encoder.begin_group state 31 | end 32 | 33 | until eos? 34 | 35 | case state 36 | 37 | when :initial 38 | if match = scan(/ \s+ /x) 39 | encoder.text_token match, :space 40 | elsif match = scan(/"/) 41 | state = check(/#{KEY}/o) ? :key : :string 42 | encoder.begin_group state 43 | encoder.text_token match, :delimiter 44 | elsif match = scan(/ [:,\[{\]}] /x) 45 | encoder.text_token match, :operator 46 | elsif match = scan(/ true | false | null /x) 47 | encoder.text_token match, :value 48 | elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x) 49 | if scan(/ \.\d+ (?:[eE][-+]?\d+)? | [eE][-+]? \d+ /x) 50 | match << matched 51 | encoder.text_token match, :float 52 | else 53 | encoder.text_token match, :integer 54 | end 55 | else 56 | encoder.text_token getch, :error 57 | end 58 | 59 | when :string, :key 60 | if match = scan(/[^\\"]+/) 61 | encoder.text_token match, :content 62 | elsif match = scan(/"/) 63 | encoder.text_token match, :delimiter 64 | encoder.end_group state 65 | state = :initial 66 | elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) 67 | encoder.text_token match, :char 68 | elsif match = scan(/\\./m) 69 | encoder.text_token match, :content 70 | elsif match = scan(/ \\ | $ /x) 71 | encoder.end_group state 72 | encoder.text_token match, :error unless match.empty? 73 | state = :initial 74 | else 75 | raise_inspect "else case \" reached; %p not handled." % peek(1), encoder 76 | end 77 | 78 | else 79 | raise_inspect 'Unknown state: %p' % [state], encoder 80 | 81 | end 82 | end 83 | 84 | if options[:keep_state] 85 | @state = state 86 | end 87 | 88 | if [:string, :key].include? state 89 | encoder.end_group state 90 | end 91 | 92 | encoder 93 | end 94 | 95 | end 96 | 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /rake_tasks/generator.rake: -------------------------------------------------------------------------------- 1 | namespace :generate do 2 | desc 'generates a new scanner NAME=lang [ALT=alternative,plugin,ids] [EXT=file,extensions] [BASE=base lang]' 3 | task :scanner do 4 | raise 'I need a scanner name; use NAME=lang' unless scanner_class_name = ENV['NAME'] 5 | raise "Invalid lang: #{scanner_class_name}; use NAME=lang." unless /\A\w+\z/ === scanner_class_name 6 | require 'active_support/all' 7 | lang = scanner_class_name.underscore 8 | class_name = scanner_class_name.camelize 9 | 10 | def scanner_file_for_lang lang 11 | File.join(LIB_ROOT, 'coderay', 'scanners', lang + '.rb') 12 | end 13 | 14 | scanner_file = scanner_file_for_lang lang 15 | if File.exist? scanner_file 16 | print "#{scanner_file} already exists. Overwrite? [y|N] " 17 | exit unless $stdin.gets.chomp.downcase == 'y' 18 | end 19 | 20 | base_lang = ENV.fetch('BASE', 'json') 21 | base_scanner_file = scanner_file_for_lang(base_lang) 22 | puts "Reading base scanner #{base_scanner_file}..." 23 | base_scanner = File.read base_scanner_file 24 | puts "Writing new scanner #{scanner_file}..." 25 | File.open(scanner_file, 'w') do |file| 26 | file.write base_scanner. 27 | sub(/class \w+ < Scanner/, "class #{class_name} < Scanner"). 28 | sub('# Scanner for JSON (JavaScript Object Notation).', "# A scanner for #{scanner_class_name}."). 29 | sub(/register_for :\w+/, "register_for :#{lang}"). 30 | sub(/file_extension '\S+'/, "file_extension '#{ENV.fetch('EXT', lang).split(',').first}'") 31 | end 32 | 33 | test_dir = File.join(ROOT, 'test', 'scanners', lang) 34 | unless File.exist? test_dir 35 | puts "Creating test folder #{test_dir}..." 36 | sh "mkdir #{test_dir}" 37 | end 38 | test_suite_file = File.join(test_dir, 'suite.rb') 39 | unless File.exist? test_suite_file 40 | puts "Creating test suite file #{test_suite_file}..." 41 | base_suite = File.read File.join(test_dir, '..', 'ruby', 'suite.rb') 42 | File.open(test_suite_file, 'w') do |file| 43 | file.write base_suite.sub(/class Ruby/, "class #{class_name}") 44 | end 45 | end 46 | 47 | if extensions = ENV['EXT'] 48 | file_type_file = File.join(LIB_ROOT, 'coderay', 'helpers', 'filetype.rb') 49 | puts "Not automated. Remember to add your extensions to #{file_type_file}:" 50 | for ext in extensions.split(',') 51 | puts " '#{ext}' => :#{lang}," 52 | end 53 | end 54 | 55 | if alternative_ids = ENV['ALT'] && alternative_ids != lang 56 | map_file = File.join(LIB_ROOT, 'coderay', 'scanners', '_map.rb') 57 | puts "Not automated. Remember to add your alternative plugin ids to #{map_file}:" 58 | for id in alternative_ids.split(',') 59 | puts " :#{id} => :#{lang}," 60 | end 61 | end 62 | 63 | print 'Add to git? [Y|n] ' 64 | answer = $stdin.gets.chomp.downcase 65 | if answer.empty? || answer == 'y' 66 | sh "git add #{scanner_file}" 67 | cd File.join('test', 'scanners') do 68 | sh "git add #{lang}" 69 | end 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /test/lib/test/unit/util/observable.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2002 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | require 'test/unit/util/procwrapper' 8 | 9 | module Test 10 | module Unit 11 | module Util 12 | 13 | # This is a utility class that allows anything mixing 14 | # it in to notify a set of listeners about interesting 15 | # events. 16 | module Observable 17 | # We use this for defaults since nil might mean something 18 | NOTHING = "NOTHING/#{__id__}" 19 | 20 | # Adds the passed proc as a listener on the 21 | # channel indicated by channel_name. listener_key 22 | # is used to remove the listener later; if none is 23 | # specified, the proc itself is used. 24 | # 25 | # Whatever is used as the listener_key is 26 | # returned, making it very easy to use the proc 27 | # itself as the listener_key: 28 | # 29 | # listener = add_listener("Channel") { ... } 30 | # remove_listener("Channel", listener) 31 | def add_listener(channel_name, listener_key=NOTHING, &listener) # :yields: value 32 | unless(block_given?) 33 | raise ArgumentError.new("No callback was passed as a listener") 34 | end 35 | 36 | key = listener_key 37 | if (listener_key == NOTHING) 38 | listener_key = listener 39 | key = ProcWrapper.new(listener) 40 | end 41 | 42 | channels[channel_name] ||= {} 43 | channels[channel_name][key] = listener 44 | return listener_key 45 | end 46 | 47 | # Removes the listener indicated by listener_key 48 | # from the channel indicated by 49 | # channel_name. Returns the registered proc, or 50 | # nil if none was found. 51 | def remove_listener(channel_name, listener_key) 52 | channel = channels[channel_name] 53 | return nil unless (channel) 54 | key = listener_key 55 | if (listener_key.instance_of?(Proc)) 56 | key = ProcWrapper.new(listener_key) 57 | end 58 | if (channel.has_key?(key)) 59 | return channel.delete(key) 60 | end 61 | return nil 62 | end 63 | 64 | # Calls all the procs registered on the channel 65 | # indicated by channel_name. If value is 66 | # specified, it is passed in to the procs, 67 | # otherwise they are called with no arguments. 68 | # 69 | #-- 70 | # 71 | # Perhaps this should be private? Would it ever 72 | # make sense for an external class to call this 73 | # method directly? 74 | def notify_listeners(channel_name, *arguments) 75 | channel = channels[channel_name] 76 | return 0 unless (channel) 77 | listeners = channel.values 78 | listeners.each { |listener| listener.call(*arguments) } 79 | return listeners.size 80 | end 81 | 82 | private 83 | def channels 84 | @channels ||= {} 85 | return @channels 86 | end 87 | end 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/coderay/for_redcloth.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # A little hack to enable CodeRay highlighting in RedCloth. 4 | # 5 | # Usage: 6 | # require 'coderay' 7 | # require 'coderay/for_redcloth' 8 | # RedCloth.new('@[ruby]puts "Hello, World!"@').to_html 9 | # 10 | # Make sure you have RedCloth 4.0.3 activated, for example by calling 11 | # require 'rubygems' 12 | # before RedCloth is loaded and before calling CodeRay.for_redcloth. 13 | module ForRedCloth 14 | 15 | def self.install 16 | gem 'RedCloth', '>= 4.0.3' if defined? gem 17 | require 'redcloth' 18 | unless RedCloth::VERSION.to_s >= '4.0.3' 19 | if defined? gem 20 | raise 'CodeRay.for_redcloth needs RedCloth version 4.0.3 or later. ' + 21 | "You have #{RedCloth::VERSION}. Please gem install RedCloth." 22 | else 23 | $".delete 'redcloth.rb' # sorry, but it works 24 | require 'rubygems' 25 | return install # retry 26 | end 27 | end 28 | unless RedCloth::VERSION.to_s >= '4.2.2' 29 | warn 'CodeRay.for_redcloth works best with RedCloth version 4.2.2 or later.' 30 | end 31 | RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc 32 | RedCloth::Formatters::HTML.module_eval do 33 | def unescape(html) # :nodoc: 34 | replacements = { 35 | '&' => '&', 36 | '"' => '"', 37 | '>' => '>', 38 | '<' => '<', 39 | } 40 | html.gsub(/&(?:amp|quot|[gl]t);/) { |entity| replacements[entity] } 41 | end 42 | undef code, bc_open, bc_close, escape_pre 43 | def code(opts) # :nodoc: 44 | opts[:block] = true 45 | if !opts[:lang] && RedCloth::VERSION.to_s >= '4.2.0' 46 | # simulating pre-4.2 behavior 47 | if opts[:text].sub!(/\A\[(\w+)\]/, '') 48 | if CodeRay::Scanners[$1].lang == :text 49 | opts[:text] = $& + opts[:text] 50 | else 51 | opts[:lang] = $1 52 | end 53 | end 54 | end 55 | if opts[:lang] && !filter_coderay 56 | require 'coderay' 57 | @in_bc ||= nil 58 | format = @in_bc ? :div : :span 59 | opts[:text] = unescape(opts[:text]) unless @in_bc 60 | highlighted_code = CodeRay.encode opts[:text], opts[:lang], format 61 | highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) } 62 | highlighted_code 63 | else 64 | "#{opts[:text]}" 65 | end 66 | end 67 | def bc_open(opts) # :nodoc: 68 | opts[:block] = true 69 | @in_bc = opts 70 | opts[:lang] ? '' : "" 71 | end 72 | def bc_close(opts) # :nodoc: 73 | opts = @in_bc 74 | @in_bc = nil 75 | opts[:lang] ? '' : "\n" 76 | end 77 | def escape_pre(text) # :nodoc: 78 | if @in_bc ||= nil 79 | text 80 | else 81 | html_esc(text, :html_escape_preformatted) 82 | end 83 | end 84 | end 85 | end 86 | 87 | module TextileDoc # :nodoc: 88 | attr_accessor :filter_coderay 89 | end 90 | 91 | end 92 | 93 | end 94 | 95 | CodeRay::ForRedCloth.install -------------------------------------------------------------------------------- /test/functional/for_redcloth.rb: -------------------------------------------------------------------------------- 1 | require 'simplecov' if RUBY_VERSION >= '1.9' 2 | require 'test/unit' 3 | 4 | $:.unshift File.expand_path('../../../lib', __FILE__) 5 | require 'coderay' 6 | 7 | begin 8 | require 'rubygems' unless defined? Gem 9 | gem 'RedCloth', '>= 4.0.3' rescue nil 10 | require 'redcloth' 11 | rescue LoadError 12 | warn 'RedCloth not found - skipping for_redcloth tests.' 13 | undef RedCloth if defined? RedCloth 14 | end 15 | 16 | class BasicTest < Test::Unit::TestCase 17 | 18 | def test_for_redcloth 19 | require 'coderay/for_redcloth' 20 | assert_equal "

puts "Hello, World!"

", 21 | RedCloth.new('@[ruby]puts "Hello, World!"@').to_html 22 | assert_equal <<-BLOCKCODE.chomp, 23 |
24 | 
puts "Hello, World!"
25 |
26 | BLOCKCODE 27 | RedCloth.new('bc[ruby]. puts "Hello, World!"').to_html 28 | end 29 | 30 | def test_for_redcloth_no_lang 31 | require 'coderay/for_redcloth' 32 | assert_equal "

puts \"Hello, World!\"

", 33 | RedCloth.new('@puts "Hello, World!"@').to_html 34 | assert_equal <<-BLOCKCODE.chomp, 35 | 
puts \"Hello, World!\"
36 | BLOCKCODE 37 | RedCloth.new('bc. puts "Hello, World!"').to_html 38 | end 39 | 40 | def test_for_redcloth_style 41 | require 'coderay/for_redcloth' 42 | assert_equal <<-BLOCKCODE.chomp, 43 | 
puts \"Hello, World!\"
44 | BLOCKCODE 45 | RedCloth.new('bc{color: red}. puts "Hello, World!"').to_html 46 | end 47 | 48 | def test_for_redcloth_escapes 49 | require 'coderay/for_redcloth' 50 | assert_equal '

>

', 51 | RedCloth.new('@[ruby]>@').to_html 52 | assert_equal <<-BLOCKCODE.chomp, 53 |
54 | 
&
55 |
56 | BLOCKCODE 57 | RedCloth.new('bc[ruby]. &').to_html 58 | end 59 | 60 | def test_for_redcloth_escapes2 61 | require 'coderay/for_redcloth' 62 | assert_equal "

#include <test.h>

", 63 | RedCloth.new('@[c]#include @').to_html 64 | end 65 | 66 | # See http://jgarber.lighthouseapp.com/projects/13054/tickets/124-code-markup-does-not-allow-brackets. 67 | def test_for_redcloth_false_positive 68 | require 'coderay/for_redcloth' 69 | assert_equal '

[project]_dff.skjd

', 70 | RedCloth.new('@[project]_dff.skjd@').to_html 71 | # false positive, but expected behavior / known issue 72 | assert_equal "

_dff.skjd

", 73 | RedCloth.new('@[ruby]_dff.skjd@').to_html 74 | assert_equal <<-BLOCKCODE.chomp, RedCloth.new('bc. [project]_dff.skjd').to_html 75 | 
[project]_dff.skjd
76 | BLOCKCODE 77 | end 78 | 79 | end if defined? RedCloth -------------------------------------------------------------------------------- /test/lib/test/unit/collector/dir.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit/testsuite' 2 | require 'test/unit/collector' 3 | 4 | module Test 5 | module Unit 6 | module Collector 7 | class Dir 8 | include Collector 9 | 10 | attr_reader :pattern, :exclude 11 | attr_accessor :base 12 | 13 | def initialize(dir=::Dir, file=::File, object_space=::ObjectSpace, req=nil) 14 | super() 15 | @dir = dir 16 | @file = file 17 | @object_space = object_space 18 | @req = req 19 | @pattern = [/\btest_.*\.rb\Z/m] 20 | @exclude = [] 21 | end 22 | 23 | def collect(*from) 24 | basedir = @base 25 | $:.push(basedir) if basedir 26 | if(from.empty?) 27 | recursive_collect('.', find_test_cases) 28 | elsif(from.size == 1) 29 | recursive_collect(from.first, find_test_cases) 30 | else 31 | suites = [] 32 | from.each do |f| 33 | suite = recursive_collect(f, find_test_cases) 34 | suites << suite unless(suite.tests.empty?) 35 | end 36 | suite = TestSuite.new("[#{from.join(', ')}]") 37 | sort(suites).each{|s| suite << s} 38 | suite 39 | end 40 | ensure 41 | $:.delete_at($:.rindex(basedir)) if basedir 42 | end 43 | 44 | def find_test_cases(ignore=[]) 45 | cases = [] 46 | @object_space.each_object(Class) do |c| 47 | cases << c if(c < TestCase && !ignore.include?(c)) 48 | end 49 | ignore.concat(cases) 50 | cases 51 | end 52 | 53 | def recursive_collect(name, already_gathered) 54 | sub_suites = [] 55 | path = realdir(name) 56 | if @file.directory?(path) 57 | dir_name = name unless name == '.' 58 | @dir.entries(path).each do |e| 59 | next if(e == '.' || e == '..') 60 | e_name = dir_name ? @file.join(dir_name, e) : e 61 | if @file.directory?(realdir(e_name)) 62 | next if /\ACVS\z/ =~ e 63 | sub_suite = recursive_collect(e_name, already_gathered) 64 | sub_suites << sub_suite unless(sub_suite.empty?) 65 | else 66 | next if /~\z/ =~ e_name or /\A\.\#/ =~ e 67 | if @pattern and !@pattern.empty? 68 | next unless @pattern.any? {|pat| pat =~ e_name} 69 | end 70 | if @exclude and !@exclude.empty? 71 | next if @exclude.any? {|pat| pat =~ e_name} 72 | end 73 | collect_file(e_name, sub_suites, already_gathered) 74 | end 75 | end 76 | else 77 | collect_file(name, sub_suites, already_gathered) 78 | end 79 | suite = TestSuite.new(@file.basename(name)) 80 | sort(sub_suites).each{|s| suite << s} 81 | suite 82 | end 83 | 84 | def collect_file(name, suites, already_gathered) 85 | dir = @file.dirname(@file.expand_path(name, @base)) 86 | $:.unshift(dir) 87 | if(@req) 88 | @req.require(name) 89 | else 90 | require(name) 91 | end 92 | find_test_cases(already_gathered).each{|t| add_suite(suites, t.suite)} 93 | ensure 94 | $:.delete_at($:.rindex(dir)) if(dir) 95 | end 96 | 97 | def realdir(path) 98 | if @base 99 | @file.join(@base, path) 100 | else 101 | path 102 | end 103 | end 104 | end 105 | end 106 | end 107 | end 108 | -------------------------------------------------------------------------------- /CREDITS.textile: -------------------------------------------------------------------------------- 1 | h1. Credits 2 | 3 | h3. Special Thanks to 4 | 5 | * licenser (Heinz N. Gies) for ending my QBasic career, inventing the Coder project and the input/output plugin system. CodeRay would not exist without him. 6 | * bovi (Daniel Bovensiepen) for helping me out on various occasions. 7 | 8 | h3. Thanks to 9 | 10 | * Caleb Clausen for writing "RubyLexer":http://rubyforge.org/projects/rubylexer and lots of very interesting mail traffic 11 | * birkenfeld (Georg Brandl) and mitsuhiku (Arnim Ronacher) for PyKleur, now Pygments. You guys rock! 12 | * Jamis Buck for writing "Syntax":http://rubyforge.org/projects/syntax — I got some useful ideas from it. 13 | * Doug Kearns and everyone else who worked on ruby.vim - it not only helped me coding CodeRay, but also gave me a wonderful target to reach for the Ruby scanner. 14 | * everyone who uses CodeBB on "http://www.rubyforen.de":http://www.rubyforen.de and "http://www.python-forum.de":http://www.python-forum.de 15 | * iGEL, magichisoka, manveru, WoNáDo and everyone I forgot from rubyforen.de 16 | * Dethix from ruby-mine.de 17 | * zickzackw 18 | * Dookie (who is no longer with us...) and Leonidas from "http://www.python-forum.de":http://www.python-forum.de 19 | * Andreas Schwarz for finding out that CaseIgnoringWordList was not case ignoring! Such things really make you write tests. 20 | * closure for the first version of the Scheme scanner. 21 | * Stefan Walk for the first version of the JavaScript and PHP scanners. 22 | * Josh Goebel for another version of the JavaScript scanner, a SQL and a Diff scanner. 23 | * Jonathan Younger for pointing out the licence confusion caused by wrong LICENSE file. 24 | * Jeremy Hinegardner for finding the shebang-on-empty-file bug in FileType. 25 | * Charles Oliver Nutter and Yehuda Katz for helping me benchmark CodeRay on JRuby. 26 | * Andreas Neuhaus for pointing out a markup bug in coderay/for_redcloth. 27 | * 0xf30fc7 for the FileType patch concerning Delphi file extensions. 28 | * The folks at redmine.org - thank you for using and fixing CodeRay! 29 | * Keith Pitt for his SQL scanners 30 | * Rob Aldred for the terminal encoder 31 | * Trans for pointing out $DEBUG dependencies 32 | * Flameeyes for finding that Term::ANSIColor was obsolete 33 | * matz and all Ruby gods and gurus 34 | * The inventors of: the computer, the internet, the true color display, HTML & CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime, manga, coke and green ice tea. 35 | 36 | Where would we be without all those people? 37 | 38 | h3. Created using 39 | 40 | * "Ruby":http://ruby-lang.org/ 41 | * Chihiro (my Sony VAIO laptop); Henrietta (my old MacBook); Triella, born Rico (my new MacBook); as well as Seras and Hikari (my PCs) 42 | * "RDE":http://homepage2.nifty.com/sakazuki/rde_e.html, "VIM":http://vim.org and "TextMate":http://macromates.com 43 | * "Subversion":http://subversion.tigris.org/ 44 | * "Redmine":http://redmine.org/ 45 | * "Firefox":http://www.mozilla.org/products/firefox/, "Firebug":http://getfirebug.com/, "Safari":http://www.apple.com/safari/, and "Thunderbird":http://www.mozilla.org/products/thunderbird/ 46 | * "RubyGems":http://docs.rubygems.org/ and "Rake":http://rake.rubyforge.org/ 47 | * "TortoiseSVN":http://tortoisesvn.tigris.org/ using Apache via "XAMPP":http://www.apachefriends.org/en/xampp.html 48 | * RDoc (though I'm quite unsatisfied with it) 49 | * Microsoft Windows (yes, I confess!) and MacOS X 50 | * GNUWin32, MinGW and some other tools to make the shell under windows a bit less useless 51 | * Term::"ANSIColor":http://term-ansicolor.rubyforge.org/ 52 | * "PLEAC":http://pleac.sourceforge.net/ code examples 53 | * Github 54 | * "Travis CI":http://travis-ci.org/rubychan/github 55 | 56 | h3. Free 57 | 58 | * As you can see, CodeRay was created under heavy use of *free* software. 59 | * So CodeRay is also *free*. 60 | * If you use CodeRay to create software, think about making this software *free*, too. 61 | * Thanks :) 62 | -------------------------------------------------------------------------------- /test/unit/file_type.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require File.expand_path('../../lib/assert_warning', __FILE__) 3 | 4 | require 'coderay/helpers/file_type' 5 | 6 | class FileTypeTests < Test::Unit::TestCase 7 | 8 | include CodeRay 9 | 10 | def test_fetch 11 | assert_raise FileType::UnknownFileType do 12 | FileType.fetch '' 13 | end 14 | 15 | assert_throws :not_found do 16 | FileType.fetch '.' do 17 | throw :not_found 18 | end 19 | end 20 | 21 | assert_equal :default, FileType.fetch('c', :default) 22 | end 23 | 24 | def test_block_supersedes_default_warning 25 | assert_warning 'Block supersedes default value argument; use either.' do 26 | FileType.fetch('c', :default) { } 27 | end 28 | end 29 | 30 | def test_ruby 31 | assert_equal :ruby, FileType[__FILE__] 32 | assert_equal :ruby, FileType['test.rb'] 33 | assert_equal :ruby, FileType['test.java.rb'] 34 | assert_equal :java, FileType['test.rb.java'] 35 | assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw'] 36 | assert_equal :ruby, FileType['/usr/bin/something/Rakefile'] 37 | assert_equal :ruby, FileType['~/myapp/gem/Rantfile'] 38 | assert_equal :ruby, FileType['./lib/tasks\repository.rake'] 39 | assert_not_equal :ruby, FileType['test_rb'] 40 | assert_not_equal :ruby, FileType['Makefile'] 41 | assert_not_equal :ruby, FileType['set.rb/set'] 42 | assert_not_equal :ruby, FileType['~/projects/blabla/rb'] 43 | end 44 | 45 | def test_c 46 | assert_equal :c, FileType['test.c'] 47 | assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h'] 48 | assert_not_equal :c, FileType['test_c'] 49 | assert_not_equal :c, FileType['Makefile'] 50 | assert_not_equal :c, FileType['set.h/set'] 51 | assert_not_equal :c, FileType['~/projects/blabla/c'] 52 | end 53 | 54 | def test_cpp 55 | assert_equal :cpp, FileType['test.c++'] 56 | assert_equal :cpp, FileType['test.cxx'] 57 | assert_equal :cpp, FileType['test.hh'] 58 | assert_equal :cpp, FileType['test.hpp'] 59 | assert_equal :cpp, FileType['test.cu'] 60 | assert_equal :cpp, FileType['test.C'] 61 | assert_not_equal :cpp, FileType['test.c'] 62 | assert_not_equal :cpp, FileType['test.h'] 63 | end 64 | 65 | def test_html 66 | assert_equal :html, FileType['test.htm'] 67 | assert_equal :html, FileType['test.xhtml'] 68 | assert_equal :html, FileType['test.html.xhtml'] 69 | assert_equal :erb, FileType['_form.rhtml'] 70 | assert_equal :erb, FileType['_form.html.erb'] 71 | end 72 | 73 | def test_yaml 74 | assert_equal :yaml, FileType['test.yml'] 75 | assert_equal :yaml, FileType['test.yaml'] 76 | assert_equal :yaml, FileType['my.html.yaml'] 77 | assert_not_equal :yaml, FileType['YAML'] 78 | end 79 | 80 | def test_pathname 81 | require 'pathname' 82 | pn = Pathname.new 'test.rb' 83 | assert_equal :ruby, FileType[pn] 84 | dir = Pathname.new '/etc/var/blubb' 85 | assert_equal :ruby, FileType[dir + pn] 86 | assert_equal :cpp, FileType[dir + 'test.cpp'] 87 | end 88 | 89 | def test_no_shebang 90 | dir = './test' 91 | if File.directory? dir 92 | Dir.chdir dir do 93 | assert_equal :c, FileType['test.c'] 94 | end 95 | end 96 | end 97 | 98 | def test_shebang_empty_file 99 | require 'tmpdir' 100 | tmpfile = File.join(Dir.tmpdir, 'bla') 101 | File.open(tmpfile, 'w') { } # touch 102 | assert_equal nil, FileType[tmpfile, true] 103 | end 104 | 105 | def test_shebang_no_file 106 | assert_equal nil, FileType['i do not exist', true] 107 | end 108 | 109 | def test_shebang 110 | require 'tmpdir' 111 | tmpfile = File.join(Dir.tmpdir, 'bla') 112 | File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' } 113 | assert_equal :ruby, FileType[tmpfile, true] 114 | end 115 | 116 | end 117 | -------------------------------------------------------------------------------- /lib/coderay/encoders/html/numbering.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | class HTML 5 | 6 | module Numbering # :nodoc: 7 | 8 | def self.number! output, mode = :table, options = {} 9 | return self unless mode 10 | 11 | options = DEFAULT_OPTIONS.merge options 12 | 13 | start = options[:line_number_start] 14 | unless start.is_a? Integer 15 | raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start 16 | end 17 | 18 | anchor_prefix = options[:line_number_anchors] 19 | anchor_prefix = 'line' if anchor_prefix == true 20 | anchor_prefix = anchor_prefix.to_s[/[\w-]+/] if anchor_prefix 21 | anchoring = 22 | if anchor_prefix 23 | proc do |line| 24 | line = line.to_s 25 | anchor = anchor_prefix + line 26 | "#{line}" 27 | end 28 | else 29 | :to_s.to_proc 30 | end 31 | 32 | bold_every = options[:bold_every] 33 | highlight_lines = options[:highlight_lines] 34 | bolding = 35 | if bold_every == false && highlight_lines == nil 36 | anchoring 37 | elsif highlight_lines.is_a? Enumerable 38 | highlight_lines = highlight_lines.to_set 39 | proc do |line| 40 | if highlight_lines.include? line 41 | "#{anchoring[line]}" # highlighted line numbers in bold 42 | else 43 | anchoring[line] 44 | end 45 | end 46 | elsif bold_every.is_a? Integer 47 | raise ArgumentError, ":bolding can't be 0." if bold_every == 0 48 | proc do |line| 49 | if line % bold_every == 0 50 | "#{anchoring[line]}" # every bold_every-th number in bold 51 | else 52 | anchoring[line] 53 | end 54 | end 55 | else 56 | raise ArgumentError, 'Invalid value %p for :bolding; false or Integer expected.' % bold_every 57 | end 58 | 59 | if position_of_last_newline = output.rindex(RUBY_VERSION >= '1.9' ? /\n/ : ?\n) 60 | after_last_newline = output[position_of_last_newline + 1 .. -1] 61 | ends_with_newline = after_last_newline[/\A(?:<\/span>)*\z/] 62 | 63 | if ends_with_newline 64 | line_count = output.count("\n") 65 | else 66 | line_count = output.count("\n") + 1 67 | end 68 | else 69 | line_count = 1 70 | end 71 | 72 | case mode 73 | when :inline 74 | max_width = (start + line_count).to_s.size 75 | line_number = start 76 | output.gsub!(/^.*$\n?/) do |line| 77 | line_number_text = bolding.call line_number 78 | indent = ' ' * (max_width - line_number.to_s.size) 79 | line_number += 1 80 | "#{indent}#{line_number_text}#{line}" 81 | end 82 | 83 | when :table 84 | line_numbers = (start ... start + line_count).map(&bolding).join("\n") 85 | line_numbers << "\n" 86 | line_numbers_table_template = Output::TABLE.apply('LINE_NUMBERS', line_numbers) 87 | 88 | output.gsub!(/<\/div>\n/, '') 89 | output.wrap_in! line_numbers_table_template 90 | output.wrapped_in = :div 91 | 92 | when :list 93 | raise NotImplementedError, 'The :list option is no longer available. Use :table.' 94 | 95 | else 96 | raise ArgumentError, 'Unknown value %p for mode: expected one of %p' % 97 | [mode, [:table, :inline]] 98 | end 99 | 100 | output 101 | end 102 | 103 | end 104 | 105 | end 106 | 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /test/lib/test/unit/ui/console/testrunner.rb: -------------------------------------------------------------------------------- 1 | #-- 2 | # 3 | # Author:: Nathaniel Talbott. 4 | # Copyright:: Copyright (c) 2000-2003 Nathaniel Talbott. All rights reserved. 5 | # License:: Ruby license. 6 | 7 | require 'test/unit/ui/testrunnermediator' 8 | require 'test/unit/ui/testrunnerutilities' 9 | 10 | module Test 11 | module Unit 12 | module UI 13 | module Console 14 | 15 | # Runs a Test::Unit::TestSuite on the console. 16 | class TestRunner 17 | extend TestRunnerUtilities 18 | 19 | # Creates a new TestRunner for running the passed 20 | # suite. If quiet_mode is true, the output while 21 | # running is limited to progress dots, errors and 22 | # failures, and the final result. io specifies 23 | # where runner output should go to; defaults to 24 | # STDOUT. 25 | def initialize(suite, output_level=NORMAL, io=STDOUT) 26 | if (suite.respond_to?(:suite)) 27 | @suite = suite.suite 28 | else 29 | @suite = suite 30 | end 31 | @output_level = output_level 32 | @io = io 33 | @already_outputted = false 34 | @faults = [] 35 | end 36 | 37 | # Begins the test run. 38 | def start 39 | setup_mediator 40 | attach_to_mediator 41 | return start_mediator 42 | end 43 | 44 | private 45 | def setup_mediator 46 | @mediator = create_mediator(@suite) 47 | suite_name = @suite.to_s 48 | if ( @suite.kind_of?(Module) ) 49 | suite_name = @suite.name 50 | end 51 | output("Loaded suite #{suite_name}") 52 | end 53 | 54 | def create_mediator(suite) 55 | return TestRunnerMediator.new(suite) 56 | end 57 | 58 | def attach_to_mediator 59 | @mediator.add_listener(TestResult::FAULT, &method(:add_fault)) 60 | @mediator.add_listener(TestRunnerMediator::STARTED, &method(:started)) 61 | @mediator.add_listener(TestRunnerMediator::FINISHED, &method(:finished)) 62 | @mediator.add_listener(TestCase::STARTED, &method(:test_started)) 63 | @mediator.add_listener(TestCase::FINISHED, &method(:test_finished)) 64 | end 65 | 66 | def start_mediator 67 | return @mediator.run_suite 68 | end 69 | 70 | def add_fault(fault) 71 | @faults << fault 72 | output_single(fault.single_character_display, PROGRESS_ONLY) 73 | @already_outputted = true 74 | end 75 | 76 | def started(result) 77 | @result = result 78 | output("Started") 79 | end 80 | 81 | def finished(elapsed_time) 82 | nl 83 | output("Finished in #{elapsed_time} seconds.") 84 | @faults.each_with_index do |fault, index| 85 | nl 86 | output("%3d) %s" % [index + 1, fault.long_display]) 87 | end 88 | nl 89 | output(@result) 90 | end 91 | 92 | def test_started(name) 93 | output_single(name + ": ", VERBOSE) 94 | end 95 | 96 | def test_finished(name) 97 | output_single(".", PROGRESS_ONLY) unless (@already_outputted) 98 | nl(VERBOSE) 99 | @already_outputted = false 100 | end 101 | 102 | def nl(level=NORMAL) 103 | output("", level) 104 | end 105 | 106 | def output(something, level=NORMAL) 107 | @io.puts(something) if (output?(level)) 108 | @io.flush 109 | end 110 | 111 | def output_single(something, level=NORMAL) 112 | @io.write(something) if (output?(level)) 113 | @io.flush 114 | end 115 | 116 | def output?(level) 117 | level <= @output_level 118 | end 119 | end 120 | end 121 | end 122 | end 123 | end 124 | 125 | if __FILE__ == $0 126 | Test::Unit::UI::Console::TestRunner.start_command_line_test 127 | end 128 | -------------------------------------------------------------------------------- /test/unit/html.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'coderay' 3 | 4 | class HtmlTest < Test::Unit::TestCase 5 | 6 | def test_break_lines_option 7 | snippets = {} 8 | 9 | snippets[:ruby] = {} 10 | 11 | snippets[:ruby][:in] = <<-RUBY 12 | ruby_inside = <<-RUBY_INSIDE 13 | This is tricky, 14 | isn't it? 15 | RUBY_INSIDE 16 | RUBY 17 | 18 | snippets[:ruby][:expected_with_option_off] = <<-HTML_OPT_INDEPENDENT_LINES_OFF 19 | ruby_inside = <<-RUBY_INSIDE 20 | This is tricky, 21 | isn't it? 22 | RUBY_INSIDE 23 | HTML_OPT_INDEPENDENT_LINES_OFF 24 | 25 | snippets[:ruby][:expected_with_option_on] = <<-HTML_OPT_INDEPENDENT_LINES_ON 26 | ruby_inside = <<-RUBY_INSIDE 27 | This is tricky, 28 | isn't it? 29 | RUBY_INSIDE 30 | HTML_OPT_INDEPENDENT_LINES_ON 31 | 32 | snippets[:java] = {} 33 | 34 | snippets[:java][:in] = <<-JAVA 35 | import java.lang.*; 36 | 37 | /** 38 | * This is some multiline javadoc 39 | * used to test the 40 | */ 41 | public class Test { 42 | public static final String MESSAGE = "My message\ 43 | To the world"; 44 | 45 | static void main() { 46 | /* 47 | * Another multiline 48 | * comment 49 | */ 50 | System.out.println(MESSAGE); 51 | } 52 | } 53 | JAVA 54 | 55 | snippets[:java][:expected_with_option_off] = <<-HTML_OPT_INDEPENDENT_LINES_OFF 56 | import java.lang.*; 57 | 58 | /** 59 | * This is some multiline javadoc 60 | * used to test the 61 | */ 62 | public class Test { 63 | public static final String MESSAGE = "My message To the world"; 64 | 65 | static void main() { 66 | /* 67 | * Another multiline 68 | * comment 69 | */ 70 | System.out.println(MESSAGE); 71 | } 72 | } 73 | HTML_OPT_INDEPENDENT_LINES_OFF 74 | 75 | snippets[:java][:expected_with_option_on] = <<-HTML_OPT_INDEPENDENT_LINES_ON 76 | import java.lang.*; 77 | 78 | /** 79 | * This is some multiline javadoc 80 | * used to test the 81 | */ 82 | public class Test { 83 | public static final String MESSAGE = "My message To the world"; 84 | 85 | static void main() { 86 | /* 87 | * Another multiline 88 | * comment 89 | */ 90 | System.out.println(MESSAGE); 91 | } 92 | } 93 | HTML_OPT_INDEPENDENT_LINES_ON 94 | 95 | for lang, code in snippets 96 | tokens = CodeRay.scan code[:in], lang 97 | 98 | assert_equal code[:expected_with_option_off], tokens.html 99 | assert_equal code[:expected_with_option_off], tokens.html(:break_lines => false) 100 | assert_equal code[:expected_with_option_on], tokens.html(:break_lines => true) 101 | end 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /lib/coderay/encoders/html/output.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | class HTML 5 | 6 | # This module is included in the output String of the HTML Encoder. 7 | # 8 | # It provides methods like wrap, div, page etc. 9 | # 10 | # Remember to use #clone instead of #dup to keep the modules the object was 11 | # extended with. 12 | # 13 | # TODO: Rewrite this without monkey patching. 14 | module Output 15 | 16 | attr_accessor :css 17 | 18 | class << self 19 | 20 | # Raises an exception if an object that doesn't respond to to_str is extended by Output, 21 | # to prevent users from misuse. Use Module#remove_method to disable. 22 | def extended o # :nodoc: 23 | warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str 24 | end 25 | 26 | def make_stylesheet css, in_tag = false # :nodoc: 27 | sheet = css.stylesheet 28 | sheet = <<-'CSS' if in_tag 29 | 32 | CSS 33 | sheet 34 | end 35 | 36 | def page_template_for_css css # :nodoc: 37 | sheet = make_stylesheet css 38 | PAGE.apply 'CSS', sheet 39 | end 40 | 41 | end 42 | 43 | def wrapped_in? element 44 | wrapped_in == element 45 | end 46 | 47 | def wrapped_in 48 | @wrapped_in ||= nil 49 | end 50 | attr_writer :wrapped_in 51 | 52 | def wrap_in! template 53 | Template.wrap! self, template, 'CONTENT' 54 | self 55 | end 56 | 57 | def apply_title! title 58 | self.sub!(/()(<\/title>)/) { $1 + title + $2 } 59 | self 60 | end 61 | 62 | def wrap! element, *args 63 | return self if not element or element == wrapped_in 64 | case element 65 | when :div 66 | raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil 67 | wrap_in! DIV 68 | when :span 69 | raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? nil 70 | wrap_in! SPAN 71 | when :page 72 | wrap! :div if wrapped_in? nil 73 | raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div 74 | wrap_in! Output.page_template_for_css(@css) 75 | if args.first.is_a?(Hash) && title = args.first[:title] 76 | apply_title! title 77 | end 78 | self 79 | else 80 | raise "Unknown value %p for :wrap" % element 81 | end 82 | @wrapped_in = element 83 | self 84 | end 85 | 86 | def stylesheet in_tag = false 87 | Output.make_stylesheet @css, in_tag 88 | end 89 | 90 | #-- don't include the templates in docu 91 | 92 | class Template < String # :nodoc: 93 | 94 | def self.wrap! str, template, target 95 | target = Regexp.new(Regexp.escape("<%#{target}%>")) 96 | if template =~ target 97 | str[0,0] = $` 98 | str << $' 99 | else 100 | raise "Template target <%%%p%%> not found" % target 101 | end 102 | end 103 | 104 | def apply target, replacement 105 | target = Regexp.new(Regexp.escape("<%#{target}%>")) 106 | if self =~ target 107 | Template.new($` + replacement + $') 108 | else 109 | raise "Template target <%%%p%%> not found" % target 110 | end 111 | end 112 | 113 | end 114 | 115 | SPAN = Template.new '<span class="CodeRay"><%CONTENT%></span>' 116 | 117 | DIV = Template.new <<-DIV 118 | <div class="CodeRay"> 119 | <div class="code"><pre><%CONTENT%></pre></div> 120 | </div> 121 | DIV 122 | 123 | TABLE = Template.new <<-TABLE 124 | <table class="CodeRay"><tr> 125 | <td class="line-numbers"><pre><%LINE_NUMBERS%></pre></td> 126 | <td class="code"><pre><%CONTENT%></pre></td> 127 | </tr></table> 128 | TABLE 129 | 130 | PAGE = Template.new <<-PAGE 131 | <!DOCTYPE html> 132 | <html> 133 | <head> 134 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> 135 | <title> 136 | 151 | 152 | 153 | 154 | <%CONTENT%> 155 | 156 | 157 | PAGE 158 | 159 | end 160 | 161 | end 162 | 163 | end 164 | end 165 | -------------------------------------------------------------------------------- /lib/coderay/helpers/file_type.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # = FileType 4 | # 5 | # A simple filetype recognizer. 6 | # 7 | # == Usage 8 | # 9 | # # determine the type of the given 10 | # lang = FileType[file_name] 11 | # 12 | # # return :text if the file type is unknown 13 | # lang = FileType.fetch file_name, :text 14 | # 15 | # # try the shebang line, too 16 | # lang = FileType.fetch file_name, :text, true 17 | module FileType 18 | 19 | UnknownFileType = Class.new Exception 20 | 21 | class << self 22 | 23 | # Try to determine the file type of the file. 24 | # 25 | # +filename+ is a relative or absolute path to a file. 26 | # 27 | # The file itself is only accessed when +read_shebang+ is set to true. 28 | # That means you can get filetypes from files that don't exist. 29 | def [] filename, read_shebang = false 30 | name = File.basename filename 31 | ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot 32 | ext2 = filename.to_s[/\.(.*)/, 1] # from first dot 33 | 34 | type = 35 | TypeFromExt[ext] || 36 | TypeFromExt[ext.downcase] || 37 | (TypeFromExt[ext2] if ext2) || 38 | (TypeFromExt[ext2.downcase] if ext2) || 39 | TypeFromName[name] || 40 | TypeFromName[name.downcase] 41 | type ||= type_from_shebang(filename) if read_shebang 42 | 43 | type 44 | end 45 | 46 | # This works like Hash#fetch. 47 | # 48 | # If the filetype cannot be found, the +default+ value 49 | # is returned. 50 | def fetch filename, default = nil, read_shebang = false 51 | if default && block_given? 52 | warn 'Block supersedes default value argument; use either.' 53 | end 54 | 55 | if type = self[filename, read_shebang] 56 | type 57 | else 58 | return yield if block_given? 59 | return default if default 60 | raise UnknownFileType, 'Could not determine type of %p.' % filename 61 | end 62 | end 63 | 64 | protected 65 | 66 | def type_from_shebang filename 67 | return unless File.exist? filename 68 | File.open filename, 'r' do |f| 69 | if first_line = f.gets 70 | if type = first_line[TypeFromShebang] 71 | type.to_sym 72 | end 73 | end 74 | end 75 | end 76 | 77 | end 78 | 79 | TypeFromExt = { 80 | 'c' => :c, 81 | 'cfc' => :xml, 82 | 'cfm' => :xml, 83 | 'clj' => :clojure, 84 | 'css' => :css, 85 | 'diff' => :diff, 86 | 'dpr' => :delphi, 87 | 'erb' => :erb, 88 | 'gemspec' => :ruby, 89 | 'go' => :go, 90 | 'groovy' => :groovy, 91 | 'gvy' => :groovy, 92 | 'h' => :c, 93 | 'haml' => :haml, 94 | 'htm' => :html, 95 | 'html' => :html, 96 | 'html.erb' => :erb, 97 | 'java' => :java, 98 | 'js' => :java_script, 99 | 'json' => :json, 100 | 'lua' => :lua, 101 | 'mab' => :ruby, 102 | 'pas' => :delphi, 103 | 'patch' => :diff, 104 | 'phtml' => :php, 105 | 'php' => :php, 106 | 'php3' => :php, 107 | 'php4' => :php, 108 | 'php5' => :php, 109 | 'prawn' => :ruby, 110 | 'py' => :python, 111 | 'py3' => :python, 112 | 'pyw' => :python, 113 | 'rake' => :ruby, 114 | 'raydebug' => :raydebug, 115 | 'rb' => :ruby, 116 | 'rbw' => :ruby, 117 | 'rhtml' => :erb, 118 | 'rjs' => :ruby, 119 | 'rpdf' => :ruby, 120 | 'ru' => :ruby, # config.ru 121 | 'rxml' => :ruby, 122 | 'sass' => :sass, 123 | 'sql' => :sql, 124 | 'taskpaper' => :taskpaper, 125 | 'template' => :json, # AWS CloudFormation template 126 | 'tmproj' => :xml, 127 | 'xaml' => :xml, 128 | 'xhtml' => :html, 129 | 'xml' => :xml, 130 | 'yaml' => :yaml, 131 | 'yml' => :yaml, 132 | } 133 | for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] 134 | TypeFromExt[cpp_alias] = :cpp 135 | end 136 | 137 | TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ 138 | 139 | TypeFromName = { 140 | 'Capfile' => :ruby, 141 | 'Rakefile' => :ruby, 142 | 'Rantfile' => :ruby, 143 | 'Gemfile' => :ruby, 144 | 'Guardfile' => :ruby, 145 | 'Vagrantfile' => :ruby, 146 | 'Appraisals' => :ruby 147 | } 148 | 149 | end 150 | 151 | end 152 | -------------------------------------------------------------------------------- /README_INDEX.rdoc: -------------------------------------------------------------------------------- 1 | = CodeRay 2 | 3 | Tired of blue'n'gray? Try the original version of this documentation on 4 | coderay.rubychan.de[http://coderay.rubychan.de/doc/] :-) 5 | 6 | == About 7 | 8 | CodeRay is a Ruby library for syntax highlighting. 9 | 10 | You put your code in, and you get it back colored; Keywords, strings, 11 | floats, comments - all in different colors. And with line numbers. 12 | 13 | *Syntax* *Highlighting*... 14 | * makes code easier to read and maintain 15 | * lets you detect syntax errors faster 16 | * helps you to understand the syntax of a language 17 | * looks nice 18 | * is what everybody wants to have on their website 19 | * solves all your problems and makes the girls run after you 20 | 21 | 22 | == Installation 23 | 24 | % gem install coderay 25 | 26 | 27 | === Dependencies 28 | 29 | CodeRay needs Ruby 1.8.7+ or 1.9.2+. It also runs on Rubinius and JRuby. 30 | 31 | 32 | == Example Usage 33 | 34 | require 'coderay' 35 | 36 | html = CodeRay.scan("puts 'Hello, world!'", :ruby).div(:line_numbers => :table) 37 | 38 | 39 | == Documentation 40 | 41 | See CodeRay. 42 | 43 | 44 | == Credits 45 | 46 | === Special Thanks to 47 | 48 | * licenser (Heinz N. Gies) for ending my QBasic career, inventing the Coder 49 | project and the input/output plugin system. 50 | CodeRay would not exist without him. 51 | * bovi (Daniel Bovensiepen) for helping me out on various occasions. 52 | 53 | === Thanks to 54 | 55 | * Caleb Clausen for writing RubyLexer (see 56 | http://rubyforge.org/projects/rubylexer) and lots of very interesting mail 57 | traffic 58 | * birkenfeld (Georg Brandl) and mitsuhiku (Arnim Ronacher) for PyKleur, now pygments. 59 | You guys rock! 60 | * Jamis Buck for writing Syntax (see http://rubyforge.org/projects/syntax) 61 | I got some useful ideas from it. 62 | * Doug Kearns and everyone else who worked on ruby.vim - it not only helped me 63 | coding CodeRay, but also gave me a wonderful target to reach for the Ruby 64 | scanner. 65 | * everyone who uses CodeBB on http://www.rubyforen.de and http://www.python-forum.de 66 | * iGEL, magichisoka, manveru, WoNáDo and everyone I forgot from rubyforen.de 67 | * Dethix from ruby-mine.de 68 | * zickzackw 69 | * Dookie (who is no longer with us...) and Leonidas from http://www.python-forum.de 70 | * Andreas Schwarz for finding out that CaseIgnoringWordList was not case 71 | ignoring! Such things really make you write tests. 72 | * closure for the first version of the Scheme scanner. 73 | * Stefan Walk for the first version of the JavaScript and PHP scanners. 74 | * Josh Goebel for another version of the JavaScript scanner, a SQL and a Diff scanner. 75 | * Jonathan Younger for pointing out the licence confusion caused by wrong LICENSE file. 76 | * Jeremy Hinegardner for finding the shebang-on-empty-file bug in FileType. 77 | * Charles Oliver Nutter and Yehuda Katz for helping me benchmark CodeRay on JRuby. 78 | * Andreas Neuhaus for pointing out a markup bug in coderay/for_redcloth. 79 | * 0xf30fc7 for the FileType patch concerning Delphi file extensions. 80 | * The folks at redmine.org - thank you for using and fixing CodeRay! 81 | * Keith Pitt for his SQL scanners 82 | * Rob Aldred for the terminal encoder 83 | * Trans for pointing out $DEBUG dependencies 84 | * Flameeyes for finding that Term::ANSIColor was obsolete 85 | * matz and all Ruby gods and gurus 86 | * The inventors of: the computer, the internet, the true color display, HTML & 87 | CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime, 88 | manga, coke and green ice tea. 89 | 90 | Where would we be without all those people? 91 | 92 | === Created using 93 | 94 | * Ruby[http://ruby-lang.org/] 95 | * Chihiro (my Sony VAIO laptop); Henrietta (my old MacBook); 96 | Triella, born Rico (my new MacBook); as well as 97 | Seras and Hikari (my PCs) 98 | * RDE[http://homepage2.nifty.com/sakazuki/rde_e.html], 99 | VIM[http://vim.org] and TextMate[http://macromates.com] 100 | * Subversion[http://subversion.tigris.org/] 101 | * Redmine[http://redmine.org/] 102 | * Firefox[http://www.mozilla.org/products/firefox/], 103 | Firebug[http://getfirebug.com/], Safari[http://www.apple.com/safari/], and 104 | Thunderbird[http://www.mozilla.org/products/thunderbird/] 105 | * RubyGems[http://docs.rubygems.org/] and Rake[http://rake.rubyforge.org/] 106 | * TortoiseSVN[http://tortoisesvn.tigris.org/] using Apache via 107 | XAMPP[http://www.apachefriends.org/en/xampp.html] 108 | * RDoc (though I'm quite unsatisfied with it) 109 | * Microsoft Windows (yes, I confess!) and MacOS X 110 | * GNUWin32, MinGW and some other tools to make the shell under windows a bit 111 | less useless 112 | * Term::ANSIColor[http://term-ansicolor.rubyforge.org/] 113 | * PLEAC[http://pleac.sourceforge.net/] code examples 114 | * Github 115 | * Travis CI (http://travis-ci.org/rubychan/github) 116 | 117 | === Free 118 | 119 | * As you can see, CodeRay was created under heavy use of *free* software. 120 | * So CodeRay is also *free*. 121 | * If you use CodeRay to create software, think about making this software 122 | *free*, too. 123 | * Thanks :) 124 | -------------------------------------------------------------------------------- /rake_tasks/code_statistics.rb: -------------------------------------------------------------------------------- 1 | # From rails (http://rubyonrails.com) 2 | # 3 | # Improved by murphy 4 | class CodeStatistics 5 | 6 | TEST_TYPES = /\btest/i 7 | 8 | # Create a new Code Statistic. 9 | # 10 | # Rakefile Example: 11 | # 12 | # desc 'Report code statistics (LOC) from the application' 13 | # task :stats => :copy_files do 14 | # require 'rake_helpers/code_statistics' 15 | # CodeStatistics.new( 16 | # ["Main", "lib"], 17 | # ["Tests", "test"], 18 | # ["Demos", "demo"] 19 | # ).to_s 20 | # end 21 | def initialize(*pairs) 22 | @pairs = pairs 23 | @statistics = calculate_statistics 24 | @total = if pairs.empty? then nil else calculate_total end 25 | end 26 | 27 | # Print a textual table viewing the stats 28 | # 29 | # Intended for console output. 30 | def print 31 | print_header 32 | @pairs.each { |name, path| print_line name, @statistics[name] } 33 | print_splitter 34 | 35 | if @total 36 | print_line 'Total', @total 37 | print_splitter 38 | end 39 | 40 | print_code_test_stats 41 | end 42 | 43 | private 44 | 45 | DEFAULT_FILE_PATTERN = /\.rb$/ 46 | 47 | def calculate_statistics 48 | @pairs.inject({}) do |stats, (name, path, pattern, is_ruby_code)| 49 | pattern ||= DEFAULT_FILE_PATTERN 50 | path = File.join path, '*.rb' 51 | stats[name] = calculate_directory_statistics path, pattern, is_ruby_code 52 | stats 53 | end 54 | end 55 | 56 | def calculate_directory_statistics directory, pattern = DEFAULT_FILE_PATTERN, is_ruby_code = true 57 | is_ruby_code = true if is_ruby_code.nil? 58 | stats = Hash.new 0 59 | 60 | Dir[directory].each do |file_name| 61 | p "Scanning #{file_name}..." if $VERBOSE 62 | next unless file_name =~ pattern 63 | 64 | lines = codelines = classes = modules = methods = 0 65 | empty_lines = comment_lines = 0 66 | in_comment_block = false 67 | 68 | File.readlines(file_name).each do |line| 69 | lines += 1 70 | if line[/^\s*$/] 71 | empty_lines += 1 72 | elsif is_ruby_code 73 | case line 74 | when /^=end\b/ 75 | comment_lines += 1 76 | in_comment_block = false 77 | when in_comment_block 78 | comment_lines += 1 79 | when /^\s*class\b/ 80 | classes += 1 81 | when /^\s*module\b/ 82 | modules += 1 83 | when /^\s*def\b/ 84 | methods += 1 85 | when /^\s*#/ 86 | comment_lines += 1 87 | when /^=begin\b/ 88 | in_comment_block = false 89 | comment_lines += 1 90 | when /^__END__$/ 91 | in_comment_block = true 92 | end 93 | end 94 | end 95 | 96 | codelines = lines - comment_lines - empty_lines 97 | 98 | stats[:lines] += lines 99 | stats[:comments] += comment_lines 100 | stats[:codelines] += codelines 101 | stats[:classes] += classes 102 | stats[:modules] += modules 103 | stats[:methods] += methods 104 | stats[:files] += 1 105 | end 106 | 107 | stats 108 | end 109 | 110 | def calculate_total 111 | total = Hash.new 0 112 | @statistics.each_value { |pair| pair.each { |k, v| total[k] += v } } 113 | total 114 | end 115 | 116 | def calculate_code 117 | code_loc = 0 118 | @statistics.each { |k, v| code_loc += v[:codelines] unless k[TEST_TYPES] } 119 | code_loc 120 | end 121 | 122 | def calculate_tests 123 | test_loc = 0 124 | @statistics.each { |k, v| test_loc += v[:codelines] if k[TEST_TYPES] } 125 | test_loc 126 | end 127 | 128 | def print_header 129 | print_splitter 130 | puts "| T=Test Name | Files | Lines | LOC | Comments | Classes | Modules | Methods | M/C | LOC/M |" 131 | print_splitter 132 | end 133 | 134 | def print_splitter 135 | puts "+---------------------------+-------+-------+-------+----------+---------+---------+---------+-----+-------+" 136 | end 137 | 138 | def print_line name, statistics 139 | m_over_c = (statistics[:methods] / (statistics[:classes] + statistics[:modules])) rescue m_over_c = 0 140 | loc_over_m = (statistics[:codelines] / statistics[:methods]) - 2 rescue loc_over_m = 0 141 | 142 | if name[TEST_TYPES] 143 | name = "T #{name}" 144 | else 145 | name = " #{name}" 146 | end 147 | 148 | line = "| %-25s | %5d | %5d | %5d | %8d | %7d | %7d | %7d | %3d | %5d |" % ( 149 | [name, *statistics.values_at(:files, :lines, :codelines, :comments, :classes, :modules, :methods)] + 150 | [m_over_c, loc_over_m] ) 151 | 152 | puts line 153 | end 154 | 155 | def print_code_test_stats 156 | code = calculate_code 157 | tests = calculate_tests 158 | 159 | puts " Code LOC = #{code} Test LOC = #{tests} Code:Test Ratio = [1 : #{sprintf("%.2f", tests.to_f / code)}]" 160 | puts "" 161 | end 162 | 163 | end 164 | 165 | # Run a test script. 166 | if $0 == __FILE__ 167 | $VERBOSE = true 168 | CodeStatistics.new( 169 | ['This dir', File.dirname(__FILE__)] 170 | ).print 171 | end 172 | -------------------------------------------------------------------------------- /lib/coderay/scanners/yaml.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | # Scanner for YAML. 5 | # 6 | # Based on the YAML scanner from Syntax by Jamis Buck. 7 | class YAML < Scanner 8 | 9 | register_for :yaml 10 | file_extension 'yml' 11 | 12 | KINDS_NOT_LOC = :all 13 | 14 | protected 15 | 16 | def scan_tokens encoder, options 17 | 18 | state = :initial 19 | key_indent = string_indent = 0 20 | 21 | until eos? 22 | 23 | key_indent = nil if bol? 24 | 25 | if match = scan(/ +[\t ]*/) 26 | encoder.text_token match, :space 27 | 28 | elsif match = scan(/\n+/) 29 | encoder.text_token match, :space 30 | state = :initial if match.index(?\n) 31 | 32 | elsif match = scan(/#.*/) 33 | encoder.text_token match, :comment 34 | 35 | elsif bol? and case 36 | when match = scan(/---|\.\.\./) 37 | encoder.begin_group :head 38 | encoder.text_token match, :head 39 | encoder.end_group :head 40 | next 41 | when match = scan(/%.*/) 42 | encoder.text_token match, :doctype 43 | next 44 | end 45 | 46 | elsif state == :value and case 47 | when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/) 48 | encoder.begin_group :string 49 | encoder.text_token match, :delimiter 50 | encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty? 51 | encoder.text_token match, :delimiter if match = scan(/"/) 52 | encoder.end_group :string 53 | next 54 | when match = scan(/[|>][-+]?/) 55 | encoder.begin_group :string 56 | encoder.text_token match, :delimiter 57 | string_indent = key_indent || column(pos - match.size) - 1 58 | encoder.text_token matched, :content if scan(/(?:\n+ {#{string_indent + 1}}.*)+/) 59 | encoder.end_group :string 60 | next 61 | when match = scan(/(?![!"*&]).+?(?=$|\s+#)/) 62 | encoder.begin_group :string 63 | encoder.text_token match, :content 64 | string_indent = key_indent || column(pos - match.size) - 1 65 | encoder.text_token matched, :content if scan(/(?:\n+ {#{string_indent + 1}}.*)+/) 66 | encoder.end_group :string 67 | next 68 | end 69 | 70 | elsif case 71 | when match = scan(/[-:](?= |$)/) 72 | state = :value if state == :colon && (match == ':' || match == '-') 73 | state = :value if state == :initial && match == '-' 74 | encoder.text_token match, :operator 75 | next 76 | when match = scan(/[,{}\[\]]/) 77 | encoder.text_token match, :operator 78 | next 79 | when state == :initial && match = scan(/[-\w.()\/ ]*\S(?= *:(?: |$))/) 80 | encoder.text_token match, :key 81 | key_indent = column(pos - match.size) - 1 82 | state = :colon 83 | next 84 | when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/) 85 | encoder.begin_group :key 86 | encoder.text_token match[0,1], :delimiter 87 | encoder.text_token match[1..-2], :content if match.size > 2 88 | encoder.text_token match[-1,1], :delimiter 89 | encoder.end_group :key 90 | key_indent = column(pos - match.size) - 1 91 | state = :colon 92 | next 93 | when match = scan(/(![\w\/]+)(:([\w:]+))?/) 94 | encoder.text_token self[1], :type 95 | if self[2] 96 | encoder.text_token ':', :operator 97 | encoder.text_token self[3], :class 98 | end 99 | next 100 | when match = scan(/&\S+/) 101 | encoder.text_token match, :variable 102 | next 103 | when match = scan(/\*\w+/) 104 | encoder.text_token match, :global_variable 105 | next 106 | when match = scan(/< e 80 | add_failure(e.message, e.backtrace) 81 | rescue Exception 82 | raise if PASSTHROUGH_EXCEPTIONS.include? $!.class 83 | add_error($!) 84 | ensure 85 | begin 86 | teardown 87 | rescue AssertionFailedError => e 88 | add_failure(e.message, e.backtrace) 89 | rescue Exception 90 | raise if PASSTHROUGH_EXCEPTIONS.include? $!.class 91 | add_error($!) 92 | end 93 | end 94 | result.add_run 95 | yield(FINISHED, name) 96 | end 97 | 98 | # Called before every test method runs. Can be used 99 | # to set up fixture information. 100 | def setup 101 | end 102 | 103 | # Called after every test method runs. Can be used to tear 104 | # down fixture information. 105 | def teardown 106 | end 107 | 108 | def default_test 109 | flunk("No tests were specified") 110 | end 111 | 112 | # Returns whether this individual test passed or 113 | # not. Primarily for use in teardown so that artifacts 114 | # can be left behind if the test fails. 115 | def passed? 116 | return @test_passed 117 | end 118 | private :passed? 119 | 120 | def size 121 | 1 122 | end 123 | 124 | def add_assertion 125 | @_result.add_assertion 126 | end 127 | private :add_assertion 128 | 129 | def add_failure(message, all_locations=caller()) 130 | @test_passed = false 131 | @_result.add_failure(Failure.new(name, filter_backtrace(all_locations), message)) 132 | end 133 | private :add_failure 134 | 135 | def add_error(exception) 136 | @test_passed = false 137 | @_result.add_error(Error.new(name, exception)) 138 | end 139 | private :add_error 140 | 141 | # Returns a human-readable name for the specific test that 142 | # this instance of TestCase represents. 143 | def name 144 | "#{@method_name}(#{self.class.name})" 145 | end 146 | 147 | # Overridden to return #name. 148 | def to_s 149 | name 150 | end 151 | 152 | # It's handy to be able to compare TestCase instances. 153 | def ==(other) 154 | return false unless(other.kind_of?(self.class)) 155 | return false unless(@method_name == other.method_name) 156 | self.class == other.class 157 | end 158 | end 159 | end 160 | end 161 | -------------------------------------------------------------------------------- /lib/coderay/scanners/delphi.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | # Scanner for the Delphi language (Object Pascal). 5 | # 6 | # Alias: +pascal+ 7 | class Delphi < Scanner 8 | 9 | register_for :delphi 10 | file_extension 'pas' 11 | 12 | KEYWORDS = [ 13 | 'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class', 14 | 'const', 'constructor', 'destructor', 'dispinterface', 'div', 'do', 15 | 'downto', 'else', 'end', 'except', 'exports', 'file', 'finalization', 16 | 'finally', 'for', 'function', 'goto', 'if', 'implementation', 'in', 17 | 'inherited', 'initialization', 'inline', 'interface', 'is', 'label', 18 | 'library', 'mod', 'nil', 'not', 'object', 'of', 'or', 'out', 'packed', 19 | 'procedure', 'program', 'property', 'raise', 'record', 'repeat', 20 | 'resourcestring', 'set', 'shl', 'shr', 'string', 'then', 'threadvar', 21 | 'to', 'try', 'type', 'unit', 'until', 'uses', 'var', 'while', 'with', 22 | 'xor', 'on', 23 | ] # :nodoc: 24 | 25 | DIRECTIVES = [ 26 | 'absolute', 'abstract', 'assembler', 'at', 'automated', 'cdecl', 27 | 'contains', 'deprecated', 'dispid', 'dynamic', 'export', 28 | 'external', 'far', 'forward', 'implements', 'local', 29 | 'near', 'nodefault', 'on', 'overload', 'override', 30 | 'package', 'pascal', 'platform', 'private', 'protected', 'public', 31 | 'published', 'read', 'readonly', 'register', 'reintroduce', 32 | 'requires', 'resident', 'safecall', 'stdcall', 'stored', 'varargs', 33 | 'virtual', 'write', 'writeonly', 34 | ] # :nodoc: 35 | 36 | IDENT_KIND = WordList::CaseIgnoring.new(:ident). 37 | add(KEYWORDS, :keyword). 38 | add(DIRECTIVES, :directive) # :nodoc: 39 | 40 | NAME_FOLLOWS = WordList::CaseIgnoring.new(false). 41 | add(%w(procedure function .)) # :nodoc: 42 | 43 | protected 44 | 45 | def scan_tokens encoder, options 46 | 47 | state = :initial 48 | last_token = '' 49 | 50 | until eos? 51 | 52 | if state == :initial 53 | 54 | if match = scan(/ \s+ /x) 55 | encoder.text_token match, :space 56 | next 57 | 58 | elsif match = scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx) 59 | encoder.text_token match, :preprocessor 60 | next 61 | 62 | elsif match = scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx) 63 | encoder.text_token match, :comment 64 | next 65 | 66 | elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x) 67 | encoder.text_token match, :operator 68 | 69 | elsif match = scan(/\./) 70 | encoder.text_token match, :operator 71 | next if last_token == 'end' 72 | 73 | elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) 74 | encoder.text_token match, NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match] 75 | 76 | elsif match = skip(/ ' ( [^\n']|'' ) (?:'|$) /x) 77 | encoder.begin_group :char 78 | encoder.text_token "'", :delimiter 79 | encoder.text_token self[1], :content 80 | encoder.text_token "'", :delimiter 81 | encoder.end_group :char 82 | next 83 | 84 | elsif match = scan(/ ' /x) 85 | encoder.begin_group :string 86 | encoder.text_token match, :delimiter 87 | state = :string 88 | 89 | elsif match = scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x) 90 | encoder.text_token match, :char 91 | 92 | elsif match = scan(/ \$ [0-9A-Fa-f]+ /x) 93 | encoder.text_token match, :hex 94 | 95 | elsif match = scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x) 96 | encoder.text_token match, :integer 97 | 98 | elsif match = scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x) 99 | encoder.text_token match, :float 100 | 101 | else 102 | encoder.text_token getch, :error 103 | next 104 | 105 | end 106 | 107 | elsif state == :string 108 | if match = scan(/[^\n']+/) 109 | encoder.text_token match, :content 110 | elsif match = scan(/''/) 111 | encoder.text_token match, :char 112 | elsif match = scan(/'/) 113 | encoder.text_token match, :delimiter 114 | encoder.end_group :string 115 | state = :initial 116 | next 117 | elsif match = scan(/\n/) 118 | encoder.end_group :string 119 | encoder.text_token match, :space 120 | state = :initial 121 | else 122 | raise "else case \' reached; %p not handled." % peek(1), encoder 123 | end 124 | 125 | else 126 | raise 'else-case reached', encoder 127 | 128 | end 129 | 130 | last_token = match 131 | 132 | end 133 | 134 | if state == :string 135 | encoder.end_group state 136 | end 137 | 138 | encoder 139 | end 140 | 141 | end 142 | 143 | end 144 | end 145 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'simplecov' 2 | 3 | # This file was generated by the `rspec --init` command. Conventionally, all 4 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 5 | # The generated `.rspec` file contains `--require spec_helper` which will cause 6 | # this file to always be loaded, without a need to explicitly require it in any 7 | # files. 8 | # 9 | # Given that it is always loaded, you are encouraged to keep this file as 10 | # light-weight as possible. Requiring heavyweight dependencies from this file 11 | # will add to the boot time of your test suite on EVERY test run, even for an 12 | # individual file that may not need all of that loaded. Instead, consider making 13 | # a separate helper file that requires the additional dependencies and performs 14 | # the additional setup, and require it from the spec files that actually need 15 | # it. 16 | # 17 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 18 | RSpec.configure do |config| 19 | # rspec-expectations config goes here. You can use an alternate 20 | # assertion/expectation library such as wrong or the stdlib/minitest 21 | # assertions if you prefer. 22 | config.expect_with :rspec do |expectations| 23 | # This option will default to `true` in RSpec 4. It makes the `description` 24 | # and `failure_message` of custom matchers include text for helper methods 25 | # defined using `chain`, e.g.: 26 | # be_bigger_than(2).and_smaller_than(4).description 27 | # # => "be bigger than 2 and smaller than 4" 28 | # ...rather than: 29 | # # => "be bigger than 2" 30 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 31 | end if RUBY_VERSION >= '1.9' 32 | 33 | # rspec-mocks config goes here. You can use an alternate test double 34 | # library (such as bogus or mocha) by changing the `mock_with` option here. 35 | config.mock_with :rspec do |mocks| 36 | # Prevents you from mocking or stubbing a method that does not exist on 37 | # a real object. This is generally recommended, and will default to 38 | # `true` in RSpec 4. 39 | mocks.verify_partial_doubles = true 40 | end 41 | 42 | # This option will default to `:apply_to_host_groups` in RSpec 4 (and will 43 | # have no way to turn it off -- the option exists only for backwards 44 | # compatibility in RSpec 3). It causes shared context metadata to be 45 | # inherited by the metadata hash of host groups and examples, rather than 46 | # triggering implicit auto-inclusion in groups with matching metadata. 47 | config.shared_context_metadata_behavior = :apply_to_host_groups 48 | 49 | # The settings below are suggested to provide a good initial experience 50 | # with RSpec, but feel free to customize to your heart's content. 51 | =begin 52 | # This allows you to limit a spec run to individual examples or groups 53 | # you care about by tagging them with `:focus` metadata. When nothing 54 | # is tagged with `:focus`, all examples get run. RSpec also provides 55 | # aliases for `it`, `describe`, and `context` that include `:focus` 56 | # metadata: `fit`, `fdescribe` and `fcontext`, respectively. 57 | config.filter_run_when_matching :focus 58 | 59 | # Allows RSpec to persist some state between runs in order to support 60 | # the `--only-failures` and `--next-failure` CLI options. We recommend 61 | # you configure your source control system to ignore this file. 62 | config.example_status_persistence_file_path = "spec/examples.txt" 63 | 64 | # Limits the available syntax to the non-monkey patched syntax that is 65 | # recommended. For more details, see: 66 | # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ 67 | # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 68 | # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode 69 | config.disable_monkey_patching! 70 | 71 | # This setting enables warnings. It's recommended, but in some cases may 72 | # be too noisy due to issues in dependencies. 73 | config.warnings = true 74 | 75 | # Many RSpec users commonly either run the entire suite or an individual 76 | # file, and it's useful to allow more verbose output when running an 77 | # individual spec file. 78 | if config.files_to_run.one? 79 | # Use the documentation formatter for detailed output, 80 | # unless a formatter has already been configured 81 | # (e.g. via a command-line flag). 82 | config.default_formatter = "doc" 83 | end 84 | 85 | # Print the 10 slowest examples and example groups at the 86 | # end of the spec run, to help surface which specs are running 87 | # particularly slow. 88 | config.profile_examples = 10 89 | 90 | # Run specs in random order to surface order dependencies. If you find an 91 | # order dependency and want to debug it, you can fix the order by providing 92 | # the seed, which is printed after each run. 93 | # --seed 1234 94 | config.order = :random 95 | 96 | # Seed global randomization in this process using the `--seed` CLI option. 97 | # Setting this allows you to use `--seed` to deterministically reproduce 98 | # test failures related to randomization by passing the same `--seed` value 99 | # as the one that triggered the failure. 100 | Kernel.srand config.seed 101 | =end 102 | end 103 | 104 | $:.unshift File.expand_path('../lib', __FILE__) 105 | require 'coderay' 106 | -------------------------------------------------------------------------------- /lib/coderay/token_kinds.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | 3 | # A Hash of all known token kinds and their associated CSS classes. 4 | TokenKinds = Hash.new(false) 5 | 6 | # speedup 7 | TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity 8 | 9 | TokenKinds.update( # :nodoc: 10 | :debug => 'debug', # highlight for debugging (white on blue background) 11 | 12 | :annotation => 'annotation', # Groovy, Java 13 | :attribute_name => 'attribute-name', # HTML, CSS 14 | :attribute_value => 'attribute-value', # HTML 15 | :binary => 'binary', # Python, Ruby 16 | :char => 'char', # most scanners, also inside of strings 17 | :class => 'class', # lots of scanners, for different purposes also in CSS 18 | :class_variable => 'class-variable', # Ruby, YAML 19 | :color => 'color', # CSS 20 | :comment => 'comment', # most scanners 21 | :constant => 'constant', # PHP, Ruby 22 | :content => 'content', # inside of strings, most scanners 23 | :decorator => 'decorator', # Python 24 | :definition => 'definition', # CSS 25 | :delimiter => 'delimiter', # inside strings, comments and other types 26 | :directive => 'directive', # lots of scanners 27 | :doctype => 'doctype', # Goorvy, HTML, Ruby, YAML 28 | :docstring => 'docstring', # Python 29 | :done => 'done', # Taskpaper 30 | :entity => 'entity', # HTML 31 | :error => 'error', # invalid token, most scanners 32 | :escape => 'escape', # Ruby (string inline variables like #$foo, #@bar) 33 | :exception => 'exception', # Java, PHP, Python 34 | :filename => 'filename', # Diff 35 | :float => 'float', # most scanners 36 | :function => 'function', # CSS, JavaScript, PHP 37 | :global_variable => 'global-variable', # Ruby, YAML 38 | :hex => 'hex', # hexadecimal number; lots of scanners 39 | :id => 'id', # CSS 40 | :imaginary => 'imaginary', # Python 41 | :important => 'important', # CSS, Taskpaper 42 | :include => 'include', # C, Groovy, Java, Python, Sass 43 | :inline => 'inline', # nested code, eg. inline string evaluation; lots of scanners 44 | :inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why use inline_delimiter? 45 | :instance_variable => 'instance-variable', # Ruby 46 | :integer => 'integer', # most scanners 47 | :key => 'key', # lots of scanners, used together with :value 48 | :keyword => 'keyword', # reserved word that's actually implemented; most scanners 49 | :label => 'label', # C, PHP 50 | :local_variable => 'local-variable', # local and magic variables; some scanners 51 | :map => 'map', # Lua tables 52 | :modifier => 'modifier', # used inside on strings; lots of scanners 53 | :namespace => 'namespace', # Clojure, Java, Taskpaper 54 | :octal => 'octal', # lots of scanners 55 | :predefined => 'predefined', # predefined function: lots of scanners 56 | :predefined_constant => 'predefined-constant',# lots of scanners 57 | :predefined_type => 'predefined-type', # C, Java, PHP 58 | :preprocessor => 'preprocessor', # C, Delphi, HTML 59 | :pseudo_class => 'pseudo-class', # CSS 60 | :regexp => 'regexp', # Groovy, JavaScript, Ruby 61 | :reserved => 'reserved', # most scanners 62 | :shell => 'shell', # Ruby 63 | :string => 'string', # most scanners 64 | :symbol => 'symbol', # Clojure, Ruby, YAML 65 | :tag => 'tag', # CSS, HTML 66 | :type => 'type', # CSS, Java, SQL, YAML 67 | :value => 'value', # used together with :key; CSS, JSON, YAML 68 | :variable => 'variable', # Sass, SQL, YAML 69 | 70 | :change => 'change', # Diff 71 | :delete => 'delete', # Diff 72 | :head => 'head', # Diff, YAML 73 | :insert => 'insert', # Diff 74 | :eyecatcher => 'eyecatcher', # Diff 75 | 76 | :ident => false, # almost all scanners 77 | :operator => false, # almost all scanners 78 | 79 | :space => false, # almost all scanners 80 | :plain => false # almost all scanners 81 | ) 82 | 83 | TokenKinds[:method] = TokenKinds[:function] 84 | TokenKinds[:unknown] = TokenKinds[:plain] 85 | end 86 | -------------------------------------------------------------------------------- /lib/coderay/styles/alpha.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Styles 3 | 4 | # A colorful theme using CSS 3 colors (with alpha channel). 5 | class Alpha < Style 6 | 7 | register_for :alpha 8 | 9 | code_background = 'hsl(0,0%,95%)' 10 | numbers_background = 'hsl(180,65%,90%)' 11 | border_color = 'silver' 12 | normal_color = 'black' 13 | 14 | CSS_MAIN_STYLES = <<-MAIN # :nodoc: 15 | .CodeRay { 16 | background-color: #{code_background}; 17 | border: 1px solid #{border_color}; 18 | color: #{normal_color}; 19 | } 20 | .CodeRay pre { 21 | margin: 0px; 22 | } 23 | 24 | span.CodeRay { white-space: pre; border: 0px; padding: 2px; } 25 | 26 | table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } 27 | table.CodeRay td { padding: 2px 4px; vertical-align: top; } 28 | 29 | .CodeRay .line-numbers { 30 | background-color: #{numbers_background}; 31 | color: gray; 32 | text-align: right; 33 | -webkit-user-select: none; 34 | -moz-user-select: none; 35 | user-select: none; 36 | } 37 | .CodeRay .line-numbers a { 38 | background-color: #{numbers_background} !important; 39 | color: gray !important; 40 | text-decoration: none !important; 41 | } 42 | .CodeRay .line-numbers pre { 43 | word-break: normal; 44 | } 45 | .CodeRay .line-numbers a:target { color: blue !important; } 46 | .CodeRay .line-numbers .highlighted { color: red !important; } 47 | .CodeRay .line-numbers .highlighted a { color: red !important; } 48 | .CodeRay span.line-numbers { padding: 0px 4px; } 49 | .CodeRay .line { display: block; float: left; width: 100%; } 50 | .CodeRay .code { width: 100%; } 51 | MAIN 52 | 53 | TOKEN_COLORS = <<-'TOKENS' 54 | .debug { color: white !important; background: blue !important; } 55 | 56 | .annotation { color:#007 } 57 | .attribute-name { color:#b48 } 58 | .attribute-value { color:#700 } 59 | .binary { color:#549 } 60 | .binary .char { color:#325 } 61 | .binary .delimiter { color:#325 } 62 | .char { color:#D20 } 63 | .char .content { color:#D20 } 64 | .char .delimiter { color:#710 } 65 | .class { color:#B06; font-weight:bold } 66 | .class-variable { color:#369 } 67 | .color { color:#0A0 } 68 | .comment { color:#777 } 69 | .comment .char { color:#444 } 70 | .comment .delimiter { color:#444 } 71 | .constant { color:#036; font-weight:bold } 72 | .decorator { color:#B0B } 73 | .definition { color:#099; font-weight:bold } 74 | .delimiter { color:black } 75 | .directive { color:#088; font-weight:bold } 76 | .docstring { color:#D42; } 77 | .doctype { color:#34b } 78 | .done { text-decoration: line-through; color: gray } 79 | .entity { color:#800; font-weight:bold } 80 | .error { color:#F00; background-color:#FAA } 81 | .escape { color:#666 } 82 | .exception { color:#C00; font-weight:bold } 83 | .float { color:#60E } 84 | .function { color:#06B; font-weight:bold } 85 | .function .delimiter { color:#059 } 86 | .function .content { color:#037 } 87 | .global-variable { color:#d70 } 88 | .hex { color:#02b } 89 | .id { color:#33D; font-weight:bold } 90 | .include { color:#B44; font-weight:bold } 91 | .inline { background-color: hsla(0,0%,0%,0.07); color: black } 92 | .inline-delimiter { font-weight: bold; color: #666 } 93 | .instance-variable { color:#33B } 94 | .integer { color:#00D } 95 | .imaginary { color:#f00 } 96 | .important { color:#D00 } 97 | .key { color: #606 } 98 | .key .char { color: #60f } 99 | .key .delimiter { color: #404 } 100 | .keyword { color:#080; font-weight:bold } 101 | .label { color:#970; font-weight:bold } 102 | .local-variable { color:#950 } 103 | .map .content { color:#808 } 104 | .map .delimiter { color:#40A} 105 | .map { background-color:hsla(200,100%,50%,0.06); } 106 | .namespace { color:#707; font-weight:bold } 107 | .octal { color:#40E } 108 | .operator { } 109 | .predefined { color:#369; font-weight:bold } 110 | .predefined-constant { color:#069 } 111 | .predefined-type { color:#0a8; font-weight:bold } 112 | .preprocessor { color:#579 } 113 | .pseudo-class { color:#00C; font-weight:bold } 114 | .regexp { background-color:hsla(300,100%,50%,0.06); } 115 | .regexp .content { color:#808 } 116 | .regexp .delimiter { color:#404 } 117 | .regexp .modifier { color:#C2C } 118 | .reserved { color:#080; font-weight:bold } 119 | .shell { background-color:hsla(120,100%,50%,0.06); } 120 | .shell .content { color:#2B2 } 121 | .shell .delimiter { color:#161 } 122 | .string { background-color:hsla(0,100%,50%,0.05); } 123 | .string .char { color: #b0b } 124 | .string .content { color: #D20 } 125 | .string .delimiter { color: #710 } 126 | .string .modifier { color: #E40 } 127 | .symbol { color:#A60 } 128 | .symbol .content { color:#A60 } 129 | .symbol .delimiter { color:#740 } 130 | .tag { color:#070; font-weight:bold } 131 | .type { color:#339; font-weight:bold } 132 | .value { color: #088 } 133 | .variable { color:#037 } 134 | 135 | .insert { background: hsla(120,100%,50%,0.12) } 136 | .delete { background: hsla(0,100%,50%,0.12) } 137 | .change { color: #bbf; background: #007 } 138 | .head { color: #f8f; background: #505 } 139 | .head .filename { color: white; } 140 | 141 | .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } 142 | .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } 143 | 144 | .insert .insert { color: #0c0; background:transparent; font-weight:bold } 145 | .delete .delete { color: #c00; background:transparent; font-weight:bold } 146 | .change .change { color: #88f } 147 | .head .head { color: #f4f } 148 | TOKENS 149 | 150 | end 151 | 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /lib/coderay/scanners/ruby/patterns.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | module CodeRay 3 | module Scanners 4 | 5 | module Ruby::Patterns # :nodoc: all 6 | 7 | KEYWORDS = %w[ 8 | and def end in or unless begin 9 | defined? ensure module redo super until 10 | BEGIN break do next rescue then 11 | when END case else for retry 12 | while alias class elsif if not return 13 | undef yield 14 | ] 15 | 16 | # See http://murfy.de/ruby-constants. 17 | PREDEFINED_CONSTANTS = %w[ 18 | nil true false self 19 | DATA ARGV ARGF ENV 20 | FALSE TRUE NIL 21 | STDERR STDIN STDOUT 22 | TOPLEVEL_BINDING 23 | RUBY_COPYRIGHT RUBY_DESCRIPTION RUBY_ENGINE RUBY_PATCHLEVEL 24 | RUBY_PLATFORM RUBY_RELEASE_DATE RUBY_REVISION RUBY_VERSION 25 | __FILE__ __LINE__ __ENCODING__ 26 | ] 27 | 28 | IDENT_KIND = WordList.new(:ident). 29 | add(KEYWORDS, :keyword). 30 | add(PREDEFINED_CONSTANTS, :predefined_constant) 31 | 32 | KEYWORD_NEW_STATE = WordList.new(:initial). 33 | add(%w[ def ], :def_expected). 34 | add(%w[ undef ], :undef_expected). 35 | add(%w[ alias ], :alias_expected). 36 | add(%w[ class module ], :module_expected) 37 | 38 | IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? Regexp.new('[[:alpha:]_[^\0-\177]][[:alnum:]_[^\0-\177]]*') : /[^\W\d]\w*/ 39 | 40 | METHOD_NAME = / #{IDENT} [?!]? /ox 41 | METHOD_NAME_OPERATOR = / 42 | \*\*? # multiplication and power 43 | | [-+~]@? # plus, minus, tilde with and without at sign 44 | | [\/%&|^`] # division, modulo or format strings, and, or, xor, system 45 | | \[\]=? # array getter and setter 46 | | << | >> # append or shift left, shift right 47 | | <=?>? | >=? # comparison, rocket operator 48 | | ===? | =~ # simple equality, case equality, match 49 | | ![~=@]? # negation with and without at sign, not-equal and not-match 50 | /ox 51 | METHOD_SUFFIX = / (?: [?!] | = (?![~>]|=(?!>)) ) /x 52 | METHOD_NAME_EX = / #{IDENT} #{METHOD_SUFFIX}? | #{METHOD_NAME_OPERATOR} /ox 53 | METHOD_AFTER_DOT = / #{IDENT} [?!]? | #{METHOD_NAME_OPERATOR} /ox 54 | INSTANCE_VARIABLE = / @ #{IDENT} /ox 55 | CLASS_VARIABLE = / @@ #{IDENT} /ox 56 | OBJECT_VARIABLE = / @@? #{IDENT} /ox 57 | GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox 58 | PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox 59 | VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox 60 | 61 | QUOTE_TO_TYPE = { 62 | '`' => :shell, 63 | '/' => :regexp, 64 | } 65 | QUOTE_TO_TYPE.default = :string 66 | 67 | REGEXP_MODIFIERS = /[mousenix]*/ 68 | 69 | DECIMAL = /\d+(?:_\d+)*/ 70 | OCTAL = /0_?[0-7]+(?:_[0-7]+)*/ 71 | HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/ 72 | BINARY = /0b[01]+(?:_[01]+)*/ 73 | 74 | EXPONENT = / [eE] [+-]? #{DECIMAL} /ox 75 | FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox 76 | FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox 77 | NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox 78 | 79 | SYMBOL = / 80 | : 81 | (?: 82 | #{METHOD_NAME_EX} 83 | | #{PREFIX_VARIABLE} 84 | | ['"] 85 | ) 86 | /ox 87 | METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox 88 | 89 | SIMPLE_ESCAPE = / 90 | [abefnrstv] 91 | | [0-7]{1,3} 92 | | x[0-9A-Fa-f]{1,2} 93 | | . 94 | /mx 95 | 96 | CONTROL_META_ESCAPE = / 97 | (?: M-|C-|c ) 98 | (?: \\ (?: M-|C-|c ) )* 99 | (?: [^\\] | \\ #{SIMPLE_ESCAPE} )? 100 | /mox 101 | 102 | ESCAPE = / 103 | #{CONTROL_META_ESCAPE} | #{SIMPLE_ESCAPE} 104 | /mox 105 | 106 | CHARACTER = / 107 | \? 108 | (?: 109 | [^\s\\] 110 | | \\ #{ESCAPE} 111 | ) 112 | /mox 113 | 114 | # NOTE: This is not completely correct, but 115 | # nobody needs heredoc delimiters ending with \n. 116 | HEREDOC_OPEN = / 117 | << ([-~])? # $1 = float 118 | (?: 119 | ( [A-Za-z_0-9]+ ) # $2 = delim 120 | | 121 | ( ["'`\/] ) # $3 = quote, type 122 | ( [^\n]*? ) \3 # $4 = delim 123 | ) 124 | /mx 125 | 126 | RUBYDOC = / 127 | =begin (?!\S) 128 | .*? 129 | (?: \Z | ^=end (?!\S) [^\n]* ) 130 | /mx 131 | 132 | DATA = / 133 | __END__$ 134 | .*? 135 | (?: \Z | (?=^\#CODE) ) 136 | /mx 137 | 138 | RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo 139 | 140 | # Checks for a valid value to follow. This enables 141 | # value_expected in method calls without parentheses. 142 | VALUE_FOLLOWS = / 143 | (?>[ \t\f\v]+) 144 | (?: 145 | [%\/][^\s=] 146 | | <<-?\S 147 | | [-+] \d 148 | | #{CHARACTER} 149 | ) 150 | /ox 151 | KEYWORDS_EXPECTING_VALUE = WordList.new.add(%w[ 152 | and end in or unless begin 153 | defined? ensure redo super until 154 | break do next rescue then 155 | when case else for retry 156 | while elsif if not return 157 | yield 158 | ]) 159 | 160 | FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x 161 | FANCY_STRING_KIND = Hash.new(:string).merge({ 162 | 'i' => :symbol, 163 | 'I' => :symbol, 164 | 'r' => :regexp, 165 | 's' => :symbol, 166 | 'x' => :shell, 167 | }) 168 | FANCY_STRING_INTERPRETED = Hash.new(true).merge({ 169 | 'i' => false, 170 | 'q' => false, 171 | 's' => false, 172 | 'w' => false, 173 | }) 174 | 175 | end 176 | 177 | end 178 | end 179 | -------------------------------------------------------------------------------- /test/functional/examples.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | 3 | $:.unshift File.expand_path('../../../lib', __FILE__) 4 | require 'coderay' 5 | 6 | class ExamplesTest < Test::Unit::TestCase 7 | 8 | def test_examples 9 | # output as HTML div (using inline CSS styles) 10 | div = CodeRay.scan('puts "Hello, world!"', :ruby).div 11 | assert_equal <<-DIV, div 12 |
13 | 
puts "Hello, world!"
14 |
15 | DIV 16 | 17 | # ...with line numbers 18 | div = CodeRay.scan(<<-CODE.chomp, :ruby).div(:line_numbers => :table) 19 | 5.times do 20 | puts 'Hello, world!' 21 | end 22 | CODE 23 | assert_equal <<-DIV, div 24 | 25 | 29 | 32 | 
1
 26 | 2
 27 | 3
 28 | 
5.times do
 30 |   puts 'Hello, world!'
 31 | end
33 | DIV 34 | 35 | # output as standalone HTML page (using CSS classes) 36 | page = CodeRay.scan('puts "Hello, world!"', :ruby).page 37 | assert_match <<-PAGE, page 38 | 39 | 40 | 41 | 43 | 44 | 
1
 42 | 
puts "Hello, world!"
45 | 46 | 47 | PAGE 48 | 49 | # keep scanned tokens for later use 50 | tokens = CodeRay.scan('{ "just": "an", "example": 42 }', :json) 51 | assert_kind_of CodeRay::TokensProxy, tokens 52 | 53 | assert_equal ["{", :operator, " ", :space, :begin_group, :key, 54 | "\"", :delimiter, "just", :content, "\"", :delimiter, 55 | :end_group, :key, ":", :operator, " ", :space, 56 | :begin_group, :string, "\"", :delimiter, "an", :content, 57 | "\"", :delimiter, :end_group, :string, ",", :operator, 58 | " ", :space, :begin_group, :key, "\"", :delimiter, 59 | "example", :content, "\"", :delimiter, :end_group, :key, 60 | ":", :operator, " ", :space, "42", :integer, 61 | " ", :space, "}", :operator], tokens.tokens 62 | 63 | # produce a token statistic 64 | assert_equal <<-STATISTIC, tokens.statistic 65 | 66 | Code Statistics 67 | 68 | Tokens 26 69 | Non-Whitespace 15 70 | Bytes Total 31 71 | 72 | Token Types (7): 73 | type count ratio size (average) 74 | ------------------------------------------------------------- 75 | TOTAL 26 100.00 % 1.2 76 | delimiter 6 23.08 % 1.0 77 | operator 5 19.23 % 1.0 78 | space 5 19.23 % 1.0 79 | key 4 15.38 % 0.0 80 | :begin_group 3 11.54 % 0.0 81 | :end_group 3 11.54 % 0.0 82 | content 3 11.54 % 4.3 83 | string 2 7.69 % 0.0 84 | integer 1 3.85 % 2.0 85 | 86 | STATISTIC 87 | 88 | # count the tokens 89 | assert_equal 26, tokens.count 90 | 91 | # produce a HTML div, but with CSS classes 92 | div = tokens.div(:css => :class) 93 | assert_equal <<-DIV, div 94 |
95 | 
{ "just": "an", "example": 42 }
96 |
97 | DIV 98 | 99 | # highlight a file (HTML div); guess the file type base on the extension 100 | assert_equal :ruby, CodeRay::FileType[__FILE__] 101 | 102 | # get a new scanner for Python 103 | python_scanner = CodeRay.scanner :python 104 | assert_kind_of CodeRay::Scanners::Python, python_scanner 105 | 106 | # get a new encoder for terminal 107 | terminal_encoder = CodeRay.encoder :term 108 | assert_kind_of CodeRay::Encoders::Terminal, terminal_encoder 109 | 110 | # scanning into tokens 111 | tokens = python_scanner.tokenize 'import this; # The Zen of Python' 112 | assert_equal ["import", :keyword, " ", :space, "this", :include, 113 | ";", :operator, " ", :space, "# The Zen of Python", :comment], tokens 114 | 115 | # format the tokens 116 | term = terminal_encoder.encode_tokens(tokens) 117 | assert_equal "\e[32mimport\e[0m \e[31mthis\e[0m; \e[1;30m# The Zen of Python\e[0m", term 118 | 119 | # re-using scanner and encoder 120 | ruby_highlighter = CodeRay::Duo[:ruby, :div] 121 | div = ruby_highlighter.encode('puts "Hello, world!"') 122 | assert_equal <<-DIV, div 123 |
124 | 
puts "Hello, world!"
125 |
126 | DIV 127 | end 128 | 129 | end 130 | -------------------------------------------------------------------------------- /lib/coderay/scanners/haml.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Scanners 3 | 4 | load :ruby 5 | load :html 6 | load :java_script 7 | 8 | class HAML < Scanner 9 | 10 | register_for :haml 11 | title 'HAML Template' 12 | 13 | KINDS_NOT_LOC = HTML::KINDS_NOT_LOC 14 | 15 | protected 16 | 17 | def setup 18 | super 19 | @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true 20 | @embedded_ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true, :state => @ruby_scanner.interpreted_string_state 21 | @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true 22 | end 23 | 24 | def scan_tokens encoder, options 25 | 26 | match = nil 27 | code = '' 28 | 29 | until eos? 30 | 31 | if bol? 32 | if match = scan(/!!!.*/) 33 | encoder.text_token match, :doctype 34 | next 35 | end 36 | 37 | if match = scan(/(?>( *)(\/(?!\[if)|-\#|:javascript|:ruby|:\w+) *)(?=\n)/) 38 | encoder.text_token match, :comment 39 | 40 | code = self[2] 41 | if match = scan(/(?:\n+#{self[1]} .*)+/) 42 | case code 43 | when '/', '-#' 44 | encoder.text_token match, :comment 45 | when ':javascript' 46 | # TODO: recognize #{...} snippets inside JavaScript 47 | @java_script_scanner ||= CodeRay.scanner :java_script, :tokens => @tokens, :keep_tokens => true 48 | @java_script_scanner.tokenize match, :tokens => encoder 49 | when ':ruby' 50 | @ruby_scanner.tokenize match, :tokens => encoder 51 | when /:\w+/ 52 | encoder.text_token match, :comment 53 | else 54 | raise 'else-case reached: %p' % [code] 55 | end 56 | end 57 | end 58 | 59 | if match = scan(/ +/) 60 | encoder.text_token match, :space 61 | end 62 | 63 | if match = scan(/\/.*/) 64 | encoder.text_token match, :comment 65 | next 66 | end 67 | 68 | if match = scan(/\\/) 69 | encoder.text_token match, :plain 70 | if match = scan(/.+/) 71 | @html_scanner.tokenize match, :tokens => encoder 72 | end 73 | next 74 | end 75 | 76 | tag = false 77 | 78 | if match = scan(/%[-\w:]+\/?/) 79 | encoder.text_token match, :tag 80 | # if match = scan(/( +)(.+)/) 81 | # encoder.text_token self[1], :space 82 | # @embedded_ruby_scanner.tokenize self[2], :tokens => encoder 83 | # end 84 | tag = true 85 | end 86 | 87 | while match = scan(/([.#])[-\w]*\w/) 88 | encoder.text_token match, self[1] == '#' ? :constant : :class 89 | tag = true 90 | end 91 | 92 | if tag && match = scan(/(\()([^)]+)?(\))?/) 93 | # TODO: recognize title=@title, class="widget_#{@widget.number}" 94 | encoder.text_token self[1], :plain 95 | @html_scanner.tokenize self[2], :tokens => encoder, :state => :attribute if self[2] 96 | encoder.text_token self[3], :plain if self[3] 97 | end 98 | 99 | if tag && match = scan(/\{/) 100 | encoder.text_token match, :plain 101 | 102 | code = '' 103 | level = 1 104 | while true 105 | code << scan(/([^\{\},\n]|, *\n?)*/) 106 | case match = getch 107 | when '{' 108 | level += 1 109 | code << match 110 | when '}' 111 | level -= 1 112 | if level > 0 113 | code << match 114 | else 115 | break 116 | end 117 | when "\n", ",", nil 118 | break 119 | end 120 | end 121 | @ruby_scanner.tokenize code, :tokens => encoder unless code.empty? 122 | 123 | encoder.text_token match, :plain if match 124 | end 125 | 126 | if tag && match = scan(/(\[)([^\]\n]+)?(\])?/) 127 | encoder.text_token self[1], :plain 128 | @ruby_scanner.tokenize self[2], :tokens => encoder if self[2] 129 | encoder.text_token self[3], :plain if self[3] 130 | end 131 | 132 | if tag && match = scan(/\//) 133 | encoder.text_token match, :tag 134 | end 135 | 136 | if scan(/(>? encoder 141 | else 142 | @ruby_scanner.tokenize self[4], :tokens => encoder 143 | end 144 | end 145 | elsif match = scan(/((?:<|> encoder if self[2] 149 | end 150 | 151 | elsif match = scan(/.+/) 152 | @html_scanner.tokenize match, :tokens => encoder 153 | 154 | end 155 | 156 | if match = scan(/\n/) 157 | encoder.text_token match, :space 158 | end 159 | end 160 | 161 | encoder 162 | 163 | end 164 | 165 | end 166 | 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /lib/coderay/encoders/terminal.rb: -------------------------------------------------------------------------------- 1 | module CodeRay 2 | module Encoders 3 | 4 | # Outputs code highlighted for a color terminal. 5 | # 6 | # Note: This encoder is in beta. It currently doesn't use the Styles. 7 | # 8 | # Alias: +term+ 9 | # 10 | # == Authors & License 11 | # 12 | # By Rob Aldred (http://robaldred.co.uk) 13 | # 14 | # Based on idea by Nathan Weizenbaum (http://nex-3.com) 15 | # 16 | # MIT License (http://www.opensource.org/licenses/mit-license.php) 17 | class Terminal < Encoder 18 | 19 | register_for :terminal 20 | 21 | TOKEN_COLORS = { 22 | :debug => "\e[1;37;44m", 23 | 24 | :annotation => "\e[34m", 25 | :attribute_name => "\e[35m", 26 | :attribute_value => "\e[31m", 27 | :binary => { 28 | :self => "\e[31m", 29 | :char => "\e[1;31m", 30 | :delimiter => "\e[1;31m", 31 | }, 32 | :char => { 33 | :self => "\e[35m", 34 | :delimiter => "\e[1;35m" 35 | }, 36 | :class => "\e[1;35;4m", 37 | :class_variable => "\e[36m", 38 | :color => "\e[32m", 39 | :comment => { 40 | :self => "\e[1;30m", 41 | :char => "\e[37m", 42 | :delimiter => "\e[37m", 43 | }, 44 | :constant => "\e[1;34;4m", 45 | :decorator => "\e[35m", 46 | :definition => "\e[1;33m", 47 | :directive => "\e[33m", 48 | :docstring => "\e[31m", 49 | :doctype => "\e[1;34m", 50 | :done => "\e[1;30;2m", 51 | :entity => "\e[31m", 52 | :error => "\e[1;37;41m", 53 | :exception => "\e[1;31m", 54 | :float => "\e[1;35m", 55 | :function => "\e[1;34m", 56 | :global_variable => "\e[1;32m", 57 | :hex => "\e[1;36m", 58 | :id => "\e[1;34m", 59 | :include => "\e[31m", 60 | :integer => "\e[1;34m", 61 | :imaginary => "\e[1;34m", 62 | :important => "\e[1;31m", 63 | :key => { 64 | :self => "\e[35m", 65 | :char => "\e[1;35m", 66 | :delimiter => "\e[1;35m", 67 | }, 68 | :keyword => "\e[32m", 69 | :label => "\e[1;33m", 70 | :local_variable => "\e[33m", 71 | :namespace => "\e[1;35m", 72 | :octal => "\e[1;34m", 73 | :predefined => "\e[36m", 74 | :predefined_constant => "\e[1;36m", 75 | :predefined_type => "\e[1;32m", 76 | :preprocessor => "\e[1;36m", 77 | :pseudo_class => "\e[1;34m", 78 | :regexp => { 79 | :self => "\e[35m", 80 | :delimiter => "\e[1;35m", 81 | :modifier => "\e[35m", 82 | :char => "\e[1;35m", 83 | }, 84 | :reserved => "\e[32m", 85 | :shell => { 86 | :self => "\e[33m", 87 | :char => "\e[1;33m", 88 | :delimiter => "\e[1;33m", 89 | :escape => "\e[1;33m", 90 | }, 91 | :string => { 92 | :self => "\e[31m", 93 | :modifier => "\e[1;31m", 94 | :char => "\e[1;35m", 95 | :delimiter => "\e[1;31m", 96 | :escape => "\e[1;31m", 97 | }, 98 | :symbol => { 99 | :self => "\e[33m", 100 | :delimiter => "\e[1;33m", 101 | }, 102 | :tag => "\e[32m", 103 | :type => "\e[1;34m", 104 | :value => "\e[36m", 105 | :variable => "\e[34m", 106 | 107 | :insert => { 108 | :self => "\e[42m", 109 | :insert => "\e[1;32;42m", 110 | :eyecatcher => "\e[102m", 111 | }, 112 | :delete => { 113 | :self => "\e[41m", 114 | :delete => "\e[1;31;41m", 115 | :eyecatcher => "\e[101m", 116 | }, 117 | :change => { 118 | :self => "\e[44m", 119 | :change => "\e[37;44m", 120 | }, 121 | :head => { 122 | :self => "\e[45m", 123 | :filename => "\e[37;45m" 124 | }, 125 | } 126 | 127 | TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved] 128 | TOKEN_COLORS[:method] = TOKEN_COLORS[:function] 129 | TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter] 130 | 131 | protected 132 | 133 | def setup(options) 134 | super 135 | @opened = [] 136 | @color_scopes = [TOKEN_COLORS] 137 | end 138 | 139 | public 140 | 141 | def text_token text, kind 142 | if color = @color_scopes.last[kind] 143 | color = color[:self] if color.is_a? Hash 144 | 145 | @out << color 146 | @out << (text.index("\n") ? text.gsub("\n", "\e[0m\n" + color) : text) 147 | @out << "\e[0m" 148 | if outer_color = @color_scopes.last[:self] 149 | @out << outer_color 150 | end 151 | else 152 | @out << text 153 | end 154 | end 155 | 156 | def begin_group kind 157 | @opened << kind 158 | @out << open_token(kind) 159 | end 160 | alias begin_line begin_group 161 | 162 | def end_group kind 163 | if @opened.pop 164 | @color_scopes.pop 165 | @out << "\e[0m" 166 | if outer_color = @color_scopes.last[:self] 167 | @out << outer_color 168 | end 169 | end 170 | end 171 | 172 | def end_line kind 173 | @out << (@line_filler ||= "\t" * 100) 174 | end_group kind 175 | end 176 | 177 | private 178 | 179 | def open_token kind 180 | if color = @color_scopes.last[kind] 181 | if color.is_a? Hash 182 | @color_scopes << color 183 | color[:self] 184 | else 185 | @color_scopes << @color_scopes.last 186 | color 187 | end 188 | else 189 | @color_scopes << @color_scopes.last 190 | '' 191 | end 192 | end 193 | end 194 | end 195 | end 196 | --------------------------------------------------------------------------------